diff --git a/manifest b/manifest index 336d04f480..1fd0e675c0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sspellfix\svirtual\stable\sto\soptimize\squeries\sof\sthe\sform\s"SELECT\s...\sFROM\stbl\sWHERE\srowid=?". -D 2014-01-22T17:43:16.604 +C Change\sthe\sWITH\sRECURSIVE\simplementation\sto\suse\sa\squeue\sinstead\sof\sa\spair\sof\ntables.\s\sAdd\ssupport\sfor\sORDER\sBY,\sLIMIT,\sand\sOFFSET\son\srecursive\squeries. +D 2014-01-22T18:16:27.891 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,12 +219,12 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c a27ac21844df3123b7c1e89d79cd7034d4eb0e8e -F src/shell.c 9f3bc02a658b8f61d2cbe60cfc482f660c1c6c48 +F src/select.c f6d84f3a109d3e43d38089da6a4f131a5ce4c6ef +F src/shell.c 24722d24d4ea8ca93db35e44db7308de786767ca F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 99fd628541e420b98fc52072635e8ba431706250 +F src/sqliteInt.h 87a90ad4818ac5d68d3463eb7fe3ed96e5209b25 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -280,7 +280,7 @@ F src/update.c c2706a6eb232a96345c35b7e1e75a188e26812bb F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c e71f19b272f05c8695cf747b4bac1732685f9e5c F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 98d96d04d9a2bef78ca850be1053dc91d031338a +F src/vdbe.c dede894c2990329f8bc5a70da7de44ce8c3c6bf5 F src/vdbe.h e6c4c610fcabad4fa80ebb1efc6822a9367e2b26 F src/vdbeInt.h 42db251e9f863401ff847b90d5fe1614c89a6a56 F src/vdbeapi.c ce4e68ea4842cc6081046f533d088dcf01d247ad @@ -293,7 +293,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 56f85486bc8d0cb57fc15e5db2a58d1dfa1114cf +F src/where.c 67ae3b5e97ecff36c70cb61ccc7d74cf228f1596 F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1092,7 +1092,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d -F test/with1.test cec63b56797a70842afa8929c241dfdb3d864283 +F test/with1.test 97166cc72de5327bbae782aece707c45ee40e41b F test/with2.test 2fe78fcd8deef2a0f9cfc49bfc755911d0b3fd64 F test/withM.test e97f2a8c506ab3ea9eab94e6f6072f6cc924c991 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 @@ -1152,7 +1152,8 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5e43bf013253921e4dfbe71de11ee7ed4b3e7eae -R d2ae80bd066cad79603d23bc83a10a19 -U dan -Z 90dc7fd11fe48035054a3857f5aa405e +P a0ba55ff0596c5f15e9cdb254c68ef50df2dfaad 1945484e6b9769c1943f750f5b09860417fb190a +R 0105f6ff7a9dec4a544ebd63835ab3ca +T +closed 1945484e6b9769c1943f750f5b09860417fb190a +U drh +Z c6c3dc0c740728b9edf5697c95361334 diff --git a/manifest.uuid b/manifest.uuid index 3b856a138c..6b5e670c10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0ba55ff0596c5f15e9cdb254c68ef50df2dfaad \ No newline at end of file +b6cea42006910d590373e8f9e296d7672edb114b \ No newline at end of file diff --git a/src/select.c b/src/select.c index dd58ed22e6..891bbf7604 100644 --- a/src/select.c +++ b/src/select.c @@ -462,13 +462,13 @@ static void pushOntoSorter( */ static void codeOffset( Vdbe *v, /* Generate code into this VM */ - Select *p, /* The SELECT statement being coded */ + int iOffset, /* Register holding the offset counter */ int iContinue /* Jump here to skip the current record */ ){ - if( p->iOffset && iContinue!=0 ){ + if( iOffset>0 && iContinue!=0 ){ int addr; - sqlite3VdbeAddOp2(v, OP_AddImm, p->iOffset, -1); - addr = sqlite3VdbeAddOp1(v, OP_IfNeg, p->iOffset); + sqlite3VdbeAddOp2(v, OP_AddImm, iOffset, -1); + addr = sqlite3VdbeAddOp1(v, OP_IfNeg, iOffset); sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); VdbeComment((v, "skip OFFSET records")); sqlite3VdbeJumpHere(v, addr); @@ -543,17 +543,16 @@ struct DistinctCtx { ** This routine generates the code for the inside of the inner loop ** of a SELECT. ** -** If srcTab and nColumn are both zero, then the pEList expressions -** are evaluated in order to get the data for this row. If nColumn>0 -** then data is pulled from srcTab and pEList is used only to get the -** datatypes for each column. +** If srcTab is negative, then the pEList expressions +** are evaluated in order to get the data for this row. If srcTab is +** zero or more, then data is pulled from srcTab and pEList is used only +** to get number columns and the datatype for each column. */ static void selectInnerLoop( Parse *pParse, /* The parser context */ Select *p, /* The complete select statement being coded */ ExprList *pEList, /* List of values being extracted */ int srcTab, /* Pull data from this table */ - int nColumn, /* Number of columns in the source table */ ExprList *pOrderBy, /* If not NULL, sort results using this key */ DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ SelectDest *pDest, /* How to dispose of the results */ @@ -569,20 +568,15 @@ static void selectInnerLoop( int nResultCol; /* Number of result columns */ assert( v ); - if( NEVER(v==0) ) return; assert( pEList!=0 ); hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; if( pOrderBy==0 && !hasDistinct ){ - codeOffset(v, p, iContinue); + codeOffset(v, p->iOffset, iContinue); } /* Pull the requested columns. */ - if( nColumn>0 ){ - nResultCol = nColumn; - }else{ - nResultCol = pEList->nExpr; - } + nResultCol = pEList->nExpr; if( pDest->iSdst==0 ){ pDest->iSdst = pParse->nMem+1; pDest->nSdst = nResultCol; @@ -591,9 +585,10 @@ static void selectInnerLoop( assert( pDest->nSdst==nResultCol ); } regResult = pDest->iSdst; - if( nColumn>0 ){ - for(i=0; i=0 ){ + for(i=0; ia[i].zName)); } }else if( eDest!=SRT_Exists ){ /* If the destination is an EXISTS(...) expression, the actual @@ -602,15 +597,12 @@ static void selectInnerLoop( sqlite3ExprCodeExprList(pParse, pEList, regResult, (eDest==SRT_Output)?SQLITE_ECEL_DUP:0); } - nColumn = nResultCol; /* If the DISTINCT keyword was present on the SELECT statement ** and this row has been seen before, then do not make this row ** part of the result. */ if( hasDistinct ){ - assert( pEList!=0 ); - assert( pEList->nExpr==nColumn ); switch( pDistinct->eTnctType ){ case WHERE_DISTINCT_ORDERED: { VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ @@ -619,7 +611,7 @@ static void selectInnerLoop( /* Allocate space for the previous row */ regPrev = pParse->nMem+1; - pParse->nMem += nColumn; + pParse->nMem += nResultCol; /* Change the OP_OpenEphemeral coded earlier to an OP_Null ** sets the MEM_Cleared bit on the first register of the @@ -633,10 +625,10 @@ static void selectInnerLoop( pOp->p1 = 1; pOp->p2 = regPrev; - iJump = sqlite3VdbeCurrentAddr(v) + nColumn; - for(i=0; ia[i].pExpr); - if( ieTnctType==WHERE_DISTINCT_UNORDERED ); - codeDistinct(pParse, pDistinct->tabTnct, iContinue, nColumn, regResult); + codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, regResult); break; } } if( pOrderBy==0 ){ - codeOffset(v, p, iContinue); + codeOffset(v, p->iOffset, iContinue); } } @@ -673,7 +665,7 @@ static void selectInnerLoop( case SRT_Union: { int r1; r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); sqlite3ReleaseTempReg(pParse, r1); break; @@ -684,10 +676,10 @@ static void selectInnerLoop( ** the temporary table iParm. */ case SRT_Except: { - sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nColumn); + sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nResultCol); break; } -#endif +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ /* Store the result as data using a unique key. */ @@ -697,7 +689,7 @@ static void selectInnerLoop( int r1 = sqlite3GetTempReg(pParse); testcase( eDest==SRT_Table ); testcase( eDest==SRT_EphemTab ); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistTable ){ /* If the destination is DistTable, then cursor (iParm+1) is open @@ -730,7 +722,7 @@ static void selectInnerLoop( ** item into the set table with bogus data. */ case SRT_Set: { - assert( nColumn==1 ); + assert( nResultCol==1 ); pDest->affSdst = sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); if( pOrderBy ){ @@ -762,7 +754,7 @@ static void selectInnerLoop( ** of the scan loop. */ case SRT_Mem: { - assert( nColumn==1 ); + assert( nResultCol==1 ); if( pOrderBy ){ pushOntoSorter(pParse, pOrderBy, p, regResult); }else{ @@ -783,18 +775,63 @@ static void selectInnerLoop( testcase( eDest==SRT_Output ); if( pOrderBy ){ int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); pushOntoSorter(pParse, pOrderBy, p, r1); sqlite3ReleaseTempReg(pParse, r1); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ - sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nColumn); - sqlite3ExprCacheAffinityChange(pParse, regResult, nColumn); + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nResultCol); + sqlite3ExprCacheAffinityChange(pParse, regResult, nResultCol); } break; } +#ifndef SQLITE_OMIT_CTE + /* Write the results into a priority queue that is order according to + ** pDest->pOrderBy (in pSO). pDest->iSDParm (in iParm) is the cursor for an + ** index with pSO->nExpr+2 columns. Build a key using pSO for the first + ** pSO->nExpr columns, then make sure all keys are unique by adding a + ** final OP_Sequence column. The last column is the record as a blob. + */ + case SRT_DistQueue: + case SRT_Queue: { + int nKey; + int r1, r2, r3; + int addrTest = 0; + ExprList *pSO; + pSO = pDest->pOrderBy; + assert( pSO ); + nKey = pSO->nExpr; + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempRange(pParse, nKey+2); + r3 = r2+nKey+1; + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r3); + if( eDest==SRT_DistQueue ){ + /* If the destination is DistQueue, then cursor (iParm+1) is open + ** on a second ephemeral index that holds all values every previously + ** added to the queue. Only add this new value if it has never before + ** been added */ + addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, r3, 0); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3); + } + for(i=0; ia[i].u.x.iOrderByCol - 1, + r2+i); + } + sqlite3VdbeAddOp2(v, OP_Sequence, iParm, r2+nKey); + sqlite3VdbeAddOp3(v, OP_MakeRecord, r2, nKey+2, r1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + if( addrTest ) sqlite3VdbeJumpHere(v, addrTest); + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r2, nKey+2); + break; + } +#endif /* SQLITE_OMIT_CTE */ + + + #if !defined(SQLITE_OMIT_TRIGGER) /* Discard the results. This is used for SELECT statements inside ** the body of a TRIGGER. The purpose of such selects is to call @@ -883,7 +920,7 @@ int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; } ** function is responsible for seeing that this structure is eventually ** freed. */ -static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList){ +static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList, int nExtra){ int nExpr; KeyInfo *pInfo; struct ExprList_item *pItem; @@ -891,7 +928,7 @@ static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList){ int i; nExpr = pList->nExpr; - pInfo = sqlite3KeyInfoAlloc(db, nExpr, 1); + pInfo = sqlite3KeyInfoAlloc(db, nExpr+nExtra, 1); if( pInfo ){ assert( sqlite3KeyInfoIsWriteable(pInfo) ); for(i=0, pItem=pList->a; inTab++; sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, pOrderBy->nExpr+2); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); - codeOffset(v, p, addrContinue); + codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); sqlite3VdbeAddOp3(v, OP_Column, ptab2, pOrderBy->nExpr+1, regRow); sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); }else{ addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); - codeOffset(v, p, addrContinue); + codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr+1, regRow); } switch( eDest ){ @@ -1602,8 +1639,13 @@ Vdbe *sqlite3GetVdbe(Parse *pParse){ ** ** This routine changes the values of iLimit and iOffset only if ** a limit or offset is defined by pLimit and pOffset. iLimit and -** iOffset should have been preset to appropriate default values -** (usually but not always -1) prior to calling this routine. +** iOffset should have been preset to appropriate default values (zero) +** prior to calling this routine. +** +** The iOffset register (if it exists) is initialized to the value +** of the OFFSET. The iLimit register is initialized to LIMIT. Register +** iOffset+1 is initialized to LIMIT+OFFSET. +** ** Only if pLimit!=0 or pOffset!=0 do the limit registers get ** redefined. The UNION ALL operator uses this property to force ** the reuse of the same limit and offset registers across multiple @@ -1627,7 +1669,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ if( p->pLimit ){ p->iLimit = iLimit = ++pParse->nMem; v = sqlite3GetVdbe(pParse); - if( NEVER(v==0) ) return; /* VDBE should have already been allocated */ + assert( v!=0 ); if( sqlite3ExprIsInteger(p->pLimit, &n) ){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); @@ -1684,7 +1726,165 @@ static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ } #endif /* SQLITE_OMIT_COMPOUND_SELECT */ -/* Forward reference */ +#ifndef SQLITE_OMIT_CTE +/* +** This routine generates VDBE code to compute the content of a WITH RECURSIVE +** query of the form: +** +** AS ( UNION [ALL] ) +** \___________/ \_______________/ +** p->pPrior p +** +** +** There is exactly one reference to the recursive-table in the FROM clause +** of recursive-query, marked with the SrcList->a[].isRecursive flag. +** +** The setup-query runs once to generate an initial set of rows that go +** into a Queue table. Rows are extracted from the Queue table one by +** one. Each row extracted from Queue is output to pDest. Then the single +** extracted row (now in the iCurrent table) becomes the content of the +** recursive-table for a recursive-query run. The output of the recursive-query +** is added back into the Queue table. Then another row is extracted from Queue +** and the iteration continues until the Queue table is empty. +** +** If the compound query operator is UNION then no duplicate rows are ever +** inserted into the Queue table. The iDistinct table keeps a copy of all rows +** that have ever been inserted into Queue and causes duplicates to be +** discarded. If the operator is UNION ALL, then duplicates are allowed. +** +** If the query has an ORDER BY, then entries in the Queue table are kept in +** ORDER BY order and the first entry is extracted for each cycle. Without +** an ORDER BY, the Queue table is just a FIFO. +** +** If a LIMIT clause is provided, then the iteration stops after LIMIT rows +** have been output to pDest. A LIMIT of zero means to output no rows and a +** negative LIMIT means to output all rows. If there is also an OFFSET clause +** with a positive value, then the first OFFSET outputs are discarded rather +** than being sent to pDest. The LIMIT count does not begin until after OFFSET +** rows have been skipped. +*/ +static void generateWithRecursiveQuery( + Parse *pParse, /* Parsing context */ + Select *p, /* The recursive SELECT to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + SrcList *pSrc = p->pSrc; /* The FROM clause of the recursive query */ + int nCol = p->pEList->nExpr; /* Number of columns in the recursive table */ + Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ + Select *pSetup = p->pPrior; /* The setup query */ + int addrTop; /* Top of the loop */ + int addrCont, addrBreak; /* CONTINUE and BREAK addresses */ + int iCurrent; /* The Current table */ + int regCurrent; /* Register holding Current table */ + int iQueue; /* The Queue table */ + int iDistinct = 0; /* To ensure unique results if UNION */ + int eDest = SRT_Table; /* How to write to Queue */ + SelectDest destQueue; /* SelectDest targetting the Queue table */ + int i; /* Loop counter */ + int rc; /* Result code */ + ExprList *pOrderBy; /* The ORDER BY clause */ + Expr *pLimit, *pOffset; /* Saved LIMIT and OFFSET */ + int regLimit, regOffset; /* Registers used by LIMIT and OFFSET */ + + /* Obtain authorization to do a recursive query */ + if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ) return; + + /* Process the LIMIT and OFFSET clauses, if they exist */ + addrBreak = sqlite3VdbeMakeLabel(v); + computeLimitRegisters(pParse, p, addrBreak); + pLimit = p->pLimit; + pOffset = p->pOffset; + regLimit = p->iLimit; + regOffset = p->iOffset; + p->pLimit = p->pOffset = 0; + p->iLimit = p->iOffset = 0; + + /* Locate the cursor number of the Current table */ + for(i=0; ALWAYS(inSrc); i++){ + if( pSrc->a[i].isRecursive ){ + iCurrent = pSrc->a[i].iCursor; + break; + } + } + + /* Detach the ORDER BY clause from the compound SELECT */ + pOrderBy = p->pOrderBy; + p->pOrderBy = 0; + + /* Allocate cursors numbers for Queue and Distinct. The cursor number for + ** the Distinct table must be exactly one greater than Queue in order + ** for the SRT_DistTable and SRT_DistQueue destinations to work. */ + iQueue = pParse->nTab++; + if( p->op==TK_UNION ){ + eDest = pOrderBy ? SRT_DistQueue : SRT_DistTable; + iDistinct = pParse->nTab++; + }else{ + eDest = pOrderBy ? SRT_Queue : SRT_Table; + } + sqlite3SelectDestInit(&destQueue, eDest, iQueue); + + /* Allocate cursors for Current, Queue, and Distinct. */ + regCurrent = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol); + if( pOrderBy ){ + KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pOrderBy, 1); + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iQueue, pOrderBy->nExpr+2, 0, + (char*)pKeyInfo, P4_KEYINFO); + destQueue.pOrderBy = pOrderBy; + }else{ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iQueue, nCol); + } + VdbeComment((v, "Queue table")); + if( iDistinct ){ + p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0); + p->selFlags |= SF_UsesEphemeral; + } + + /* Store the results of the setup-query in Queue. */ + rc = sqlite3Select(pParse, pSetup, &destQueue); + if( rc ) goto end_of_recursive_query; + + /* Find the next row in the Queue and output that row */ + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); + + /* Transfer the next row in Queue over to Current */ + sqlite3VdbeAddOp1(v, OP_NullRow, iCurrent); /* To reset column cache */ + if( pOrderBy ){ + sqlite3VdbeAddOp3(v, OP_Column, iQueue, pOrderBy->nExpr+1, regCurrent); + }else{ + sqlite3VdbeAddOp2(v, OP_RowData, iQueue, regCurrent); + } + sqlite3VdbeAddOp1(v, OP_Delete, iQueue); + + /* Output the single row in Current */ + addrCont = sqlite3VdbeMakeLabel(v); + codeOffset(v, regOffset, addrCont); + selectInnerLoop(pParse, p, p->pEList, iCurrent, + 0, 0, pDest, addrCont, addrBreak); + if( regLimit ) sqlite3VdbeAddOp3(v, OP_IfZero, regLimit, addrBreak, -1); + sqlite3VdbeResolveLabel(v, addrCont); + + /* Execute the recursive SELECT taking the single row in Current as + ** the value for the recursive-table. Store the results in the Queue. + */ + p->pPrior = 0; + sqlite3Select(pParse, p, &destQueue); + assert( p->pPrior==0 ); + p->pPrior = pSetup; + + /* Keep running the loop until the Queue is empty */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); + sqlite3VdbeResolveLabel(v, addrBreak); + +end_of_recursive_query: + p->pOrderBy = pOrderBy; + p->pLimit = pLimit; + p->pOffset = pOffset; + return; +} +#endif + +/* Forward references */ static int multiSelectOrderBy( Parse *pParse, /* Parsing context */ Select *p, /* The right-most of SELECTs to be coded */ @@ -1792,81 +1992,7 @@ static int multiSelect( #ifndef SQLITE_OMIT_CTE if( p->selFlags & SF_Recursive ){ - SrcList *pSrc = p->pSrc; - int nCol = p->pEList->nExpr; - int addrNext; - int addrSwap; - int iCont, iBreak; - int tmp1; /* Intermediate table */ - int tmp2; /* Next intermediate table */ - int tmp3 = 0; /* To ensure unique results if UNION */ - int eDest = SRT_Table; - SelectDest tmp2dest; - int i; - - /* Check that there is no ORDER BY or LIMIT clause. Neither of these - ** are supported on recursive queries. */ - assert( p->pOffset==0 || p->pLimit ); - if( p->pOrderBy || p->pLimit ){ - sqlite3ErrorMsg(pParse, "%s in a recursive query", - p->pOrderBy ? "ORDER BY" : "LIMIT" - ); - goto multi_select_end; - } - - if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ){ - goto multi_select_end; - } - iBreak = sqlite3VdbeMakeLabel(v); - iCont = sqlite3VdbeMakeLabel(v); - - for(i=0; ALWAYS(inSrc); i++){ - if( pSrc->a[i].isRecursive ){ - tmp1 = pSrc->a[i].iCursor; - break; - } - } - - tmp2 = pParse->nTab++; - if( p->op==TK_UNION ){ - eDest = SRT_DistTable; - tmp3 = pParse->nTab++; - } - sqlite3SelectDestInit(&tmp2dest, eDest, tmp2); - - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp1, nCol); - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp2, nCol); - if( tmp3 ){ - p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp3, 0); - p->selFlags |= SF_UsesEphemeral; - } - - /* Store the results of the initial SELECT in tmp2. */ - rc = sqlite3Select(pParse, pPrior, &tmp2dest); - if( rc ) goto multi_select_end; - - /* Clear tmp1. Then switch the contents of tmp1 and tmp2. Then return - ** the contents of tmp1 to the caller. Or, if tmp1 is empty at this - ** point, the recursive query has finished - jump to address iBreak. */ - addrSwap = sqlite3VdbeAddOp2(v, OP_SwapCursors, tmp1, tmp2); - sqlite3VdbeAddOp2(v, OP_Rewind, tmp1, iBreak); - addrNext = sqlite3VdbeCurrentAddr(v); - selectInnerLoop(pParse, p, p->pEList, tmp1, p->pEList->nExpr, - 0, 0, &dest, iCont, iBreak); - sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, tmp1, addrNext); - - /* Execute the recursive SELECT. Store the results in tmp2. While this - ** SELECT is running, the contents of tmp1 are read by recursive - ** references to the current CTE. */ - p->pPrior = 0; - rc = sqlite3Select(pParse, p, &tmp2dest); - assert( p->pPrior==0 ); - p->pPrior = pPrior; - if( rc ) goto multi_select_end; - - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrSwap); - sqlite3VdbeResolveLabel(v, iBreak); + generateWithRecursiveQuery(pParse, p, &dest); }else #endif @@ -2009,7 +2135,7 @@ static int multiSelect( computeLimitRegisters(pParse, p, iBreak); sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); iStart = sqlite3VdbeCurrentAddr(v); - selectInnerLoop(pParse, p, p->pEList, unionTab, p->pEList->nExpr, + selectInnerLoop(pParse, p, p->pEList, unionTab, 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); @@ -2087,7 +2213,7 @@ static int multiSelect( iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1); sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); sqlite3ReleaseTempReg(pParse, r1); - selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr, + selectInnerLoop(pParse, p, p->pEList, tab1, 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); @@ -2209,7 +2335,7 @@ static int generateOutputSubroutine( /* Suppress the first OFFSET entries if there is an OFFSET clause */ - codeOffset(v, p, iContinue); + codeOffset(v, p->iOffset, iContinue); switch( pDest->eDest ){ /* Store the result as data using a unique key. @@ -4155,7 +4281,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ "argument"); pFunc->iDistinct = -1; }else{ - KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList); + KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList, 0); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO); } @@ -4288,50 +4414,8 @@ static void explainSimpleCount( /* ** Generate code for the SELECT statement given in the p argument. ** -** The results are distributed in various ways depending on the -** contents of the SelectDest structure pointed to by argument pDest -** as follows: -** -** pDest->eDest Result -** ------------ ------------------------------------------- -** SRT_Output Generate a row of output (using the OP_ResultRow -** opcode) for each row in the result set. -** -** SRT_Mem Only valid if the result is a single column. -** Store the first column of the first result row -** in register pDest->iSDParm then abandon the rest -** of the query. This destination implies "LIMIT 1". -** -** SRT_Set The result must be a single column. Store each -** row of result as the key in table pDest->iSDParm. -** Apply the affinity pDest->affSdst before storing -** results. Used to implement "IN (SELECT ...)". -** -** SRT_Union Store results as a key in a temporary table -** identified by pDest->iSDParm. -** -** SRT_Except Remove results from the temporary table pDest->iSDParm. -** -** SRT_Table Store results in temporary table pDest->iSDParm. -** This is like SRT_EphemTab except that the table -** is assumed to already be open. -** -** SRT_EphemTab Create an temporary table pDest->iSDParm and store -** the result there. The cursor is left open after -** returning. This is like SRT_Table except that -** this destination uses OP_OpenEphemeral to create -** the table first. -** -** SRT_Coroutine Generate a co-routine that returns a new row of -** results each time it is invoked. The entry point -** of the co-routine is stored in register pDest->iSDParm. -** -** SRT_Exists Store a 1 in memory cell pDest->iSDParm if the result -** set is not empty. -** -** SRT_Discard Throw the results away. This is used by SELECT -** statements within triggers whose only purpose is -** the side-effects of functions. +** The results are returned according to the SelectDest structure. +** See comments in sqliteInt.h for further information. ** ** This routine returns the number of errors. If any errors are ** encountered, then an appropriate error message is left in @@ -4606,7 +4690,7 @@ int sqlite3Select( */ if( pOrderBy ){ KeyInfo *pKeyInfo; - pKeyInfo = keyInfoFromExprList(pParse, pOrderBy); + pKeyInfo = keyInfoFromExprList(pParse, pOrderBy, 0); pOrderBy->iECursor = pParse->nTab++; p->addrOpenEphm[2] = addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, @@ -4638,7 +4722,7 @@ int sqlite3Select( sDistinct.tabTnct = pParse->nTab++; sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, sDistinct.tabTnct, 0, 0, - (char*)keyInfoFromExprList(pParse, p->pEList), + (char*)keyInfoFromExprList(pParse, p->pEList, 0), P4_KEYINFO); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; @@ -4672,7 +4756,7 @@ int sqlite3Select( } /* Use the standard inner loop. */ - selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, &sDistinct, pDest, + selectInnerLoop(pParse, p, pEList, -1, pOrderBy, &sDistinct, pDest, sqlite3WhereContinueLabel(pWInfo), sqlite3WhereBreakLabel(pWInfo)); @@ -4762,7 +4846,7 @@ int sqlite3Select( ** will be converted into a Noop. */ sAggInfo.sortingIdx = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, pGroupBy); + pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0); addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, sAggInfo.sortingIdx, sAggInfo.nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO); @@ -4944,7 +5028,7 @@ int sqlite3Select( sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, &sAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, 0, 0, pOrderBy, + selectInnerLoop(pParse, p, p->pEList, -1, pOrderBy, &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); @@ -5087,7 +5171,7 @@ int sqlite3Select( pOrderBy = 0; sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, 0, 0, 0, 0, + selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, pDest, addrEnd, addrEnd); sqlite3ExprListDelete(db, pDel); } diff --git a/src/shell.c b/src/shell.c index b5ce90208c..1c4c4ad3e3 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1177,7 +1177,7 @@ static int str_in_array(const char *zStr, const char **azArray){ ** ** * For each "Goto", if the jump destination is earlier in the program ** and ends on one of: -** Yield SeekGt SeekLt RowSetRead +** Yield SeekGt SeekLt RowSetRead Rewind ** then indent all opcodes between the earlier instruction ** and "Goto" by 2 spaces. */ @@ -1189,7 +1189,7 @@ static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){ int iOp; /* Index of operation in p->aiIndent[] */ const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", 0 }; - const char *azYield[] = { "Yield", "SeekLt", "SeekGt", "RowSetRead", 0 }; + const char *azYield[] = { "Yield", "SeekLt", "SeekGt", "RowSetRead", "Rewind", 0 }; const char *azGoto[] = { "Goto", 0 }; /* Try to figure out if this is really an EXPLAIN statement. If this @@ -1226,7 +1226,7 @@ static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){ for(i=p2op; iaiIndent[i] += 2; } if( str_in_array(zOp, azGoto) && p2opnIndent && abYield[p2op] ){ - for(i=p2op; iaiIndent[i] += 2; + for(i=p2op+1; iaiIndent[i] += 2; } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 317e9eb227..c1fa7b69ad 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2167,8 +2167,66 @@ struct Select { /* -** The results of a select can be distributed in several ways. The -** "SRT" prefix means "SELECT Result Type". +** The results of a SELECT can be distributed in several ways, as defined +** by one of the following macros. The "SRT" prefix means "SELECT Result +** Type". +** +** SRT_Union Store results as a key in a temporary index +** identified by pDest->iSDParm. +** +** SRT_Except Remove results from the temporary index pDest->iSDParm. +** +** SRT_Exists Store a 1 in memory cell pDest->iSDParm if the result +** set is not empty. +** +** SRT_Discard Throw the results away. This is used by SELECT +** statements within triggers whose only purpose is +** the side-effects of functions. +** +** All of the above are free to ignore their ORDER BY clause. Those that +** follow must honor the ORDER BY clause. +** +** SRT_Output Generate a row of output (using the OP_ResultRow +** opcode) for each row in the result set. +** +** SRT_Mem Only valid if the result is a single column. +** Store the first column of the first result row +** in register pDest->iSDParm then abandon the rest +** of the query. This destination implies "LIMIT 1". +** +** SRT_Set The result must be a single column. Store each +** row of result as the key in table pDest->iSDParm. +** Apply the affinity pDest->affSdst before storing +** results. Used to implement "IN (SELECT ...)". +** +** SRT_EphemTab Create an temporary table pDest->iSDParm and store +** the result there. The cursor is left open after +** returning. This is like SRT_Table except that +** this destination uses OP_OpenEphemeral to create +** the table first. +** +** SRT_Coroutine Generate a co-routine that returns a new row of +** results each time it is invoked. The entry point +** of the co-routine is stored in register pDest->iSDParm +** and the result row is stored in pDest->nDest registers +** starting with pDest->iSdst. +** +** SRT_Table Store results in temporary table pDest->iSDParm. +** This is like SRT_EphemTab except that the table +** is assumed to already be open. +** +** SRT_DistTable Store results in a temporary table pDest->iSDParm. +** But also use temporary table pDest->iSDParm+1 as +** a record of all prior results and ignore any duplicate +** rows. Name means: "Distinct Table". +** +** SRT_Queue Store results in priority queue pDest->iSDParm (really +** an index). Append a sequence number so that all entries +** are distinct. +** +** SRT_DistQueue Store results in priority queue pDest->iSDParm only if +** the same record has never been stored before. The +** index at pDest->iSDParm+1 hold all prior stores. */ #define SRT_Union 1 /* Store result as keys in an index */ #define SRT_Except 2 /* Remove result from a UNION index */ @@ -2181,21 +2239,24 @@ struct Select { #define SRT_Output 5 /* Output each row of result */ #define SRT_Mem 6 /* Store result in a memory cell */ #define SRT_Set 7 /* Store results as keys in an index */ -#define SRT_Table 8 /* Store result as data with an automatic rowid */ -#define SRT_EphemTab 9 /* Create transient tab and store like SRT_Table */ -#define SRT_Coroutine 10 /* Generate a single row of result */ -#define SRT_DistTable 11 /* Like SRT_TABLE, but unique results only */ +#define SRT_EphemTab 8 /* Create transient tab and store like SRT_Table */ +#define SRT_Coroutine 9 /* Generate a single row of result */ +#define SRT_Table 10 /* Store result as data with an automatic rowid */ +#define SRT_DistTable 11 /* Like SRT_Table, but unique results only */ +#define SRT_Queue 12 /* Store result in an queue */ +#define SRT_DistQueue 13 /* Like SRT_Queue, but unique results only */ /* ** An instance of this object describes where to put of the results of ** a SELECT statement. */ struct SelectDest { - u8 eDest; /* How to dispose of the results. On of SRT_* above. */ - char affSdst; /* Affinity used when eDest==SRT_Set */ - int iSDParm; /* A parameter used by the eDest disposal method */ - int iSdst; /* Base register where results are written */ - int nSdst; /* Number of registers allocated */ + u8 eDest; /* How to dispose of the results. On of SRT_* above. */ + char affSdst; /* Affinity used when eDest==SRT_Set */ + int iSDParm; /* A parameter used by the eDest disposal method */ + int iSdst; /* Base register where results are written */ + int nSdst; /* Number of registers allocated */ + ExprList *pOrderBy; /* Key columns for SRT_Queue and SRT_DistQueue */ }; /* diff --git a/src/vdbe.c b/src/vdbe.c index 86aae3c65d..10a70750e3 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3369,33 +3369,6 @@ case OP_OpenEphemeral: { break; } -#ifndef SQLITE_OMIT_CTE -/* Opcode: SwapCursors P1 P2 * * * -** -** Parameters P1 and P2 are both cursors opened by the OpenEphemeral -** opcode. This opcode deletes the contents of epheremal table P1, -** then renames P2 to P1 and P1 to P2. In other words, following this -** opcode cursor P2 is open on an empty table and P1 is open on the -** table that was initially accessed by P2. -*/ -case OP_SwapCursors: { - Mem tmp; - VdbeCursor *pTmp; - - tmp = p->aMem[p->nMem - pOp->p1]; - p->aMem[p->nMem - pOp->p1] = p->aMem[p->nMem - pOp->p2]; - p->aMem[p->nMem - pOp->p2] = tmp; - - pTmp = p->apCsr[pOp->p1]; - p->apCsr[pOp->p1] = p->apCsr[pOp->p2]; - p->apCsr[pOp->p2] = pTmp; - - assert( pTmp->isTable ); - rc = sqlite3BtreeClearTable(pTmp->pBt, MASTER_ROOT, 0); - break; -} -#endif /* ifndef SQLITE_OMIT_CTE */ - /* Opcode: SorterOpen P1 * * P4 * ** ** This opcode works like OP_OpenEphemeral except that it opens @@ -4393,7 +4366,6 @@ case OP_NullRow: { pC->nullRow = 1; pC->rowidIsValid = 0; pC->cacheStatus = CACHE_STALE; - assert( pC->pCursor || pC->pVtabCursor ); if( pC->pCursor ){ sqlite3BtreeClearCursor(pC->pCursor); } diff --git a/src/where.c b/src/where.c index 86376c4955..6fac5e5ed1 100644 --- a/src/where.c +++ b/src/where.c @@ -3410,10 +3410,16 @@ static Bitmask codeOneLoopStart( static const u8 aStep[] = { OP_Next, OP_Prev }; static const u8 aStart[] = { OP_Rewind, OP_Last }; assert( bRev==0 || bRev==1 ); - pLevel->op = aStep[bRev]; - pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + if( pTabItem->isRecursive ){ + /* Tables marked isRecursive have only a single row that is stored in + ** a pseudo-cursor. No need to Rewind or Next such cursors. */ + pLevel->op = OP_Noop; + }else{ + pLevel->op = aStep[bRev]; + pLevel->p1 = iCur; + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + } } /* Insert code to test every subexpression that can be completely diff --git a/test/with1.test b/test/with1.test index fcf59267f2..fcd1929277 100644 --- a/test/with1.test +++ b/test/with1.test @@ -152,12 +152,60 @@ do_execsql_test 5.1 { do_catchsql_test 5.2 { WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i ORDER BY 1) SELECT x FROM i LIMIT 10; -} {1 {ORDER BY in a recursive query}} +} {0 {1 2 3 4 5 6 7 8 9 10}} + +do_execsql_test 5.2.1 { + CREATE TABLE edge(xfrom, xto, seq, PRIMARY KEY(xfrom, xto)) WITHOUT ROWID; + INSERT INTO edge VALUES(0, 1, 10); + INSERT INTO edge VALUES(1, 2, 20); + INSERT INTO edge VALUES(0, 3, 30); + INSERT INTO edge VALUES(2, 4, 40); + INSERT INTO edge VALUES(3, 4, 40); + INSERT INTO edge VALUES(2, 5, 50); + INSERT INTO edge VALUES(3, 6, 60); + INSERT INTO edge VALUES(5, 7, 70); + INSERT INTO edge VALUES(3, 7, 70); + INSERT INTO edge VALUES(4, 8, 80); + INSERT INTO edge VALUES(7, 8, 80); + INSERT INTO edge VALUES(8, 9, 90); + + WITH RECURSIVE + ancest(id, mtime) AS + (VALUES(0, 0) + UNION + SELECT edge.xto, edge.seq FROM edge, ancest + WHERE edge.xfrom=ancest.id + ORDER BY 2 + ) + SELECT * FROM ancest; +} {0 0 1 10 2 20 3 30 4 40 5 50 6 60 7 70 8 80 9 90} +do_execsql_test 5.2.2 { + WITH RECURSIVE + ancest(id, mtime) AS + (VALUES(0, 0) + UNION ALL + SELECT edge.xto, edge.seq FROM edge, ancest + WHERE edge.xfrom=ancest.id + ORDER BY 2 + ) + SELECT * FROM ancest; +} {0 0 1 10 2 20 3 30 4 40 4 40 5 50 6 60 7 70 7 70 8 80 8 80 8 80 8 80 9 90 9 90 9 90 9 90} +do_execsql_test 5.2.3 { + WITH RECURSIVE + ancest(id, mtime) AS + (VALUES(0, 0) + UNION ALL + SELECT edge.xto, edge.seq FROM edge, ancest + WHERE edge.xfrom=ancest.id + ORDER BY 2 LIMIT 4 OFFSET 2 + ) + SELECT * FROM ancest; +} {2 20 3 30 4 40 4 40} do_catchsql_test 5.3 { - WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i LIMIT 10 ) - SELECT x FROM i LIMIT 10; -} {1 {LIMIT in a recursive query}} + WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i LIMIT 5) + SELECT x FROM i; +} {0 {1 2 3 4 5}} do_execsql_test 5.4 { WITH i(x) AS ( VALUES(1) UNION ALL SELECT (x+1)%10 FROM i)