From c6aff30ca470f67c72c3a133ad3616989657fcec Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 1 Sep 2011 15:32:47 +0000 Subject: [PATCH 01/10] Experimental code-generator changes to utilize new opcodes for sorting. FossilOrigin-Name: bab2e560f6cb989c83a96aad60f666960ede7abe --- manifest | 23 +++++++++++++---------- manifest.uuid | 2 +- src/select.c | 35 ++++++++++++++++++++++++++++++----- src/sqliteInt.h | 1 + src/vdbe.c | 12 +++++++++++- src/vdbeInt.h | 1 + src/vdbeaux.c | 2 +- 7 files changed, 58 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index cf95f75fde..b233b88396 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\susing\suninitialized\svariables\safter\sfailures\sin\sthe\smerge\ssort\scode. -D 2011-08-31T23:57:22.695 +C Experimental\scode-generator\schanges\sto\sutilize\snew\sopcodes\sfor\ssorting. +D 2011-09-01T15:32:47.873 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,11 +179,11 @@ F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c 14552e9ff4b27ec027a43fafb62ea5d049cd2809 +F src/select.c 20bef6860c5f69c6a90666b4968d4c5cb88056c0 F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd F src/sqlite.h.in 0a6c9c23337fd1352c5c75a613ff9533aa7d91cb F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 -F src/sqliteInt.h 86a4fdb3ba9ab31d98b266797606f30fefe5b8a9 +F src/sqliteInt.h f6debf9a9eb8463ab2ef8be4b2b740ea9af5afba F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -238,11 +238,11 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c 9165b35da939f4a66c7e68b0c6d3f017ca982cb1 +F src/vdbe.c 9260e5138855399bea2611a29da336688bfa1b79 F src/vdbe.h c1eeedacab6bcf1e7c2cf8203ba9763a616f9a86 -F src/vdbeInt.h 70767f6504aac4f0057ec2a55738470a890789ac +F src/vdbeInt.h 51a902e12c7d571e3b516e5407e30f996494aafe F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 -F src/vdbeaux.c de1e4cab060a45df9ebee68dd63543d14559f0e7 +F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 F src/vdbesort.c f3d043a1bab7409d4a23cd7a35287c3ac440a167 @@ -961,7 +961,10 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 70b5b309568ac55565558d5456aca1e431cfd26b -R 09e04eb9163c46e529a61f2438cff2a1 +P 2869ed28299b1c9f355ecc24635830f7f1249126 +R e08eee2093eee8a9fb40d6646a4c9c76 +T *branch * merge-sort +T *sym-merge-sort * +T -sym-trunk * U drh -Z bdb2832352809bf647849d42b7aa6060 +Z c5e8ef1774d371c2becb6d0057b76727 diff --git a/manifest.uuid b/manifest.uuid index 67e9d42efe..c35621c659 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2869ed28299b1c9f355ecc24635830f7f1249126 \ No newline at end of file +bab2e560f6cb989c83a96aad60f666960ede7abe \ No newline at end of file diff --git a/src/select.c b/src/select.c index 2e44522c6d..8825326594 100644 --- a/src/select.c +++ b/src/select.c @@ -419,12 +419,18 @@ static void pushOntoSorter( int nExpr = pOrderBy->nExpr; int regBase = sqlite3GetTempRange(pParse, nExpr+2); int regRecord = sqlite3GetTempReg(pParse); + int op; sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pOrderBy, regBase, 0); sqlite3VdbeAddOp2(v, OP_Sequence, pOrderBy->iECursor, regBase+nExpr); sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nExpr + 2, regRecord); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pOrderBy->iECursor, regRecord); + if( pSelect->selFlags & SF_UseSorter ){ + op = OP_SorterInsert; + }else{ + op = OP_IdxInsert; + } + sqlite3VdbeAddOp2(v, op, pOrderBy->iECursor, regRecord); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); if( pSelect->iLimit ){ @@ -893,9 +899,20 @@ static void generateSortTail( }else{ regRowid = sqlite3GetTempReg(pParse); } - addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); - codeOffset(v, p, addrContinue); - sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr + 1, regRow); + if( p->selFlags & SF_UseSorter ){ + int regSortOut = sqlite3GetTempReg(pParse); + int ptab2 = pParse->nTab++; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, pOrderBy->nExpr+2); + addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); + codeOffset(v, p, addrContinue); + sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); + sqlite3VdbeAddOp3(v, OP_Column, ptab2, pOrderBy->nExpr+1, regRow); + sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); + }else{ + addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); + codeOffset(v, p, addrContinue); + sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr+1, regRow); + } switch( eDest ){ case SRT_Table: case SRT_EphemTab: { @@ -948,7 +965,11 @@ static void generateSortTail( /* The bottom of the loop */ sqlite3VdbeResolveLabel(v, addrContinue); - sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); + if( p->selFlags & SF_UseSorter ){ + sqlite3VdbeAddOp2(v, OP_SorterNext, iTab, addr); + }else{ + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); + } sqlite3VdbeResolveLabel(v, addrBreak); if( eDest==SRT_Output || eDest==SRT_Coroutine ){ sqlite3VdbeAddOp2(v, OP_Close, pseudoTab, 0); @@ -3914,6 +3935,10 @@ int sqlite3Select( iEnd = sqlite3VdbeMakeLabel(v); p->nSelectRow = (double)LARGEST_INT64; computeLimitRegisters(pParse, p, iEnd); + if( p->iLimit==0 && addrSortIndex>=0 ){ + sqlite3VdbeGetOp(v, addrSortIndex)->opcode = OP_SorterOpen; + p->selFlags |= SF_UseSorter; + } /* Open a virtual index to use for the distinct set. */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5934c74316..18beb32e0f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2082,6 +2082,7 @@ struct Select { #define SF_UsesEphemeral 0x0008 /* Uses the OpenEphemeral opcode */ #define SF_Expanded 0x0010 /* sqlite3SelectExpand() called on this */ #define SF_HasTypeInfo 0x0020 /* FROM subqueries have Table metadata */ +#define SF_UseSorter 0x0040 /* Sort using a sorter */ /* diff --git a/src/vdbe.c b/src/vdbe.c index e7c7ed7fe2..f4ae4c44a6 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3170,6 +3170,7 @@ case OP_OpenWrite: { */ case OP_OpenSorter: case OP_OpenAutoindex: +case OP_SorterOpen: case OP_OpenEphemeral: { VdbeCursor *pCx; static const int vfsFlags = @@ -3214,6 +3215,7 @@ case OP_OpenEphemeral: { } pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); pCx->isIndex = !pCx->isTable; + pCx->isSorter = pOp->opcode==OP_SorterOpen; #ifndef SQLITE_OMIT_MERGE_SORT if( rc==SQLITE_OK && pOp->opcode==OP_OpenSorter ){ rc = sqlite3VdbeSorterInit(db, pCx); @@ -4090,6 +4092,7 @@ case OP_ResetCount: { ** If the P1 cursor must be pointing to a valid row (not a NULL row) ** of a real table, not a pseudo-table. */ +case OP_SorterData: case OP_RowKey: case OP_RowData: { VdbeCursor *pC; @@ -4103,11 +4106,12 @@ case OP_RowData: { /* Note that RowKey and RowData are really exactly the same instruction */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; - assert( pC->isTable || pOp->opcode==OP_RowKey ); + assert( pC->isTable || pOp->opcode!=OP_RowData ); assert( pC->isIndex || pOp->opcode==OP_RowData ); assert( pC!=0 ); assert( pC->nullRow==0 ); assert( pC->pseudoTableReg==0 ); + assert( pC->isSorter==(pOp->opcode==OP_SorterData) ); if( isSorter(pC) ){ assert( pOp->opcode==OP_RowKey ); @@ -4271,6 +4275,7 @@ case OP_Last: { /* jump */ ** regression tests can determine whether or not the optimizer is ** correctly optimizing out sorts. */ +case OP_SorterSort: /* jump */ case OP_Sort: { /* jump */ #ifdef SQLITE_TEST sqlite3_sort_count++; @@ -4295,6 +4300,7 @@ case OP_Rewind: { /* jump */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); + assert( pC->isSorter==(pOp->opcode==OP_SorterSort) ); res = 1; if( isSorter(pC) ){ rc = sqlite3VdbeSorterRewind(db, pC, &res); @@ -4347,6 +4353,7 @@ case OP_Rewind: { /* jump */ ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. */ +case OP_SorterNext: /* jump */ case OP_Prev: /* jump */ case OP_Next: { /* jump */ VdbeCursor *pC; @@ -4359,6 +4366,7 @@ case OP_Next: { /* jump */ if( pC==0 ){ break; /* See ticket #2273 */ } + assert( pC->isSorter==(pOp->opcode==OP_SorterNext) ); if( isSorter(pC) ){ assert( pOp->opcode==OP_Next ); rc = sqlite3VdbeSorterNext(db, pC, &res); @@ -4395,6 +4403,7 @@ case OP_Next: { /* jump */ ** This instruction only works for indices. The equivalent instruction ** for tables is OP_Insert. */ +case OP_SorterInsert: case OP_IdxInsert: { /* in2 */ VdbeCursor *pC; BtCursor *pCrsr; @@ -4404,6 +4413,7 @@ case OP_IdxInsert: { /* in2 */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); + assert( pC->isSorter==(pOp->opcode==OP_SorterInsert) ); pIn2 = &aMem[pOp->p2]; assert( pIn2->flags & MEM_Blob ); pCrsr = pC->pCursor; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 846d807075..70d80c9d38 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -59,6 +59,7 @@ struct VdbeCursor { Bool isTable; /* True if a table requiring integer keys */ Bool isIndex; /* True if an index containing keys only - no data */ Bool isOrdered; /* True if the underlying table is BTREE_UNORDERED */ + Bool isSorter; /* True if a new-style sorter */ sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ const sqlite3_module *pModule; /* Module for cursor pVtabCursor */ i64 seqCount; /* Sequence counter */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 053d89f3b5..f622e8d09a 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -433,7 +433,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ n = pOp[-1].p1; if( n>nMaxArgs ) nMaxArgs = n; #endif - }else if( opcode==OP_Next ){ + }else if( opcode==OP_Next || opcode==OP_SorterNext ){ pOp->p4.xAdvance = sqlite3BtreeNext; pOp->p4type = P4_ADVANCE; }else if( opcode==OP_Prev ){ From 1c9d835d495f2c23642dce5956ffb3dfcaa80d22 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 1 Sep 2011 16:01:27 +0000 Subject: [PATCH 02/10] Use OP_SorterOpen instead of OP_OpenEphemeral to implement GROUP BY. FossilOrigin-Name: ebf819aaa555bd79fddfc0a6f9827a2539095d6c --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/select.c | 21 +++++++++++++++------ test/distinct.test | 2 +- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index b233b88396..4ed8e1a4b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\scode-generator\schanges\sto\sutilize\snew\sopcodes\sfor\ssorting. -D 2011-09-01T15:32:47.873 +C Use\sOP_SorterOpen\sinstead\sof\sOP_OpenEphemeral\sto\simplement\sGROUP\sBY. +D 2011-09-01T16:01:27.777 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,7 +179,7 @@ F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c 20bef6860c5f69c6a90666b4968d4c5cb88056c0 +F src/select.c 037ee5501fe0e743fa98936902c200ed9ed69156 F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd F src/sqlite.h.in 0a6c9c23337fd1352c5c75a613ff9533aa7d91cb F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 @@ -369,7 +369,7 @@ F test/descidx1.test 533dcbda614b0463b0ea029527fd27e5a9ab2d66 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test fe720e8b37d59f4cef808b0bf4e1b391c2e56b6f F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e -F test/distinct.test 8c4d951fc40aba84421060e07b16099d2f4c2fdf +F test/distinct.test df5b11ad606439129c88720a86787bc9ca181f31 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_createtable.test 4771686a586b6ae414f927c389b2c101cc05c028 F test/e_delete.test e2ae0d3fce5efd70fef99025e932afffc5616fab @@ -961,10 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 2869ed28299b1c9f355ecc24635830f7f1249126 -R e08eee2093eee8a9fb40d6646a4c9c76 -T *branch * merge-sort -T *sym-merge-sort * -T -sym-trunk * +P bab2e560f6cb989c83a96aad60f666960ede7abe +R 47ae22012e17b717f429ee6a9d305e96 U drh -Z c5e8ef1774d371c2becb6d0057b76727 +Z 4edda214a8d1e652ef01c1f98ad8fd39 diff --git a/manifest.uuid b/manifest.uuid index c35621c659..e5664d914b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bab2e560f6cb989c83a96aad60f666960ede7abe \ No newline at end of file +ebf819aaa555bd79fddfc0a6f9827a2539095d6c \ No newline at end of file diff --git a/src/select.c b/src/select.c index 8825326594..d8332eaaea 100644 --- a/src/select.c +++ b/src/select.c @@ -4033,6 +4033,8 @@ int sqlite3Select( int iAbortFlag; /* Mem address which causes query abort if positive */ int groupBySort; /* Rows come from source in GROUP BY order */ int addrEnd; /* End of processing for this SELECT */ + int sortPTab = 0; /* Pseudotable used to decode sorting results */ + int sortOut = 0; /* Output register from the sorter */ /* Remove any and all aliases between the result set and the ** GROUP BY clause. @@ -4094,12 +4096,12 @@ int sqlite3Select( /* If there is a GROUP BY clause we might need a sorting index to ** implement it. Allocate that sorting index now. If it turns out - ** that we do not need it after all, the OpenEphemeral instruction + ** that we do not need it after all, the OP_SorterOpen instruction ** will be converted into a Noop. */ sAggInfo.sortingIdx = pParse->nTab++; pKeyInfo = keyInfoFromExprList(pParse, pGroupBy); - addrSortingIdx = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, sAggInfo.sortingIdx, sAggInfo.nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO_HANDOFF); @@ -4180,11 +4182,14 @@ int sqlite3Select( } regRecord = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); - sqlite3VdbeAddOp2(v, OP_IdxInsert, sAggInfo.sortingIdx, regRecord); + sqlite3VdbeAddOp2(v, OP_SorterInsert, sAggInfo.sortingIdx, regRecord); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nCol); sqlite3WhereEnd(pWInfo); - sqlite3VdbeAddOp2(v, OP_Sort, sAggInfo.sortingIdx, addrEnd); + sortPTab = pParse->nTab++; + sortOut = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); + sqlite3VdbeAddOp2(v, OP_SorterSort, sAggInfo.sortingIdx, addrEnd); VdbeComment((v, "GROUP BY sort")); sAggInfo.useSortingIdx = 1; sqlite3ExprCacheClear(pParse); @@ -4197,9 +4202,13 @@ int sqlite3Select( */ addrTopOfLoop = sqlite3VdbeCurrentAddr(v); sqlite3ExprCacheClear(pParse); + if( groupBySort ){ + sqlite3VdbeAddOp2(v, OP_SorterData, sAggInfo.sortingIdx, sortOut); + } for(j=0; jnExpr; j++){ if( groupBySort ){ - sqlite3VdbeAddOp3(v, OP_Column, sAggInfo.sortingIdx, j, iBMem+j); + sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j); + if( j==0 ) sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); }else{ sAggInfo.directMode = 1; sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); @@ -4238,7 +4247,7 @@ int sqlite3Select( /* End of the loop */ if( groupBySort ){ - sqlite3VdbeAddOp2(v, OP_Next, sAggInfo.sortingIdx, addrTopOfLoop); + sqlite3VdbeAddOp2(v, OP_SorterNext, sAggInfo.sortingIdx, addrTopOfLoop); }else{ sqlite3WhereEnd(pWInfo); sqlite3VdbeChangeToNoop(v, addrSortingIdx, 1); diff --git a/test/distinct.test b/test/distinct.test index 87456de876..e0a913604d 100644 --- a/test/distinct.test +++ b/test/distinct.test @@ -45,7 +45,7 @@ proc do_temptables_test {tn sql temptables} { uplevel [list do_test $tn [subst -novar { set ret "" db eval "EXPLAIN [set sql]" { - if {$opcode == "OpenEphemeral"} { + if {$opcode == "OpenEphemeral" || $opcode == "SorterOpen"} { if {$p5 != "10" && $p5!="00"} { error "p5 = $p5" } if {$p5 == "10"} { lappend ret hash From 5134d1357e477c856109cecddb2e59be320eef0a Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Sep 2011 10:31:11 +0000 Subject: [PATCH 03/10] Instead of a temporary b-tree, use a linked-list and merge-sort to sort records in main memory in vdbesort.c. FossilOrigin-Name: 7769fb988d9be0f2d8129aaac19620ac88f9b4a6 --- manifest | 32 ++-- manifest.uuid | 2 +- src/btree.c | 11 -- src/btree.h | 1 - src/build.c | 26 ++- src/expr.c | 2 +- src/select.c | 2 +- src/sqliteInt.h | 1 + src/vdbe.c | 90 ++++++---- src/vdbeInt.h | 6 +- src/vdbesort.c | 438 +++++++++++++++++++++++++++++++++-------------- test/index4.test | 14 ++ 12 files changed, 423 insertions(+), 202 deletions(-) diff --git a/manifest b/manifest index 4ed8e1a4b9..9513f2ff2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sOP_SorterOpen\sinstead\sof\sOP_OpenEphemeral\sto\simplement\sGROUP\sBY. -D 2011-09-01T16:01:27.777 +C Instead\sof\sa\stemporary\sb-tree,\suse\sa\slinked-list\sand\smerge-sort\sto\ssort\srecords\sin\smain\smemory\sin\svdbesort.c. +D 2011-09-02T10:31:11.173 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,16 +124,16 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 28a4fe55327ff708bfaf9d4326d02686f7a553c3 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 4a2856b3bde9959986a7b9327841b3ff94023784 -F src/btree.h 9ddf04226eac592d4cc3709c5a8b33b2351ff5f7 +F src/btree.c bc2099e7d3c22c52b2c54349b9c07c04f2a810d0 +F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 -F src/build.c 2d5de52df616a3bf5a659cbca85211c46e2ba9bd +F src/build.c dc367138cb3625e6d42b389e05d7267aece5753c F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c e3132ec65240b2e2f3d50831021eac387f27584d F src/date.c a3c6842bad7ae632281811de112a8ba63ff08ab3 F src/delete.c ff68e5ef23aee08c0ff528f699a19397ed8bbed8 -F src/expr.c 4bbdfaf66bc614be9254ce0c26a17429067a3e07 +F src/expr.c cbcd8c2f1588a9862291a081699854c5e1cb28ab F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 9f00ea98f6b360d477b5a78b5b59a1fbde82431c F src/func.c 59bb046d7e3df1ab512ac339ccb0a6f996a17cb7 @@ -179,11 +179,11 @@ F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c 037ee5501fe0e743fa98936902c200ed9ed69156 +F src/select.c 32d0f4e5513362706b8973e7f1b87cd0885dfbf5 F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd F src/sqlite.h.in 0a6c9c23337fd1352c5c75a613ff9533aa7d91cb F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 -F src/sqliteInt.h f6debf9a9eb8463ab2ef8be4b2b740ea9af5afba +F src/sqliteInt.h 723cda73a33c91f5a0a145f4c0ced45d94921079 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -238,14 +238,14 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c 9260e5138855399bea2611a29da336688bfa1b79 +F src/vdbe.c da9c7efc48dc79d7785f3f17a1c3df514bf18489 F src/vdbe.h c1eeedacab6bcf1e7c2cf8203ba9763a616f9a86 -F src/vdbeInt.h 51a902e12c7d571e3b516e5407e30f996494aafe +F src/vdbeInt.h a255da14be8c2794ce38e0d2142877bb29df9105 F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c f3d043a1bab7409d4a23cd7a35287c3ac440a167 +F src/vdbesort.c 9c2e8ca23c2413a5162a4433cb72377588d896f8 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -511,7 +511,7 @@ F test/incrvacuum_ioerr.test 22f208d01c528403240e05beecc41dc98ed01637 F test/index.test b5429732b3b983fa810e3ac867d7ca85dae35097 F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test 423a25c789fc8cc51aaf2a4370bbdde2d9e9eed7 -F test/index4.test c82a59c9ae2ac01804bdb100162dca057318f40f +F test/index4.test 2983216eb8c86ee62d9ed7cb206b5cc3331c0026 F test/indexedby.test be501e381b82b2f8ab406309ba7aac46e221f4ad F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P bab2e560f6cb989c83a96aad60f666960ede7abe -R 47ae22012e17b717f429ee6a9d305e96 -U drh -Z 4edda214a8d1e652ef01c1f98ad8fd39 +P ebf819aaa555bd79fddfc0a6f9827a2539095d6c +R c9d892df81c49366e067933871c3b776 +U dan +Z b3581bd03fb5a181801f57c568eb418b diff --git a/manifest.uuid b/manifest.uuid index e5664d914b..f82002ddab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ebf819aaa555bd79fddfc0a6f9827a2539095d6c \ No newline at end of file +7769fb988d9be0f2d8129aaac19620ac88f9b4a6 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 7166b93b90..b335579bd0 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1734,22 +1734,11 @@ int sqlite3BtreeOpen( /* A BTREE_SINGLE database is always a temporary and/or ephemeral */ assert( (flags & BTREE_SINGLE)==0 || isTempDb ); - /* The BTREE_SORTER flag is only used if SQLITE_OMIT_MERGE_SORT is undef */ -#ifdef SQLITE_OMIT_MERGE_SORT - assert( (flags & BTREE_SORTER)==0 ); -#endif - - /* BTREE_SORTER is always on a BTREE_SINGLE, BTREE_OMIT_JOURNAL */ - assert( (flags & BTREE_SORTER)==0 || - (flags & (BTREE_SINGLE|BTREE_OMIT_JOURNAL)) - ==(BTREE_SINGLE|BTREE_OMIT_JOURNAL) ); - if( db->flags & SQLITE_NoReadlock ){ flags |= BTREE_NO_READLOCK; } if( isMemdb ){ flags |= BTREE_MEMORY; - flags &= ~BTREE_SORTER; } if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){ vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; diff --git a/src/btree.h b/src/btree.h index ce19826ad8..9e3a73b3b6 100644 --- a/src/btree.h +++ b/src/btree.h @@ -61,7 +61,6 @@ int sqlite3BtreeOpen( #define BTREE_MEMORY 4 /* This is an in-memory DB */ #define BTREE_SINGLE 8 /* The file contains at most 1 b-tree */ #define BTREE_UNORDERED 16 /* Use of a hash implementation is OK */ -#define BTREE_SORTER 32 /* Used as accumulator in external merge sort */ int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); diff --git a/src/build.c b/src/build.c index 29fbf92713..0de1020b9b 100644 --- a/src/build.c +++ b/src/build.c @@ -2326,6 +2326,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ int iIdx = pParse->nTab++; /* Btree cursor used for pIndex */ int iSorter = iTab; /* Cursor opened by OpenSorter (if in use) */ int addr1; /* Address of top of loop */ + int addr2; /* Address to jump to for next iteration */ int tnum; /* Root page of index */ Vdbe *v; /* Generate code into this virtual machine */ KeyInfo *pKey; /* KeyInfo for index */ @@ -2372,25 +2373,34 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ if( bUseSorter ){ iSorter = pParse->nTab++; sqlite3VdbeAddOp4(v, OP_OpenSorter, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); - sqlite3VdbeChangeP5(v, BTREE_SORTER); } /* Open the table. Loop through all rows of the table, inserting index ** records into the sorter. */ sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); + addr2 = addr1 + 1; regRecord = sqlite3GetTempReg(pParse); regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); if( bUseSorter ){ - sqlite3VdbeAddOp2(v, OP_IdxInsert, iSorter, regRecord); + sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); sqlite3VdbeJumpHere(v, addr1); - addr1 = sqlite3VdbeAddOp2(v, OP_Sort, iSorter, 0); - sqlite3VdbeAddOp2(v, OP_RowKey, iSorter, regRecord); - } - - if( pIndex->onError!=OE_None ){ + addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); + if( pIndex->onError!=OE_None ){ + int j2 = sqlite3VdbeCurrentAddr(v) + 3; + sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); + addr2 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_SorterCompare, iSorter, j2, regRecord); + sqlite3HaltConstraint( + pParse, OE_Abort, "indexed columns are not unique", P4_STATIC + ); + }else{ + addr2 = sqlite3VdbeCurrentAddr(v); + } + sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); + }else if( pIndex->onError!=OE_None ){ const int regRowid = regIdxKey + pIndex->nColumn; const int j2 = sqlite3VdbeCurrentAddr(v) + 2; void * const pRegKey = SQLITE_INT_TO_PTR(regIdxKey); @@ -2411,7 +2421,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, bUseSorter); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3VdbeAddOp2(v, OP_Next, iSorter, addr1+1); + sqlite3VdbeAddOp2(v, bUseSorter ? OP_SorterNext : OP_Next, iSorter, addr2); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp1(v, OP_Close, iTab); diff --git a/src/expr.c b/src/expr.c index ab4547db9c..ab218078db 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2287,7 +2287,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ inReg = pCol->iMem; break; }else if( pAggInfo->useSortingIdx ){ - sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdx, + sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pCol->iSorterColumn, target); break; } diff --git a/src/select.c b/src/select.c index d8332eaaea..7d9d26602e 100644 --- a/src/select.c +++ b/src/select.c @@ -4186,7 +4186,7 @@ int sqlite3Select( sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nCol); sqlite3WhereEnd(pWInfo); - sortPTab = pParse->nTab++; + sAggInfo.sortingIdxPTab = sortPTab = pParse->nTab++; sortOut = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); sqlite3VdbeAddOp2(v, OP_SorterSort, sAggInfo.sortingIdx, addrEnd); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 18beb32e0f..694f4694eb 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1550,6 +1550,7 @@ struct AggInfo { u8 useSortingIdx; /* In direct mode, reference the sorting index rather ** than the source table */ int sortingIdx; /* Cursor number of the sorting index */ + int sortingIdxPTab; /* Cursor number of pseudo-table */ ExprList *pGroupBy; /* The group by clause */ int nSortingColumn; /* Number of columns in the sorting index */ struct AggInfo_col { /* For each column used in source tables */ diff --git a/src/vdbe.c b/src/vdbe.c index f4ae4c44a6..d9310e0563 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3162,15 +3162,7 @@ case OP_OpenWrite: { ** by this opcode will be used for automatically created transient ** indices in joins. */ -/* Opcode: OpenSorter P1 P2 * P4 * -** -** This opcode works like OP_OpenEphemeral except that it opens -** a transient index that is specifically designed to sort large -** tables using an external merge-sort algorithm. -*/ -case OP_OpenSorter: case OP_OpenAutoindex: -case OP_SorterOpen: case OP_OpenEphemeral: { VdbeCursor *pCx; static const int vfsFlags = @@ -3181,7 +3173,6 @@ case OP_OpenEphemeral: { SQLITE_OPEN_TRANSIENT_DB; assert( pOp->p1>=0 ); - assert( (pOp->opcode==OP_OpenSorter)==((pOp->p5 & BTREE_SORTER)!=0) ); pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; @@ -3215,12 +3206,24 @@ case OP_OpenEphemeral: { } pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); pCx->isIndex = !pCx->isTable; - pCx->isSorter = pOp->opcode==OP_SorterOpen; -#ifndef SQLITE_OMIT_MERGE_SORT - if( rc==SQLITE_OK && pOp->opcode==OP_OpenSorter ){ - rc = sqlite3VdbeSorterInit(db, pCx); - } -#endif + break; +} + +/* Opcode: OpenSorter P1 P2 * P4 * +** +** This opcode works like OP_OpenEphemeral except that it opens +** a transient index that is specifically designed to sort large +** tables using an external merge-sort algorithm. +*/ +case OP_SorterOpen: +case OP_OpenSorter: { + VdbeCursor *pCx; + pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); + if( pCx==0 ) goto no_mem; + pCx->pKeyInfo = pOp->p4.pKeyInfo; + pCx->pKeyInfo->enc = ENC(p->db); + pCx->isSorter = 1; + rc = sqlite3VdbeSorterInit(db, pCx); break; } @@ -4072,6 +4075,40 @@ case OP_ResetCount: { break; } +/* Opcode: SorterCompare P1 P2 P3 +** +** P1 is a sorter cursor. This instruction compares the record blob in +** register P3 with the entry that the sorter cursor currently points to. +** If, excluding the rowid fields at the end, the two records are a match, +** fall through to the next instruction. Otherwise, jump to instruction P2. +*/ +case OP_SorterCompare: { + VdbeCursor *pC; + int res; + + pC = p->apCsr[pOp->p1]; + assert( isSorter(pC) ); + pIn3 = &aMem[pOp->p3]; + rc = sqlite3VdbeSorterCompare(pC, pIn3, &res); + if( res ){ + pc = pOp->p2-1; + } + break; +}; + +/* Opcode: SorterData P1 P2 * * * +** +** Write into register P2 the current sorter data for sorter cursor P1. +*/ +case OP_SorterData: { + VdbeCursor *pC; + pOut = &aMem[pOp->p2]; + pC = p->apCsr[pOp->p1]; + assert( pC->isSorter ); + rc = sqlite3VdbeSorterRowkey(pC, pOut); + break; +} + /* Opcode: RowData P1 P2 * * * ** ** Write into register P2 the complete row data for cursor P1. @@ -4092,7 +4129,6 @@ case OP_ResetCount: { ** If the P1 cursor must be pointing to a valid row (not a NULL row) ** of a real table, not a pseudo-table. */ -case OP_SorterData: case OP_RowKey: case OP_RowData: { VdbeCursor *pC; @@ -4106,6 +4142,7 @@ case OP_RowData: { /* Note that RowKey and RowData are really exactly the same instruction */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; + assert( pC->isSorter==0 ); assert( pC->isTable || pOp->opcode!=OP_RowData ); assert( pC->isIndex || pOp->opcode==OP_RowData ); assert( pC!=0 ); @@ -4113,12 +4150,6 @@ case OP_RowData: { assert( pC->pseudoTableReg==0 ); assert( pC->isSorter==(pOp->opcode==OP_SorterData) ); - if( isSorter(pC) ){ - assert( pOp->opcode==OP_RowKey ); - rc = sqlite3VdbeSorterRowkey(pC, pOut); - break; - } - assert( pC->pCursor!=0 ); pCrsr = pC->pCursor; assert( sqlite3BtreeCursorIsValid(pCrsr) ); @@ -4368,7 +4399,7 @@ case OP_Next: { /* jump */ } assert( pC->isSorter==(pOp->opcode==OP_SorterNext) ); if( isSorter(pC) ){ - assert( pOp->opcode==OP_Next ); + assert( pOp->opcode==OP_SorterNext ); rc = sqlite3VdbeSorterNext(db, pC, &res); }else{ res = 1; @@ -4421,16 +4452,17 @@ case OP_IdxInsert: { /* in2 */ assert( pC->isTable==0 ); rc = ExpandBlob(pIn2); if( rc==SQLITE_OK ){ - nKey = pIn2->n; - zKey = pIn2->z; - rc = sqlite3VdbeSorterWrite(db, pC, nKey); - if( rc==SQLITE_OK ){ + if( isSorter(pC) ){ + rc = sqlite3VdbeSorterWrite(db, pC, pIn2); + }else{ + nKey = pIn2->n; + zKey = pIn2->z; rc = sqlite3BtreeInsert(pCrsr, zKey, nKey, "", 0, 0, pOp->p3, ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) - ); + ); assert( pC->deferredMoveto==0 ); + pC->cacheStatus = CACHE_STALE; } - pC->cacheStatus = CACHE_STALE; } } break; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 70d80c9d38..68ad8dcbf7 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -403,13 +403,15 @@ void sqlite3VdbeMemStoreType(Mem *pMem); # define sqlite3VdbeSorterRowkey(Y,Z) SQLITE_OK # define sqlite3VdbeSorterRewind(X,Y,Z) SQLITE_OK # define sqlite3VdbeSorterNext(X,Y,Z) SQLITE_OK +# define sqlite3VdbeSorterCompare(X,Y,Z) SQLITE_OK #else int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); -int sqlite3VdbeSorterWrite(sqlite3 *, VdbeCursor *, int); void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); int sqlite3VdbeSorterRowkey(VdbeCursor *, Mem *); -int sqlite3VdbeSorterRewind(sqlite3 *, VdbeCursor *, int *); int sqlite3VdbeSorterNext(sqlite3 *, VdbeCursor *, int *); +int sqlite3VdbeSorterRewind(sqlite3 *, VdbeCursor *, int *); +int sqlite3VdbeSorterWrite(sqlite3 *, VdbeCursor *, Mem *); +int sqlite3VdbeSorterCompare(VdbeCursor *, Mem *, int *); #endif #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 diff --git a/src/vdbesort.c b/src/vdbesort.c index c3214b3afe..3ab5d7930a 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -21,6 +21,7 @@ #ifndef SQLITE_OMIT_MERGE_SORT typedef struct VdbeSorterIter VdbeSorterIter; +typedef struct SorterRecord SorterRecord; /* ** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES: @@ -92,8 +93,7 @@ typedef struct VdbeSorterIter VdbeSorterIter; ** being merged (rounded up to the next power of 2). */ struct VdbeSorter { - int nWorking; /* Start a new b-tree after this many pages */ - int nBtree; /* Current size of b-tree contents as PMA */ + int nInMemory; /* Current size of b-tree contents as PMA */ int nTree; /* Used size of aTree/aIter (power of 2) */ VdbeSorterIter *aIter; /* Array of iterators to merge */ int *aTree; /* Current state of incremental merge */ @@ -101,6 +101,9 @@ struct VdbeSorter { i64 iReadOff; /* Current read offset within file pTemp1 */ sqlite3_file *pTemp1; /* PMA file 1 */ int nPMA; /* Number of PMAs stored in pTemp1 */ + SorterRecord *pRecord; /* Head of in-memory record list */ + int nLimit1; /* Minimum PMA size, in bytes */ + int nLimit2; /* Maximum PMA size, in bytes */ }; /* @@ -117,6 +120,17 @@ struct VdbeSorterIter { u8 *aKey; /* Pointer to current key */ }; +/* +** A structure to store a single record. All in-memory records are connected +** together into a linked list headed at VdbeSorter.pRecord using the +** SorterRecord.pNext pointer. +*/ +struct SorterRecord { + void *pVal; + int nVal; + SorterRecord *pNext; +}; + /* Minimum allowable value for the VdbeSorter.nWorking variable */ #define SORTER_MIN_WORKING 10 @@ -275,6 +289,50 @@ static int vdbeSorterIterInit( return rc; } + +/* +** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, +** size nKey2 bytes). Argument pKeyInfo supplies the collation functions +** used by the comparison. If an error occurs, return an SQLite error code. +** Otherwise, return SQLITE_OK and set *pRes to a negative, zero or positive +** value, depending on whether key1 is smaller, equal to or larger than key2. +** +** If the bOmitRowid argument is non-zero, assume both keys end in a rowid +** field. For the purposes of the comparison, ignore it. Also, if bOmitRowid +** is true and key1 contains even a single NULL value, it is considered to +** be less than key2. Even if key2 also contains NULL values. +*/ +static int vdbeSorterCompare( + KeyInfo *pKeyInfo, /* Collation functions to use in comparison */ + int bOmitRowid, /* Ignore rowid field at end of keys */ + void *pKey1, int nKey1, /* Left side of comparison */ + void *pKey2, int nKey2, /* Right side of comparison */ + int *pRes /* OUT: Result of comparison */ +){ + char aSpace[150]; + UnpackedRecord *r2; + int i; + + r2 = sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, aSpace, sizeof(aSpace)); + if( r2==0 ) return SQLITE_NOMEM; + if( bOmitRowid ){ + for(i=0; inField-1; i++){ + if( r2->aMem[i].flags & MEM_Null ){ + *pRes = -1; + sqlite3VdbeDeleteUnpackedRecord(r2); + return SQLITE_OK; + } + } + r2->flags |= UNPACKED_PREFIX_MATCH; + r2->nField--; + assert( r2->nField>0 ); + } + + *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2); + sqlite3VdbeDeleteUnpackedRecord(r2); + return SQLITE_OK; +} + /* ** This function is called to compare two iterator keys when merging ** multiple b-tree segments. Parameter iOut is the index of the aTree[] @@ -306,20 +364,16 @@ static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){ }else if( p2->pFile==0 ){ iRes = i1; }else{ - char aSpace[150]; - UnpackedRecord *r1; - - r1 = sqlite3VdbeRecordUnpack( - pCsr->pKeyInfo, p1->nKey, p1->aKey, aSpace, sizeof(aSpace) + int res; + int rc = vdbeSorterCompare( + pCsr->pKeyInfo, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res ); - if( r1==0 ) return SQLITE_NOMEM; - - if( sqlite3VdbeRecordCompare(p2->nKey, p2->aKey, r1)>=0 ){ + if( rc!=SQLITE_OK ) return rc; + if( res<=0 ){ iRes = i1; }else{ iRes = i2; } - sqlite3VdbeDeleteUnpackedRecord(r1); } pSorter->aTree[iOut] = iRes; @@ -330,9 +384,32 @@ static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){ ** Initialize the temporary index cursor just opened as a sorter cursor. */ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ - assert( pCsr->pKeyInfo && pCsr->pBt ); + int pgsz; /* Page size of main database */ + + assert( pCsr->pKeyInfo && pCsr->pBt==0 ); pCsr->pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter)); - return (pCsr->pSorter ? SQLITE_OK : SQLITE_NOMEM); + if( pCsr->pSorter==0 ){ + return SQLITE_NOMEM; + } + + pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); + pCsr->pSorter->nLimit1 = 10 * pgsz; + pCsr->pSorter->nLimit2 = db->aDb[0].pSchema->cache_size * pgsz; + + return SQLITE_OK; +} + +/* +** Free the list of sorted records starting at pRecord. +*/ +static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ + SorterRecord *p; + SorterRecord *pNext; + for(p=pRecord; p; p=pNext){ + pNext = p->pNext; + sqlite3DbFree(db, p->pVal); + sqlite3DbFree(db, p); + } } /* @@ -351,6 +428,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ if( pSorter->pTemp1 ){ sqlite3OsCloseFree(pSorter->pTemp1); } + vdbeSorterRecordFree(db, pSorter->pRecord); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } @@ -370,10 +448,128 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ ); } +/* +** Attemp to merge the two sorted lists p1 and p2 into a single list. If no +** error occurs set *ppOut to the head of the new list and return SQLITE_OK. +*/ +static int vdbeSorterMerge( + sqlite3 *db, /* Database handle */ + KeyInfo *pKeyInfo, /* Collation functions to use in comparison */ + SorterRecord *p1, /* First list to merge */ + SorterRecord *p2, /* Second list to merge */ + SorterRecord **ppOut /* OUT: Head of merged list */ +){ + int rc = SQLITE_OK; + SorterRecord *pFinal = 0; + SorterRecord **pp = &pFinal; + + while( p1 || p2 ){ + if( p1==0 ){ + *pp = p2; + p2 = 0; + }else if( p2==0 ){ + *pp = p1; + p1 = 0; + }else{ + int res; + rc = vdbeSorterCompare( + pKeyInfo, 0, p1->pVal, p1->nVal, p2->pVal, p2->nVal, &res + ); + if( rc!=SQLITE_OK ){ + vdbeSorterRecordFree(db, p1); + vdbeSorterRecordFree(db, p2); + vdbeSorterRecordFree(db, pFinal); + pFinal = 0; + break; + } + if( res<=0 ){ + *pp = p1; + pp = &p1->pNext; + p1 = p1->pNext; + }else{ + *pp = p2; + pp = &p2->pNext; + p2 = p2->pNext; + } + *pp = 0; + } + } + + *ppOut = pFinal; + return rc; +} /* -** Write the current contents of the b-tree to a PMA. Return SQLITE_OK -** if successful, or an SQLite error code otherwise. +** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK +** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error +** occurs. +*/ +static int vdbeSorterSort(sqlite3 *db, VdbeCursor *pCsr){ + int rc = SQLITE_OK; + int i; + SorterRecord **aSlot; + SorterRecord *p; + VdbeSorter *pSorter = pCsr->pSorter; + KeyInfo *pKeyInfo = pCsr->pKeyInfo; + + aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); + if( !aSlot ){ + return SQLITE_NOMEM; + } + + p = pSorter->pRecord; + while( p ){ + SorterRecord *pNext = p->pNext; + p->pNext = 0; + for(i=0; rc==SQLITE_OK && aSlot[i]; i++){ + rc = vdbeSorterMerge(db, pKeyInfo, p, aSlot[i], &p); + aSlot[i] = 0; + } + if( rc!=SQLITE_OK ){ + vdbeSorterRecordFree(db, pNext); + break; + } + aSlot[i] = p; + p = pNext; + } + + p = 0; + for(i=0; i<64; i++){ + if( rc==SQLITE_OK ){ + rc = vdbeSorterMerge(db, pKeyInfo, p, aSlot[i], &p); + }else{ + vdbeSorterRecordFree(db, aSlot[i]); + } + } + pSorter->pRecord = p; + +#if 0 + { + SorterRecord *pTmp1 = 0; + SorterRecord *pTmp2; + for(pTmp2=pSorter->pRecord; pTmp2 && rc==SQLITE_OK; pTmp2=pTmp2->pNext){ + if( pTmp1 ){ + int res; + rc = vdbeSorterCompare(pKeyInfo, + 0, pTmp1->pVal, pTmp1->nVal, pTmp2->pVal, pTmp2->nVal, &res + ); + assert( rc!=SQLITE_OK || res<0 ); + } + pTmp1 = pTmp2; + } + } +#endif + + if( rc!=SQLITE_OK ){ + } + sqlite3_free(aSlot); + return rc; +} + + +/* +** Write the current contents of the in-memory linked-list to a PMA. Return +** SQLITE_OK if successful, or an SQLite error code otherwise. ** ** The format of a PMA is: ** @@ -384,19 +580,19 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ ** Each record consists of a varint followed by a blob of data (the ** key). The varint is the number of bytes in the blob of data. */ -static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){ +static int vdbeSorterListToPMA(sqlite3 *db, VdbeCursor *pCsr){ int rc = SQLITE_OK; /* Return code */ VdbeSorter *pSorter = pCsr->pSorter; - int res = 0; - /* sqlite3BtreeFirst() cannot fail because sorter btrees are always held - ** in memory and so an I/O error is not possible. */ - rc = sqlite3BtreeFirst(pCsr->pCursor, &res); - if( NEVER(rc!=SQLITE_OK) || res ) return rc; - assert( pSorter->nBtree>0 ); + if( pSorter->nInMemory==0 ){ + assert( pSorter->pRecord==0 ); + return rc; + } + + rc = vdbeSorterSort(db, pCsr); /* If the first temporary PMA file has not been opened, open it now. */ - if( pSorter->pTemp1==0 ){ + if( rc==SQLITE_OK && pSorter->pTemp1==0 ){ rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1); assert( rc!=SQLITE_OK || pSorter->pTemp1 ); assert( pSorter->iWriteOff==0 ); @@ -404,129 +600,89 @@ static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){ } if( rc==SQLITE_OK ){ - i64 iWriteOff = pSorter->iWriteOff; - void *aMalloc = 0; /* Array used to hold a single record */ - int nMalloc = 0; /* Allocated size of aMalloc[] in bytes */ + i64 iOff = pSorter->iWriteOff; + SorterRecord *p; + SorterRecord *pNext = 0; pSorter->nPMA++; - for( - rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nBtree, &iWriteOff); - rc==SQLITE_OK && res==0; - rc = sqlite3BtreeNext(pCsr->pCursor, &res) - ){ - i64 nKey; /* Size of this key in bytes */ + rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nInMemory, &iOff); + for(p=pSorter->pRecord; rc==SQLITE_OK && p; p=pNext){ + pNext = p->pNext; + rc = vdbeSorterWriteVarint(pSorter->pTemp1, p->nVal, &iOff); - /* Write the size of the record in bytes to the output file */ - (void)sqlite3BtreeKeySize(pCsr->pCursor, &nKey); - rc = vdbeSorterWriteVarint(pSorter->pTemp1, nKey, &iWriteOff); - - /* Make sure the aMalloc[] buffer is large enough for the record */ - if( rc==SQLITE_OK && nKey>nMalloc ){ - aMalloc = sqlite3DbReallocOrFree(db, aMalloc, nKey); - if( !aMalloc ){ - rc = SQLITE_NOMEM; - }else{ - nMalloc = nKey; - } - } - - /* Write the record itself to the output file */ if( rc==SQLITE_OK ){ - /* sqlite3BtreeKey() cannot fail because sorter btrees held in memory */ - rc = sqlite3BtreeKey(pCsr->pCursor, 0, nKey, aMalloc); - if( ALWAYS(rc==SQLITE_OK) ){ - rc = sqlite3OsWrite(pSorter->pTemp1, aMalloc, nKey, iWriteOff); - iWriteOff += nKey; - } + rc = sqlite3OsWrite(pSorter->pTemp1, p->pVal, p->nVal, iOff); + iOff += p->nVal; } - if( rc!=SQLITE_OK ) break; + sqlite3DbFree(db, p->pVal); + sqlite3DbFree(db, p); } /* This assert verifies that unless an error has occurred, the size of ** the PMA on disk is the same as the expected size stored in - ** pSorter->nBtree. */ - assert( rc!=SQLITE_OK || pSorter->nBtree==( - iWriteOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nBtree) + ** pSorter->nInMemory. */ + assert( rc!=SQLITE_OK || pSorter->nInMemory==( + iOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nInMemory) )); - pSorter->iWriteOff = iWriteOff; - sqlite3DbFree(db, aMalloc); + pSorter->iWriteOff = iOff; + pSorter->pRecord = p; } - pSorter->nBtree = 0; return rc; } /* -** This function is called on a sorter cursor by the VDBE before each row -** is inserted into VdbeCursor.pCsr. Argument nKey is the size of the key, in -** bytes, about to be inserted. -** -** If it is determined that the temporary b-tree accessed via VdbeCursor.pCsr -** is large enough, its contents are written to a sorted PMA on disk and the -** tree emptied. This prevents the b-tree (which must be small enough to -** fit entirely in the cache in order to support efficient inserts) from -** growing too large. -** -** An SQLite error code is returned if an error occurs. Otherwise, SQLITE_OK. +** Add a record to the sorter. */ -int sqlite3VdbeSorterWrite(sqlite3 *db, VdbeCursor *pCsr, int nKey){ - int rc = SQLITE_OK; /* Return code */ +int sqlite3VdbeSorterWrite( + sqlite3 *db, /* Database handle */ + VdbeCursor *pCsr, /* Sorter cursor */ + Mem *pVal /* Memory cell containing record */ +){ VdbeSorter *pSorter = pCsr->pSorter; - if( pSorter ){ - Pager *pPager = sqlite3BtreePager(pCsr->pBt); - int nPage; /* Current size of temporary file in pages */ + int rc; + SorterRecord *pNew; - /* Sorters never spill to disk */ - assert( sqlite3PagerFile(pPager)->pMethods==0 ); + assert( pSorter ); + pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; - /* Determine how many pages the temporary b-tree has grown to */ - sqlite3PagerPagecount(pPager, &nPage); - - /* If pSorter->nWorking is still zero, but the temporary file has been - ** created in the file-system, then the most recent insert into the - ** current b-tree segment probably caused the cache to overflow (it is - ** also possible that sqlite3_release_memory() was called). So set the - ** size of the working set to a little less than the current size of the - ** file in pages. */ - if( pSorter->nWorking==0 && sqlite3PagerUnderStress(pPager) ){ - pSorter->nWorking = nPage-5; - if( pSorter->nWorkingnWorking = SORTER_MIN_WORKING; - } + pNew = (SorterRecord *)sqlite3DbMallocZero(db, sizeof(SorterRecord)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3VdbeMemMakeWriteable(pVal); + if( rc==SQLITE_OK ){ + pNew->pVal = pVal->z; + pNew->nVal = pVal->n; + pVal->zMalloc = 0; + sqlite3VdbeMemSetNull(pVal); + pNew->pNext = pSorter->pRecord; + pSorter->pRecord = pNew; + }else{ + sqlite3DbFree(db, pNew); + rc = SQLITE_NOMEM; } - - /* If the number of pages used by the current b-tree segment is greater - ** than the size of the working set (VdbeSorter.nWorking), start a new - ** segment b-tree. */ - if( pSorter->nWorking && nPage>=pSorter->nWorking ){ - BtCursor *p = pCsr->pCursor;/* Cursor structure to close and reopen */ - int iRoot; /* Root page of new tree */ - - /* Copy the current contents of the b-tree into a PMA in sorted order. - ** Close the currently open b-tree cursor. */ - rc = vdbeSorterBtreeToPMA(db, pCsr); - sqlite3BtreeCloseCursor(p); - - if( rc==SQLITE_OK ){ - rc = sqlite3BtreeDropTable(pCsr->pBt, 2, 0); -#ifdef SQLITE_DEBUG - sqlite3PagerPagecount(pPager, &nPage); - assert( rc!=SQLITE_OK || nPage==1 ); -#endif - } - if( rc==SQLITE_OK ){ - rc = sqlite3BtreeCreateTable(pCsr->pBt, &iRoot, BTREE_BLOBKEY); - } - if( rc==SQLITE_OK ){ - assert( iRoot==2 ); - rc = sqlite3BtreeCursor(pCsr->pBt, iRoot, 1, pCsr->pKeyInfo, p); - } - } - - pSorter->nBtree += sqlite3VarintLen(nKey) + nKey; } + + /* See if the contents of the sorter should now be written out. They + ** are written out when either of the following are true: + ** + ** * The total memory allocated for the in-memory list is greater + ** than (page-size * cache-size), or + ** + ** * The total memory allocated for the in-memory list is greater + ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. + */ + if( rc==SQLITE_OK && ( + (pSorter->nInMemory>pSorter->nLimit2) + || (pSorter->nInMemory>pSorter->nLimit1 && sqlite3HeapNearlyFull()) + )){ + rc = vdbeSorterListToPMA(db, pCsr); + pSorter->nInMemory = 0; + } + return rc; } @@ -577,8 +733,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ assert( pSorter ); /* Write the current b-tree to a PMA. Close the b-tree cursor. */ - rc = vdbeSorterBtreeToPMA(db, pCsr); - sqlite3BtreeCloseCursor(pCsr->pCursor); + rc = vdbeSorterListToPMA(db, pCsr); if( rc!=SQLITE_OK ) return rc; if( pSorter->nPMA==0 ){ *pbEof = 1; @@ -692,14 +847,7 @@ int sqlite3VdbeSorterRowkey(VdbeCursor *pCsr, Mem *pOut){ VdbeSorterIter *pIter; pIter = &pSorter->aIter[ pSorter->aTree[1] ]; - - /* Coverage testing note: As things are currently, this call will always - ** succeed. This is because the memory cell passed by the VDBE layer - ** happens to be the same one as was used to assemble the keys before they - ** were passed to the sorter - meaning it is always large enough for the - ** largest key. But this could change very easily, so we leave the call - ** to sqlite3VdbeMemGrow() in. */ - if( NEVER(sqlite3VdbeMemGrow(pOut, pIter->nKey, 0)) ){ + if( sqlite3VdbeMemGrow(pOut, pIter->nKey, 0) ){ return SQLITE_NOMEM; } pOut->n = pIter->nKey; @@ -709,4 +857,30 @@ int sqlite3VdbeSorterRowkey(VdbeCursor *pCsr, Mem *pOut){ return SQLITE_OK; } +/* +** Compare the key in memory cell pVal with the key that the sorter cursor +** passed as the first argument currently points to. For the purposes of +** the comparison, ignore the rowid field at the end of each record. +** +** If an error occurs, return an SQLite error code (i.e. SQLITE_NOMEM). +** Otherwise, set *pRes to a negative, zero or positive value if the +** key in pVal is smaller than, equal to or larger than the current sorter +** key. +*/ +int sqlite3VdbeSorterCompare( + VdbeCursor *pCsr, /* Sorter cursor */ + Mem *pVal, /* Value to compare to current sorter key */ + int *pRes /* OUT: Result of comparison */ +){ + int rc; + VdbeSorter *pSorter = pCsr->pSorter; + VdbeSorterIter *pIter; + pIter = &pSorter->aIter[ pSorter->aTree[1] ]; + rc = vdbeSorterCompare(pCsr->pKeyInfo, 1, + pVal->z, pVal->n, pIter->aKey, pIter->nKey, pRes + ); + assert( rc!=SQLITE_OK || *pRes<=0 ); + return rc; +} + #endif /* #ifndef SQLITE_OMIT_MERGE_SORT */ diff --git a/test/index4.test b/test/index4.test index 6400e34e4e..018ed744a1 100644 --- a/test/index4.test +++ b/test/index4.test @@ -108,5 +108,19 @@ do_execsql_test 1.8 { PRAGMA integrity_check } {ok} +do_execsql_test 2.1 { + BEGIN; + CREATE TABLE t2(x); + INSERT INTO t2 VALUES(14); + INSERT INTO t2 VALUES(35); + INSERT INTO t2 VALUES(15); + INSERT INTO t2 VALUES(35); + INSERT INTO t2 VALUES(16); + COMMIT; +} +do_catchsql_test 2.2 { + CREATE UNIQUE INDEX i3 ON t2(x); +} {1 {indexed columns are not unique}} + finish_test From 9fed558df9dd95fe9442ba40927b8a86b12f998e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Sep 2011 11:45:31 +0000 Subject: [PATCH 04/10] If all data being sorted fits in memory, avoid writing any data out to temporary files in vdbesort.c. FossilOrigin-Name: 71075673c625f243969c3f34c73f28f378924007 --- manifest | 12 ++++---- manifest.uuid | 2 +- src/vdbesort.c | 77 ++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 63 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index 9513f2ff2e..7f37801bfe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Instead\sof\sa\stemporary\sb-tree,\suse\sa\slinked-list\sand\smerge-sort\sto\ssort\srecords\sin\smain\smemory\sin\svdbesort.c. -D 2011-09-02T10:31:11.173 +C If\sall\sdata\sbeing\ssorted\sfits\sin\smemory,\savoid\swriting\sany\sdata\sout\sto\stemporary\sfiles\sin\svdbesort.c. +D 2011-09-02T11:45:31.600 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -245,7 +245,7 @@ F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c 9c2e8ca23c2413a5162a4433cb72377588d896f8 +F src/vdbesort.c 1e9d4437c7520e09b18f37d2c520bfa01e638655 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P ebf819aaa555bd79fddfc0a6f9827a2539095d6c -R c9d892df81c49366e067933871c3b776 +P 7769fb988d9be0f2d8129aaac19620ac88f9b4a6 +R 023e08b29250dcfb353aa799966d9eeb U dan -Z b3581bd03fb5a181801f57c568eb418b +Z 052620c5c09aadfa96f446950f8f8dff diff --git a/manifest.uuid b/manifest.uuid index f82002ddab..6963963883 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7769fb988d9be0f2d8129aaac19620ac88f9b4a6 \ No newline at end of file +71075673c625f243969c3f34c73f28f378924007 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 3ab5d7930a..83c1c5d132 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -732,13 +732,18 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ assert( pSorter ); + /* If no data has been written to disk, then do not do so now. Instead, + ** sort the VdbeSorter.pRecord list. The vdbe layer will read data directly + ** from the in-memory list. */ + if( pSorter->nPMA==0 ){ + *pbEof = !pSorter->pRecord; + assert( pSorter->aTree==0 ); + return vdbeSorterSort(db, pCsr); + } + /* Write the current b-tree to a PMA. Close the b-tree cursor. */ rc = vdbeSorterListToPMA(db, pCsr); if( rc!=SQLITE_OK ) return rc; - if( pSorter->nPMA==0 ){ - *pbEof = 1; - return SQLITE_OK; - } /* Allocate space for aIter[] and aTree[]. */ nIter = pSorter->nPMA; @@ -826,33 +831,64 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ */ int sqlite3VdbeSorterNext(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; - int iPrev = pSorter->aTree[1]; /* Index of iterator to advance */ - int i; /* Index of aTree[] to recalculate */ int rc; /* Return code */ - rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]); - for(i=(pSorter->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ - rc = vdbeSorterDoCompare(pCsr, i); - } + if( pSorter->aTree ){ + int iPrev = pSorter->aTree[1];/* Index of iterator to advance */ + int i; /* Index of aTree[] to recalculate */ - *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]); + for(i=(pSorter->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ + rc = vdbeSorterDoCompare(pCsr, i); + } + + *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + }else{ + SorterRecord *pFree = pSorter->pRecord; + pSorter->pRecord = pFree->pNext; + pFree->pNext = 0; + vdbeSorterRecordFree(db, pFree); + *pbEof = !pSorter->pRecord; + rc = SQLITE_OK; + } return rc; } +/* +** Return a pointer to a buffer owned by the sorter that contains the +** current key. +*/ +static void *vdbeSorterRowkey( + VdbeSorter *pSorter, /* Sorter object */ + int *pnKey /* OUT: Size of current key in bytes */ +){ + void *pKey; + if( pSorter->aTree ){ + VdbeSorterIter *pIter; + pIter = &pSorter->aIter[ pSorter->aTree[1] ]; + *pnKey = pIter->nKey; + pKey = pIter->aKey; + }else{ + *pnKey = pSorter->pRecord->nVal; + pKey = pSorter->pRecord->pVal; + } + return pKey; +} + /* ** Copy the current sorter key into the memory cell pOut. */ int sqlite3VdbeSorterRowkey(VdbeCursor *pCsr, Mem *pOut){ VdbeSorter *pSorter = pCsr->pSorter; - VdbeSorterIter *pIter; + void *pKey; int nKey; /* Sorter key to copy into pOut */ - pIter = &pSorter->aIter[ pSorter->aTree[1] ]; - if( sqlite3VdbeMemGrow(pOut, pIter->nKey, 0) ){ + pKey = vdbeSorterRowkey(pSorter, &nKey); + if( sqlite3VdbeMemGrow(pOut, nKey, 0) ){ return SQLITE_NOMEM; } - pOut->n = pIter->nKey; + pOut->n = nKey; MemSetTypeFlag(pOut, MEM_Blob); - memcpy(pOut->z, pIter->aKey, pIter->nKey); + memcpy(pOut->z, pKey, nKey); return SQLITE_OK; } @@ -874,11 +910,10 @@ int sqlite3VdbeSorterCompare( ){ int rc; VdbeSorter *pSorter = pCsr->pSorter; - VdbeSorterIter *pIter; - pIter = &pSorter->aIter[ pSorter->aTree[1] ]; - rc = vdbeSorterCompare(pCsr->pKeyInfo, 1, - pVal->z, pVal->n, pIter->aKey, pIter->nKey, pRes - ); + void *pKey; int nKey; /* Sorter key to compare pVal with */ + + pKey = vdbeSorterRowkey(pSorter, &nKey); + rc = vdbeSorterCompare(pCsr->pKeyInfo, 1, pVal->z, pVal->n, pKey, nKey, pRes); assert( rc!=SQLITE_OK || *pRes<=0 ); return rc; } From 7f72114547ec18b5458e332faca908f81916f27c Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Sep 2011 15:41:33 +0000 Subject: [PATCH 05/10] Reduce the number of malloc() calls made when creating an index on more than 2 columns. FossilOrigin-Name: 065b0c9858da0ebb41722f3c56bdaf62f28b2f2c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 41 ++++++++++++++++++++++++++++------------- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 7f37801bfe..3c9140975f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sall\sdata\sbeing\ssorted\sfits\sin\smemory,\savoid\swriting\sany\sdata\sout\sto\stemporary\sfiles\sin\svdbesort.c. -D 2011-09-02T11:45:31.600 +C Reduce\sthe\snumber\sof\smalloc()\scalls\smade\swhen\screating\san\sindex\son\smore\sthan\s2\scolumns. +D 2011-09-02T15:41:33.781 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -245,7 +245,7 @@ F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c 1e9d4437c7520e09b18f37d2c520bfa01e638655 +F src/vdbesort.c a49d44498cc19af9bdbdc8e9fa882dd26cc17066 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 7769fb988d9be0f2d8129aaac19620ac88f9b4a6 -R 023e08b29250dcfb353aa799966d9eeb +P 71075673c625f243969c3f34c73f28f378924007 +R 16433222f8a97f40a1aa6bc60c6c3db0 U dan -Z 052620c5c09aadfa96f446950f8f8dff +Z 8c008bdf93f87673bd214d386e7f6cf5 diff --git a/manifest.uuid b/manifest.uuid index 6963963883..c870d49dd3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -71075673c625f243969c3f34c73f28f378924007 \ No newline at end of file +065b0c9858da0ebb41722f3c56bdaf62f28b2f2c \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 83c1c5d132..63c6f543b8 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -104,6 +104,8 @@ struct VdbeSorter { SorterRecord *pRecord; /* Head of in-memory record list */ int nLimit1; /* Minimum PMA size, in bytes */ int nLimit2; /* Maximum PMA size, in bytes */ + char *aSpace; /* Space for UnpackRecord() */ + int nSpace; /* Size of aSpace in bytes */ }; /* @@ -303,18 +305,31 @@ static int vdbeSorterIterInit( ** be less than key2. Even if key2 also contains NULL values. */ static int vdbeSorterCompare( - KeyInfo *pKeyInfo, /* Collation functions to use in comparison */ + VdbeCursor *pCsr, /* Cursor object (for pKeyInfo) */ int bOmitRowid, /* Ignore rowid field at end of keys */ void *pKey1, int nKey1, /* Left side of comparison */ void *pKey2, int nKey2, /* Right side of comparison */ int *pRes /* OUT: Result of comparison */ ){ - char aSpace[150]; + KeyInfo *pKeyInfo = pCsr->pKeyInfo; + VdbeSorter *pSorter = pCsr->pSorter; + char *aSpace = pSorter->aSpace; + int nSpace = pSorter->nSpace; UnpackedRecord *r2; int i; - r2 = sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, aSpace, sizeof(aSpace)); - if( r2==0 ) return SQLITE_NOMEM; + if( aSpace==0 ){ + nSpace = ROUND8(sizeof(UnpackedRecord))+(pKeyInfo->nField+1)*sizeof(Mem); + aSpace = (char *)sqlite3Malloc(nSpace); + if( aSpace==0 ) return SQLITE_NOMEM; + pSorter->aSpace = aSpace; + pSorter->nSpace = nSpace; + } + + /* This call cannot fail. As the memory is already allocated. */ + r2 = sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, aSpace, nSpace); + assert( r2 && (r2->flags & UNPACKED_NEED_FREE)==0 ); + if( bOmitRowid ){ for(i=0; inField-1; i++){ if( r2->aMem[i].flags & MEM_Null ){ @@ -329,7 +344,7 @@ static int vdbeSorterCompare( } *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2); - sqlite3VdbeDeleteUnpackedRecord(r2); + /* sqlite3VdbeDeleteUnpackedRecord(r2); */ return SQLITE_OK; } @@ -366,7 +381,7 @@ static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){ }else{ int res; int rc = vdbeSorterCompare( - pCsr->pKeyInfo, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res + pCsr, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res ); if( rc!=SQLITE_OK ) return rc; if( res<=0 ){ @@ -429,6 +444,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ sqlite3OsCloseFree(pSorter->pTemp1); } vdbeSorterRecordFree(db, pSorter->pRecord); + sqlite3_free(pSorter->aSpace); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } @@ -454,7 +470,7 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ */ static int vdbeSorterMerge( sqlite3 *db, /* Database handle */ - KeyInfo *pKeyInfo, /* Collation functions to use in comparison */ + VdbeCursor *pCsr, /* For pKeyInfo */ SorterRecord *p1, /* First list to merge */ SorterRecord *p2, /* Second list to merge */ SorterRecord **ppOut /* OUT: Head of merged list */ @@ -473,7 +489,7 @@ static int vdbeSorterMerge( }else{ int res; rc = vdbeSorterCompare( - pKeyInfo, 0, p1->pVal, p1->nVal, p2->pVal, p2->nVal, &res + pCsr, 0, p1->pVal, p1->nVal, p2->pVal, p2->nVal, &res ); if( rc!=SQLITE_OK ){ vdbeSorterRecordFree(db, p1); @@ -510,7 +526,6 @@ static int vdbeSorterSort(sqlite3 *db, VdbeCursor *pCsr){ SorterRecord **aSlot; SorterRecord *p; VdbeSorter *pSorter = pCsr->pSorter; - KeyInfo *pKeyInfo = pCsr->pKeyInfo; aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); if( !aSlot ){ @@ -522,7 +537,7 @@ static int vdbeSorterSort(sqlite3 *db, VdbeCursor *pCsr){ SorterRecord *pNext = p->pNext; p->pNext = 0; for(i=0; rc==SQLITE_OK && aSlot[i]; i++){ - rc = vdbeSorterMerge(db, pKeyInfo, p, aSlot[i], &p); + rc = vdbeSorterMerge(db, pCsr, p, aSlot[i], &p); aSlot[i] = 0; } if( rc!=SQLITE_OK ){ @@ -536,7 +551,7 @@ static int vdbeSorterSort(sqlite3 *db, VdbeCursor *pCsr){ p = 0; for(i=0; i<64; i++){ if( rc==SQLITE_OK ){ - rc = vdbeSorterMerge(db, pKeyInfo, p, aSlot[i], &p); + rc = vdbeSorterMerge(db, pCsr, p, aSlot[i], &p); }else{ vdbeSorterRecordFree(db, aSlot[i]); } @@ -550,7 +565,7 @@ static int vdbeSorterSort(sqlite3 *db, VdbeCursor *pCsr){ for(pTmp2=pSorter->pRecord; pTmp2 && rc==SQLITE_OK; pTmp2=pTmp2->pNext){ if( pTmp1 ){ int res; - rc = vdbeSorterCompare(pKeyInfo, + rc = vdbeSorterCompare(pCsr, 0, pTmp1->pVal, pTmp1->nVal, pTmp2->pVal, pTmp2->nVal, &res ); assert( rc!=SQLITE_OK || res<0 ); @@ -913,7 +928,7 @@ int sqlite3VdbeSorterCompare( void *pKey; int nKey; /* Sorter key to compare pVal with */ pKey = vdbeSorterRowkey(pSorter, &nKey); - rc = vdbeSorterCompare(pCsr->pKeyInfo, 1, pVal->z, pVal->n, pKey, nKey, pRes); + rc = vdbeSorterCompare(pCsr, 1, pVal->z, pVal->n, pKey, nKey, pRes); assert( rc!=SQLITE_OK || *pRes<=0 ); return rc; } From 7733a4db4bce40774142c4fe86b13e6721703e12 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Sep 2011 18:03:16 +0000 Subject: [PATCH 06/10] Combine two malloc calls in vdbesort.c. FossilOrigin-Name: cf48ad8353e28339d00f448bb729e10a7f2aad72 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 8 +------- src/vdbesort.c | 27 +++++++++------------------ 4 files changed, 18 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 3c9140975f..b71aeffabe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\snumber\sof\smalloc()\scalls\smade\swhen\screating\san\sindex\son\smore\sthan\s2\scolumns. -D 2011-09-02T15:41:33.781 +C Combine\stwo\smalloc\scalls\sin\svdbesort.c. +D 2011-09-02T18:03:16.975 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,7 +124,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 28a4fe55327ff708bfaf9d4326d02686f7a553c3 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c bc2099e7d3c22c52b2c54349b9c07c04f2a810d0 +F src/btree.c dc3f244447fd5aade08f66d475e2a6316bbef4e7 F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c dc367138cb3625e6d42b389e05d7267aece5753c @@ -245,7 +245,7 @@ F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c a49d44498cc19af9bdbdc8e9fa882dd26cc17066 +F src/vdbesort.c b56437b8b3a0a17a2b110d37633dc9217c72fd60 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 71075673c625f243969c3f34c73f28f378924007 -R 16433222f8a97f40a1aa6bc60c6c3db0 +P 065b0c9858da0ebb41722f3c56bdaf62f28b2f2c +R 06bb7ef7ee2f3a8f904497ea31ee4edb U dan -Z 8c008bdf93f87673bd214d386e7f6cf5 +Z ad89820409358feeee63c15f85c9298f diff --git a/manifest.uuid b/manifest.uuid index c870d49dd3..4ac5f7ba98 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -065b0c9858da0ebb41722f3c56bdaf62f28b2f2c \ No newline at end of file +cf48ad8353e28339d00f448bb729e10a7f2aad72 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index b335579bd0..f20afb6bf3 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7288,13 +7288,7 @@ int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ BtShared *pBt = p->pBt; int rc; sqlite3BtreeEnter(p); - if( (pBt->openFlags&BTREE_SINGLE) ){ - pBt->nPage = 0; - sqlite3PagerTruncateImage(pBt->pPager, 1); - rc = newDatabase(pBt); - }else{ - rc = btreeDropTable(p, iTable, piMoved); - } + rc = btreeDropTable(p, iTable, piMoved); sqlite3BtreeLeave(p); return rc; } diff --git a/src/vdbesort.c b/src/vdbesort.c index 63c6f543b8..74399edf5d 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -93,7 +93,7 @@ typedef struct SorterRecord SorterRecord; ** being merged (rounded up to the next power of 2). */ struct VdbeSorter { - int nInMemory; /* Current size of b-tree contents as PMA */ + int nInMemory; /* Current size of pRecord list as PMA */ int nTree; /* Used size of aTree/aIter (power of 2) */ VdbeSorterIter *aIter; /* Array of iterators to merge */ int *aTree; /* Current state of incremental merge */ @@ -422,7 +422,6 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ SorterRecord *pNext; for(p=pRecord; p; p=pNext){ pNext = p->pNext; - sqlite3DbFree(db, p->pVal); sqlite3DbFree(db, p); } } @@ -630,7 +629,6 @@ static int vdbeSorterListToPMA(sqlite3 *db, VdbeCursor *pCsr){ iOff += p->nVal; } - sqlite3DbFree(db, p->pVal); sqlite3DbFree(db, p); } @@ -657,28 +655,21 @@ int sqlite3VdbeSorterWrite( Mem *pVal /* Memory cell containing record */ ){ VdbeSorter *pSorter = pCsr->pSorter; - int rc; - SorterRecord *pNew; + int rc = SQLITE_OK; /* Return Code */ + SorterRecord *pNew; /* New list element */ assert( pSorter ); pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; - pNew = (SorterRecord *)sqlite3DbMallocZero(db, sizeof(SorterRecord)); + pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n + sizeof(SorterRecord)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3VdbeMemMakeWriteable(pVal); - if( rc==SQLITE_OK ){ - pNew->pVal = pVal->z; - pNew->nVal = pVal->n; - pVal->zMalloc = 0; - sqlite3VdbeMemSetNull(pVal); - pNew->pNext = pSorter->pRecord; - pSorter->pRecord = pNew; - }else{ - sqlite3DbFree(db, pNew); - rc = SQLITE_NOMEM; - } + pNew->pVal = (void *)&pNew[1]; + memcpy(pNew->pVal, pVal->z, pVal->n); + pNew->nVal = pVal->n; + pNew->pNext = pSorter->pRecord; + pSorter->pRecord = pNew; } /* See if the contents of the sorter should now be written out. They From 34163c68310d010907847a68cf0a9e46dba31aef Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 Sep 2011 21:42:33 +0000 Subject: [PATCH 07/10] Remove some dead code. Fix a faulty assert(). Improve some variable names. FossilOrigin-Name: a9a64592cf88580cb254fb0aac65a2f2085976ec --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/btree.c | 1 - src/pager.c | 27 --------------------------- src/pager.h | 3 --- src/vdbesort.c | 41 +++++++++++++---------------------------- 6 files changed, 24 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index b71aeffabe..1f98484757 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Combine\stwo\smalloc\scalls\sin\svdbesort.c. -D 2011-09-02T18:03:16.975 +C Remove\ssome\sdead\scode.\s\sFix\sa\sfaulty\sassert().\s\sImprove\ssome\svariable\snames. +D 2011-09-02T21:42:33.425 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,7 +124,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 28a4fe55327ff708bfaf9d4326d02686f7a553c3 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c dc3f244447fd5aade08f66d475e2a6316bbef4e7 +F src/btree.c 4d46fe30b8bc920f68b7d58a5f45316fa5d023ec F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c dc367138cb3625e6d42b389e05d7267aece5753c @@ -167,8 +167,8 @@ F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c 10e0c4dcdbec8d4189890fdf3e71b32efae194e3 F src/os_win.c 45de13c6c3501cfd6469b2b34149b823060e39f4 -F src/pager.c 817f7f7140c9fa2641f28e6330e924708ddd870d -F src/pager.h 2bab1b2ea4eac58663b5833e3522e36b5ff63447 +F src/pager.c 5545863e4e246e1744cfb6993821c6e4b63ffb64 +F src/pager.h 6bea8d1949db33768de1c5b4133b267b40845f8b F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 49e718c095810c6b3334e3a6d89970aceaddefce F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 @@ -245,7 +245,7 @@ F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c b56437b8b3a0a17a2b110d37633dc9217c72fd60 +F src/vdbesort.c 2ae618bec4bec5faa075e82078008763666c655d F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 065b0c9858da0ebb41722f3c56bdaf62f28b2f2c -R 06bb7ef7ee2f3a8f904497ea31ee4edb -U dan -Z ad89820409358feeee63c15f85c9298f +P cf48ad8353e28339d00f448bb729e10a7f2aad72 +R d2091eb5dd8da1e99612b5b3cd229b4c +U drh +Z e655a0d761c2ed1b9d5806522d72983b diff --git a/manifest.uuid b/manifest.uuid index 4ac5f7ba98..2cb8cd7e4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cf48ad8353e28339d00f448bb729e10a7f2aad72 \ No newline at end of file +a9a64592cf88580cb254fb0aac65a2f2085976ec \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f20afb6bf3..f0eb376dec 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7285,7 +7285,6 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return rc; } int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ - BtShared *pBt = p->pBt; int rc; sqlite3BtreeEnter(p); rc = btreeDropTable(p, iTable, piMoved); diff --git a/src/pager.c b/src/pager.c index 373d06aeca..f40d2af74b 100644 --- a/src/pager.c +++ b/src/pager.c @@ -621,7 +621,6 @@ struct Pager { u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ u8 hasSeenStress; /* pagerStress() called one or more times */ - u8 isSorter; /* True for a PAGER_SORTER */ /************************************************************************** ** The following block contains those class members that change during @@ -845,15 +844,6 @@ static int assert_pager_state(Pager *p){ assert( pagerUseWal(p)==0 ); } - /* A sorter is a temp file that never spills to disk and always has - ** the doNotSpill flag set - */ - if( p->isSorter ){ - assert( p->tempFile ); - assert( p->doNotSpill ); - assert( p->fd->pMethods==0 ); - } - /* If changeCountDone is set, a RESERVED lock or greater must be held ** on the file. */ @@ -4557,12 +4547,6 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ -#ifndef SQLITE_OMIT_MERGE_SORT - if( flags & PAGER_SORTER ){ - pPager->doNotSpill = 1; - pPager->isSorter = 1; - } -#endif *ppPager = pPager; return SQLITE_OK; @@ -6107,17 +6091,6 @@ int sqlite3PagerIsMemdb(Pager *pPager){ return MEMDB; } -#ifndef SQLITE_OMIT_MERGE_SORT -/* -** Return true if the pager has seen a pagerStress callback. -*/ -int sqlite3PagerUnderStress(Pager *pPager){ - assert( pPager->isSorter ); - assert( pPager->doNotSpill ); - return pPager->hasSeenStress; -} -#endif - /* ** Check that there are at least nSavepoint savepoints open. If there are ** currently less than nSavepoints open, then open one or more savepoints diff --git a/src/pager.h b/src/pager.h index ccd7467d6a..2a02eff75c 100644 --- a/src/pager.h +++ b/src/pager.h @@ -156,9 +156,6 @@ const char *sqlite3PagerJournalname(Pager*); int sqlite3PagerNosync(Pager*); void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); -#ifndef SQLITE_OMIT_MERGE_SORT -int sqlite3PagerUnderStress(Pager*); -#endif /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); diff --git a/src/vdbesort.c b/src/vdbesort.c index 74399edf5d..c1ea8cadd3 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -102,8 +102,8 @@ struct VdbeSorter { sqlite3_file *pTemp1; /* PMA file 1 */ int nPMA; /* Number of PMAs stored in pTemp1 */ SorterRecord *pRecord; /* Head of in-memory record list */ - int nLimit1; /* Minimum PMA size, in bytes */ - int nLimit2; /* Maximum PMA size, in bytes */ + int mnPmaSize; /* Minimum PMA size, in bytes */ + int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ char *aSpace; /* Space for UnpackRecord() */ int nSpace; /* Size of aSpace in bytes */ }; @@ -400,16 +400,20 @@ static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){ */ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ int pgsz; /* Page size of main database */ + int mxCache; /* Cache size */ + VdbeSorter *pSorter; /* The new sorter */ assert( pCsr->pKeyInfo && pCsr->pBt==0 ); - pCsr->pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter)); - if( pCsr->pSorter==0 ){ + pCsr->pSorter = pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter)); + if( pSorter==0 ){ return SQLITE_NOMEM; } pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - pCsr->pSorter->nLimit1 = 10 * pgsz; - pCsr->pSorter->nLimit2 = db->aDb[0].pSchema->cache_size * pgsz; + pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; + mxCache = db->aDb[0].pSchema->cache_size; + if( mxCachemxPmaSize = mxCache * pgsz; return SQLITE_OK; } @@ -557,25 +561,6 @@ static int vdbeSorterSort(sqlite3 *db, VdbeCursor *pCsr){ } pSorter->pRecord = p; -#if 0 - { - SorterRecord *pTmp1 = 0; - SorterRecord *pTmp2; - for(pTmp2=pSorter->pRecord; pTmp2 && rc==SQLITE_OK; pTmp2=pTmp2->pNext){ - if( pTmp1 ){ - int res; - rc = vdbeSorterCompare(pCsr, - 0, pTmp1->pVal, pTmp1->nVal, pTmp2->pVal, pTmp2->nVal, &res - ); - assert( rc!=SQLITE_OK || res<0 ); - } - pTmp1 = pTmp2; - } - } -#endif - - if( rc!=SQLITE_OK ){ - } sqlite3_free(aSlot); return rc; } @@ -682,8 +667,8 @@ int sqlite3VdbeSorterWrite( ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. */ if( rc==SQLITE_OK && ( - (pSorter->nInMemory>pSorter->nLimit2) - || (pSorter->nInMemory>pSorter->nLimit1 && sqlite3HeapNearlyFull()) + (pSorter->nInMemory>pSorter->mxPmaSize) + || (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull()) )){ rc = vdbeSorterListToPMA(db, pCsr); pSorter->nInMemory = 0; @@ -920,7 +905,7 @@ int sqlite3VdbeSorterCompare( pKey = vdbeSorterRowkey(pSorter, &nKey); rc = vdbeSorterCompare(pCsr, 1, pVal->z, pVal->n, pKey, nKey, pRes); - assert( rc!=SQLITE_OK || *pRes<=0 ); + assert( rc!=SQLITE_OK || pVal->db->mallocFailed || (*pRes)<=0 ); return rc; } From ca892a725284f1ce7d9f4f5e3c4f75f4f01f055b Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 3 Sep 2011 00:17:51 +0000 Subject: [PATCH 08/10] The build works again with -DSQLITE_OMIT_MERGE_SORT. The merge-sorter now avoids spilling to disk (letting the in-memory linked list grow without bound) if PRAGMA temp_store=3. FossilOrigin-Name: 68e26c4487696d194ee85370380e4b0e56d206ee --- manifest | 20 ++++++++-------- manifest.uuid | 2 +- src/build.c | 61 +++++++++++++++++++++++-------------------------- src/sqliteInt.h | 8 ------- src/vdbe.c | 27 ++++++++++++++++++---- src/vdbeInt.h | 2 +- src/vdbesort.c | 14 +++++++----- 7 files changed, 70 insertions(+), 64 deletions(-) diff --git a/manifest b/manifest index 1f98484757..6c9743bfa6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\sdead\scode.\s\sFix\sa\sfaulty\sassert().\s\sImprove\ssome\svariable\snames. -D 2011-09-02T21:42:33.425 +C The\sbuild\sworks\sagain\swith\s-DSQLITE_OMIT_MERGE_SORT.\s\sThe\smerge-sorter\snow\navoids\sspilling\sto\sdisk\s(letting\sthe\sin-memory\slinked\slist\sgrow\swithout\nbound)\sif\sPRAGMA\stemp_store=3. +D 2011-09-03T00:17:51.023 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -127,7 +127,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 4d46fe30b8bc920f68b7d58a5f45316fa5d023ec F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 -F src/build.c dc367138cb3625e6d42b389e05d7267aece5753c +F src/build.c 851e81f26a75abbb98bd99a7c5f10e8670d867bb F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c e3132ec65240b2e2f3d50831021eac387f27584d @@ -183,7 +183,7 @@ F src/select.c 32d0f4e5513362706b8973e7f1b87cd0885dfbf5 F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd F src/sqlite.h.in 0a6c9c23337fd1352c5c75a613ff9533aa7d91cb F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 -F src/sqliteInt.h 723cda73a33c91f5a0a145f4c0ced45d94921079 +F src/sqliteInt.h c7e37ee49b1a922ddcd18fa98dd750efa4d2db14 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -238,14 +238,14 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c da9c7efc48dc79d7785f3f17a1c3df514bf18489 +F src/vdbe.c d4c8224cc931c6082557501d7f822fb12f273922 F src/vdbe.h c1eeedacab6bcf1e7c2cf8203ba9763a616f9a86 -F src/vdbeInt.h a255da14be8c2794ce38e0d2142877bb29df9105 +F src/vdbeInt.h 693d6ac6810298fc6b4c503cfbe3f99a240f40af F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c 2ae618bec4bec5faa075e82078008763666c655d +F src/vdbesort.c aa7ad2ef91ad7d33eb614d3fb1c4cbd068b321ad F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P cf48ad8353e28339d00f448bb729e10a7f2aad72 -R d2091eb5dd8da1e99612b5b3cd229b4c +P a9a64592cf88580cb254fb0aac65a2f2085976ec +R 4edd7e2341e47b4ce23f768c53615b8c U drh -Z e655a0d761c2ed1b9d5806522d72983b +Z cbdc1caefa14410ba02a8248a6138d65 diff --git a/manifest.uuid b/manifest.uuid index 2cb8cd7e4f..2bc60c00a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a9a64592cf88580cb254fb0aac65a2f2085976ec \ No newline at end of file +68e26c4487696d194ee85370380e4b0e56d206ee \ No newline at end of file diff --git a/src/build.c b/src/build.c index 0de1020b9b..27130ef281 100644 --- a/src/build.c +++ b/src/build.c @@ -2335,15 +2335,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3 *db = pParse->db; /* The database connection */ int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); - /* Set bUseSorter to use OP_OpenSorter, or clear it to insert directly - ** into the index. The sorter is used unless either OMIT_MERGE_SORT is - ** defined or the system is configured to store temp files in-memory. */ -#ifdef SQLITE_OMIT_MERGE_SORT - static const int bUseSorter = 0; -#else - const int bUseSorter = !sqlite3TempInMemory(pParse->db); -#endif - #ifndef SQLITE_OMIT_AUTHORIZATION if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0, db->aDb[iDb].zName ) ){ @@ -2369,11 +2360,11 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeChangeP5(v, 1); } +#ifndef SQLITE_OMIT_MERGE_SORT /* Open the sorter cursor if we are to use one. */ - if( bUseSorter ){ - iSorter = pParse->nTab++; - sqlite3VdbeAddOp4(v, OP_OpenSorter, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); - } + iSorter = pParse->nTab++; + sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); +#endif /* Open the table. Loop through all rows of the table, inserting index ** records into the sorter. */ @@ -2383,24 +2374,27 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ regRecord = sqlite3GetTempReg(pParse); regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); - if( bUseSorter ){ - sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); - sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); - sqlite3VdbeJumpHere(v, addr1); - addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); - if( pIndex->onError!=OE_None ){ - int j2 = sqlite3VdbeCurrentAddr(v) + 3; - sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); - addr2 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp3(v, OP_SorterCompare, iSorter, j2, regRecord); - sqlite3HaltConstraint( - pParse, OE_Abort, "indexed columns are not unique", P4_STATIC - ); - }else{ - addr2 = sqlite3VdbeCurrentAddr(v); - } - sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); - }else if( pIndex->onError!=OE_None ){ +#ifndef SQLITE_OMIT_MERGE_SORT + sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); + sqlite3VdbeJumpHere(v, addr1); + addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); + if( pIndex->onError!=OE_None ){ + int j2 = sqlite3VdbeCurrentAddr(v) + 3; + sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); + addr2 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_SorterCompare, iSorter, j2, regRecord); + sqlite3HaltConstraint( + pParse, OE_Abort, "indexed columns are not unique", P4_STATIC + ); + }else{ + addr2 = sqlite3VdbeCurrentAddr(v); + } + sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); + sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); +#else + if( pIndex->onError!=OE_None ){ const int regRowid = regIdxKey + pIndex->nColumn; const int j2 = sqlite3VdbeCurrentAddr(v) + 2; void * const pRegKey = SQLITE_INT_TO_PTR(regIdxKey); @@ -2418,10 +2412,11 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3HaltConstraint( pParse, OE_Abort, "indexed columns are not unique", P4_STATIC); } - sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, bUseSorter); + sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 0); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); +#endif sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3VdbeAddOp2(v, bUseSorter ? OP_SorterNext : OP_Next, iSorter, addr2); + sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp1(v, OP_Close, iTab); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 694f4694eb..f27fea2b3a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -372,14 +372,6 @@ # define SQLITE_TEMP_STORE 1 #endif -/* -** If all temporary storage is in-memory, then omit the external merge-sort -** logic since it is superfluous. -*/ -#if SQLITE_TEMP_STORE==3 && !defined(SQLITE_OMIT_MERGE_SORT) -# define SQLITE_OMIT_MERGE_SORT -#endif - /* ** GCC does not define the offsetof() macro so we'll have to do it ** ourselves. diff --git a/src/vdbe.c b/src/vdbe.c index d9310e0563..acf113515f 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3215,15 +3215,19 @@ case OP_OpenEphemeral: { ** a transient index that is specifically designed to sort large ** tables using an external merge-sort algorithm. */ -case OP_SorterOpen: -case OP_OpenSorter: { +case OP_SorterOpen: { VdbeCursor *pCx; +#ifndef SQLITE_OMIT_MERGE_SORT pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->pKeyInfo = pOp->p4.pKeyInfo; pCx->pKeyInfo->enc = ENC(p->db); pCx->isSorter = 1; rc = sqlite3VdbeSorterInit(db, pCx); +#else + pOp->opcode = OP_OpenEphemeral; + pc--; +#endif break; } @@ -4102,10 +4106,15 @@ case OP_SorterCompare: { */ case OP_SorterData: { VdbeCursor *pC; +#ifndef SQLITE_OMIT_MERGE_SORT pOut = &aMem[pOp->p2]; pC = p->apCsr[pOp->p1]; assert( pC->isSorter ); rc = sqlite3VdbeSorterRowkey(pC, pOut); +#else + pOp->opcode = OP_RowKey; + pc--; +#endif break; } @@ -4148,8 +4157,7 @@ case OP_RowData: { assert( pC!=0 ); assert( pC->nullRow==0 ); assert( pC->pseudoTableReg==0 ); - assert( pC->isSorter==(pOp->opcode==OP_SorterData) ); - + assert( !pC->isSorter ); assert( pC->pCursor!=0 ); pCrsr = pC->pCursor; assert( sqlite3BtreeCursorIsValid(pCrsr) ); @@ -4307,6 +4315,9 @@ case OP_Last: { /* jump */ ** correctly optimizing out sorts. */ case OP_SorterSort: /* jump */ +#ifdef SQLITE_OMIT_MERGE_SORT + pOp->opcode = OP_Sort; +#endif case OP_Sort: { /* jump */ #ifdef SQLITE_TEST sqlite3_sort_count++; @@ -4385,6 +4396,9 @@ case OP_Rewind: { /* jump */ ** number P5-1 in the prepared statement is incremented. */ case OP_SorterNext: /* jump */ +#ifdef SQLITE_OMIT_MERGE_SORT + pOp->opcode = OP_Next; +#endif case OP_Prev: /* jump */ case OP_Next: { /* jump */ VdbeCursor *pC; @@ -4434,7 +4448,10 @@ case OP_Next: { /* jump */ ** This instruction only works for indices. The equivalent instruction ** for tables is OP_Insert. */ -case OP_SorterInsert: +case OP_SorterInsert: /* in2 */ +#ifdef SQLITE_OMIT_MERGE_SORT + pOp->opcode = OP_IdxInsert; +#endif case OP_IdxInsert: { /* in2 */ VdbeCursor *pC; BtCursor *pCrsr; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 68ad8dcbf7..7fbd5693fb 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -65,7 +65,7 @@ struct VdbeCursor { i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ i64 lastRowid; /* Last rowid from a Next or NextIdx operation */ - VdbeSorter *pSorter; /* Sorter object for OP_OpenSorter cursors */ + VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ /* Result of last sqlite3BtreeMoveto() done by an OP_NotExists or ** OP_IsUnique opcode on this cursor. */ diff --git a/src/vdbesort.c b/src/vdbesort.c index c1ea8cadd3..e8135f0f14 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -409,11 +409,13 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ return SQLITE_NOMEM; } - pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; - mxCache = db->aDb[0].pSchema->cache_size; - if( mxCachemxPmaSize = mxCache * pgsz; + if( !sqlite3TempInMemory(db) ){ + pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); + pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; + mxCache = db->aDb[0].pSchema->cache_size; + if( mxCachemxPmaSize = mxCache * pgsz; + } return SQLITE_OK; } @@ -666,7 +668,7 @@ int sqlite3VdbeSorterWrite( ** * The total memory allocated for the in-memory list is greater ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. */ - if( rc==SQLITE_OK && ( + if( rc==SQLITE_OK && pSorter->mxPmaSize>0 && ( (pSorter->nInMemory>pSorter->mxPmaSize) || (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull()) )){ From 8b1ea14f8ca6883fdea4bd64b7f2b8a97c8401ac Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 3 Sep 2011 14:36:13 +0000 Subject: [PATCH 09/10] Reduce the number of VdbeRecordUnpack() calls made in vdbesort.c. FossilOrigin-Name: 666c2c3cff51dac2ba5689b75705d99c3705673b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbesort.c | 24 +++++++++++++++++------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 6c9743bfa6..dd25184f0e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sbuild\sworks\sagain\swith\s-DSQLITE_OMIT_MERGE_SORT.\s\sThe\smerge-sorter\snow\navoids\sspilling\sto\sdisk\s(letting\sthe\sin-memory\slinked\slist\sgrow\swithout\nbound)\sif\sPRAGMA\stemp_store=3. -D 2011-09-03T00:17:51.023 +C Reduce\sthe\snumber\sof\sVdbeRecordUnpack()\scalls\smade\sin\svdbesort.c. +D 2011-09-03T14:36:13.912 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -245,7 +245,7 @@ F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c aa7ad2ef91ad7d33eb614d3fb1c4cbd068b321ad +F src/vdbesort.c 5e38e7e1f5f6900b1b8bea49decd23c69221aed9 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P a9a64592cf88580cb254fb0aac65a2f2085976ec -R 4edd7e2341e47b4ce23f768c53615b8c -U drh -Z cbdc1caefa14410ba02a8248a6138d65 +P 68e26c4487696d194ee85370380e4b0e56d206ee +R f6e37dce79fea0b74c498631cfb45ff1 +U dan +Z cc6b7d2a23c219696518afb4690c5277 diff --git a/manifest.uuid b/manifest.uuid index 2bc60c00a6..af74b9885a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -68e26c4487696d194ee85370380e4b0e56d206ee \ No newline at end of file +666c2c3cff51dac2ba5689b75705d99c3705673b \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index e8135f0f14..9e169c3e36 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -303,6 +303,9 @@ static int vdbeSorterIterInit( ** field. For the purposes of the comparison, ignore it. Also, if bOmitRowid ** is true and key1 contains even a single NULL value, it is considered to ** be less than key2. Even if key2 also contains NULL values. +** +** If pKey2 is passed a NULL pointer, then it is assumed that the pCsr->aSpace +** has been allocated and contains an unpacked record that is used as key2. */ static int vdbeSorterCompare( VdbeCursor *pCsr, /* Cursor object (for pKeyInfo) */ @@ -326,15 +329,20 @@ static int vdbeSorterCompare( pSorter->nSpace = nSpace; } - /* This call cannot fail. As the memory is already allocated. */ - r2 = sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, aSpace, nSpace); - assert( r2 && (r2->flags & UNPACKED_NEED_FREE)==0 ); + if( pKey2 ){ + /* This call cannot fail. As the memory is already allocated. */ + r2 = sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, aSpace, nSpace); + assert( r2 && (r2->flags & UNPACKED_NEED_FREE)==0 ); + assert( r2==aSpace ); + }else{ + r2 = (UnpackedRecord *)aSpace; + assert( !bOmitRowid ); + } if( bOmitRowid ){ for(i=0; inField-1; i++){ if( r2->aMem[i].flags & MEM_Null ){ *pRes = -1; - sqlite3VdbeDeleteUnpackedRecord(r2); return SQLITE_OK; } } @@ -344,7 +352,6 @@ static int vdbeSorterCompare( } *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2); - /* sqlite3VdbeDeleteUnpackedRecord(r2); */ return SQLITE_OK; } @@ -483,6 +490,7 @@ static int vdbeSorterMerge( int rc = SQLITE_OK; SorterRecord *pFinal = 0; SorterRecord **pp = &pFinal; + int bKey2InSpace = 0; /* True if pCsr->aSpace contains key2 */ while( p1 || p2 ){ if( p1==0 ){ @@ -493,8 +501,8 @@ static int vdbeSorterMerge( p1 = 0; }else{ int res; - rc = vdbeSorterCompare( - pCsr, 0, p1->pVal, p1->nVal, p2->pVal, p2->nVal, &res + rc = vdbeSorterCompare(pCsr, 0, + p1->pVal, p1->nVal, (bKey2InSpace ? 0 : p2->pVal), p2->nVal, &res ); if( rc!=SQLITE_OK ){ vdbeSorterRecordFree(db, p1); @@ -507,10 +515,12 @@ static int vdbeSorterMerge( *pp = p1; pp = &p1->pNext; p1 = p1->pNext; + bKey2InSpace = 1; }else{ *pp = p2; pp = &p2->pNext; p2 = p2->pNext; + bKey2InSpace = 0; } *pp = 0; } From 04a962fe02e030f10f3bb1b984bb7a49da647a3b Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 3 Sep 2011 16:42:38 +0000 Subject: [PATCH 10/10] Simplification and performance tweaks in vdbeSorterMerge(). FossilOrigin-Name: 99e34bdce4ccca15b79159b03b96787e7a7ff85b --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/vdbesort.c | 56 ++++++++++++++++++++++---------------------------- 3 files changed, 32 insertions(+), 40 deletions(-) diff --git a/manifest b/manifest index dd25184f0e..aa9396df26 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\snumber\sof\sVdbeRecordUnpack()\scalls\smade\sin\svdbesort.c. -D 2011-09-03T14:36:13.912 +C Simplification\sand\sperformance\stweaks\sin\svdbeSorterMerge(). +D 2011-09-03T16:42:38.728 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -245,7 +245,7 @@ F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c 5e38e7e1f5f6900b1b8bea49decd23c69221aed9 +F src/vdbesort.c e6d6f0c2aa003f7cbdea8c9be47a15a8e854fb97 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 68e26c4487696d194ee85370380e4b0e56d206ee -R f6e37dce79fea0b74c498631cfb45ff1 -U dan -Z cc6b7d2a23c219696518afb4690c5277 +P 666c2c3cff51dac2ba5689b75705d99c3705673b +R cfceda33d1a4c239172824f11326438a +U drh +Z 0da609a19e6bdcd2176629a62a8129db diff --git a/manifest.uuid b/manifest.uuid index af74b9885a..f0c89b9c6d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -666c2c3cff51dac2ba5689b75705d99c3705673b \ No newline at end of file +99e34bdce4ccca15b79159b03b96787e7a7ff85b \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 9e169c3e36..6681aad52d 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -490,44 +490,36 @@ static int vdbeSorterMerge( int rc = SQLITE_OK; SorterRecord *pFinal = 0; SorterRecord **pp = &pFinal; - int bKey2InSpace = 0; /* True if pCsr->aSpace contains key2 */ + void *pVal2 = p2 ? p2->pVal : 0; - while( p1 || p2 ){ - if( p1==0 ){ - *pp = p2; - p2 = 0; - }else if( p2==0 ){ - *pp = p1; - p1 = 0; - }else{ - int res; - rc = vdbeSorterCompare(pCsr, 0, - p1->pVal, p1->nVal, (bKey2InSpace ? 0 : p2->pVal), p2->nVal, &res - ); - if( rc!=SQLITE_OK ){ - vdbeSorterRecordFree(db, p1); - vdbeSorterRecordFree(db, p2); - vdbeSorterRecordFree(db, pFinal); - pFinal = 0; - break; - } - if( res<=0 ){ - *pp = p1; - pp = &p1->pNext; - p1 = p1->pNext; - bKey2InSpace = 1; - }else{ - *pp = p2; - pp = &p2->pNext; - p2 = p2->pNext; - bKey2InSpace = 0; - } + while( p1 && p2 ){ + int res; + rc = vdbeSorterCompare(pCsr, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res); + if( rc!=SQLITE_OK ){ *pp = 0; + vdbeSorterRecordFree(db, p1); + vdbeSorterRecordFree(db, p2); + vdbeSorterRecordFree(db, pFinal); + *ppOut = 0; + return rc; + } + if( res<=0 ){ + *pp = p1; + pp = &p1->pNext; + p1 = p1->pNext; + pVal2 = 0; + }else{ + *pp = p2; + pp = &p2->pNext; + p2 = p2->pNext; + if( p2==0 ) break; + pVal2 = p2->pVal; } } + *pp = p1 ? p1 : p2; *ppOut = pFinal; - return rc; + return SQLITE_OK; } /*