From cd8629e4bbc8f91e634337d72453b3963f50990e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 13 Nov 2013 12:27:25 +0000 Subject: [PATCH 1/4] Add the ability to use an index even if the left-most columns of the index are unconstrainted, provided that the left-most columns have few distinct values. FossilOrigin-Name: 27dd5993d1ae5625eb94bf406421eb390d001be9 --- manifest | 17 ++++++---- manifest.uuid | 2 +- src/where.c | 87 +++++++++++++++++++++++++++++++++++++++++--------- src/whereInt.h | 5 +++ 4 files changed, 88 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 1bdb81fbcb..5d4c908bea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\san\sunnecessary\sOP_IfNull\swhile\sdoing\san\sindexed\ssearch. -D 2013-11-13T08:55:02.907 +C Add\sthe\sability\sto\suse\san\sindex\seven\sif\sthe\sleft-most\scolumns\sof\sthe\sindex\nare\sunconstrainted,\sprovided\sthat\sthe\sleft-most\scolumns\shave\sfew\sdistinct\nvalues. +D 2013-11-13T12:27:25.442 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -293,8 +293,8 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 346944c20b9c1ac191f16cfc1100dfbe58fa98d4 -F src/whereInt.h 63c8345d01d12ded6250b72e7c16855f0a358041 +F src/where.c 15170b152697e8819de5909031d72ad52a28a7ef +F src/whereInt.h a0e8fa5364122c6cb27c6fa340b257a05c592bfb F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1138,7 +1138,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4e7e805e1139b2dc05d85e86e5c8254e5d361bf2 -R 24bac22755e297cd5d201f4faa2841ac +P 5196000930600d0cd931b87e864507791b9dab08 +R d1d503c972952b4e29bfacf424d1532a +T *branch * skip-scan +T *sym-skip-scan * +T -sym-trunk * U drh -Z b52683f99c1f13f39fe1493c1b7d78a3 +Z c64da1e6d225931e6198f9f21b69efb9 diff --git a/manifest.uuid b/manifest.uuid index 5a991f73f9..309dd4f350 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5196000930600d0cd931b87e864507791b9dab08 \ No newline at end of file +27dd5993d1ae5625eb94bf406421eb390d001be9 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 617d373cbb..021cab187d 100644 --- a/src/where.c +++ b/src/where.c @@ -2422,7 +2422,7 @@ static int codeEqualityTerm( /* ** Generate code that will evaluate all == and IN constraints for an -** index. +** index scan. ** ** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). ** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 @@ -2437,9 +2437,15 @@ static int codeEqualityTerm( ** The only thing it does is allocate the pLevel->iMem memory cell and ** compute the affinity string. ** -** This routine always allocates at least one memory cell and returns -** the index of that memory cell. The code that -** calls this routine will use that memory cell to store the termination +** The nExtraReg parameter is 0 or 1. It is 0 if all WHERE clause constraints +** are == or IN and are covered by the nEq. nExtraReg is 1 if there is +** an inequality constraint (such as the "c>=5 AND c<10" in the example) that +** occurs after the nEq quality constraints. +** +** This routine allocates a range of nEq+nExtraReg memory cells and returns +** the index of the first memory cell in that range. The code that +** calls this routine will use that memory range to store keys for +** start and termination conditions of the loop. ** key value of the loop. If one or more IN operators appear, then ** this routine allocates an additional nEq memory cells for internal ** use. @@ -2466,7 +2472,8 @@ static int codeAllEqualityTerms( int nExtraReg, /* Number of extra registers to allocate */ char **pzAff /* OUT: Set to point to affinity string */ ){ - int nEq; /* The number of == or IN constraints to code */ + u16 nEq; /* The number of == or IN constraints to code */ + u16 nSkip; /* Number of left-most columns to skip */ Vdbe *v = pParse->pVdbe; /* The vm under construction */ Index *pIdx; /* The index being used for this loop */ WhereTerm *pTerm; /* A single constraint term */ @@ -2480,6 +2487,7 @@ static int codeAllEqualityTerms( pLoop = pLevel->pWLoop; assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); nEq = pLoop->u.btree.nEq; + nSkip = pLoop->u.btree.nSkip; pIdx = pLoop->u.btree.pIndex; assert( pIdx!=0 ); @@ -2494,14 +2502,28 @@ static int codeAllEqualityTerms( pParse->db->mallocFailed = 1; } + if( nSkip ){ + int iIdxCur = pLevel->iIdxCur; + sqlite3VdbeAddOp2(v, (bRev?OP_Last:OP_Rewind), iIdxCur, pLevel->addrNxt); + pLevel->addrSkip = sqlite3VdbeCurrentAddr(v); + pLevel->opSkip = bRev ? OP_SeekLt : OP_SeekGt; + pLevel->p3Skip = regBase; + pLevel->p4Skip = nSkip; + for(j=0; jaiColumn[j]>=0 ); + VdbeComment((v, "%s", pIdx->pTable->aCol[pIdx->aiColumn[j]].zName)); + } + } + /* Evaluate the equality constraints */ assert( zAff==0 || (int)strlen(zAff)>=nEq ); - for(j=0; jaLTerm[j]; assert( pTerm!=0 ); - /* The following true for indices with redundant columns. + /* The following testcase is true for indices with redundant columns. ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); @@ -2575,7 +2597,8 @@ static void explainAppendTerm( */ static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){ Index *pIndex = pLoop->u.btree.pIndex; - int nEq = pLoop->u.btree.nEq; + u16 nEq = pLoop->u.btree.nEq; + u16 nSkip = pLoop->u.btree.nSkip; int i, j; Column *aCol = pTab->aCol; i16 *aiColumn = pIndex->aiColumn; @@ -2589,7 +2612,14 @@ static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){ sqlite3StrAccumAppend(&txt, " (", 2); for(i=0; inKeyCol ) ? "rowid" : aCol[aiColumn[i]].zName; - explainAppendTerm(&txt, i, z, "="); + if( i>=nSkip ){ + explainAppendTerm(&txt, i, z, "="); + }else{ + if( i ) sqlite3StrAccumAppend(&txt, " AND ", 5); + sqlite3StrAccumAppend(&txt, "ANY(", 4); + sqlite3StrAccumAppend(&txt, z, -1); + sqlite3StrAccumAppend(&txt, ")", 1); + } } j = i; @@ -2952,8 +2982,9 @@ static Bitmask codeOneLoopStart( OP_IdxGE, /* 1: (end_constraints && !bRev) */ OP_IdxLT /* 2: (end_constraints && bRev) */ }; - int nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ - int isMinQuery = 0; /* If this is an optimized SELECT min(x).. */ + u16 nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ + u16 nSkip = pLoop->u.btree.nSkip; /* Number of left index terms to skip */ + int isMinQuery = 0; /* If this is an optimized SELECT min(x).. */ int regBase; /* Base register holding constraint values */ int r1; /* Temp register */ WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ @@ -2971,6 +3002,7 @@ static Bitmask codeOneLoopStart( pIdx = pLoop->u.btree.pIndex; iIdxCur = pLevel->iIdxCur; + assert( nEq>=nSkip ); /* If this loop satisfies a sort order (pOrderBy) request that ** was passed to this function to implement a "SELECT min(x) ..." @@ -2984,8 +3016,7 @@ static Bitmask codeOneLoopStart( && (pWInfo->bOBSat!=0) && (pIdx->nKeyCol>nEq) ){ - /* assert( pOrderBy->nExpr==1 ); */ - /* assert( pOrderBy->a[0].pExpr->iColumn==pIdx->aiColumn[nEq] ); */ + assert( nSkip==0 ); isMinQuery = 1; nExtraReg = 1; } @@ -3536,6 +3567,7 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3ExplainBegin(v); for(i=0; inLTerm; i++){ WhereTerm *pTerm = p->aLTerm[i]; + if( pTerm==0 ) continue; sqlite3ExplainPrintf(v, " (%d) #%-2d ", i+1, (int)(pTerm-pWC->a)); sqlite3ExplainPush(v); whereExplainTerm(v, pTerm); @@ -3848,7 +3880,8 @@ static int whereLoopAddBtreeIndex( WhereScan scan; /* Iterator for WHERE terms */ Bitmask saved_prereq; /* Original value of pNew->prereq */ u16 saved_nLTerm; /* Original value of pNew->nLTerm */ - int saved_nEq; /* Original value of pNew->u.btree.nEq */ + u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ + u16 saved_nSkip; /* Original value of pNew->u.btree.nSkip */ u32 saved_wsFlags; /* Original value of pNew->wsFlags */ LogEst saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ @@ -3883,12 +3916,24 @@ static int whereLoopAddBtreeIndex( pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, pProbe); saved_nEq = pNew->u.btree.nEq; + saved_nSkip = pNew->u.btree.nSkip; saved_nLTerm = pNew->nLTerm; saved_wsFlags = pNew->wsFlags; saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; pNew->rSetup = 0; rLogSize = estLog(sqlite3LogEst(pProbe->aiRowEst[0])); + if( pTerm==0 + && saved_nEq==saved_nSkip + && saved_nEq+1nKeyCol + && pProbe->aiRowEst[saved_nEq+1]>50 + ){ + pNew->u.btree.nEq++; + pNew->u.btree.nSkip++; + pNew->aLTerm[pNew->nLTerm++] = 0; + pNew->wsFlags |= WHERE_SKIP_SCAN; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul); + } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -4006,6 +4051,7 @@ static int whereLoopAddBtreeIndex( } pNew->prereq = saved_prereq; pNew->u.btree.nEq = saved_nEq; + pNew->u.btree.nSkip = saved_nSkip; pNew->wsFlags = saved_wsFlags; pNew->nOut = saved_nOut; pNew->nLTerm = saved_nLTerm; @@ -4152,6 +4198,7 @@ static int whereLoopAddBtree( if( pTerm->prereqRight & pNew->maskSelf ) continue; if( termCanDriveIndex(pTerm, pSrc, 0) ){ pNew->u.btree.nEq = 1; + pNew->u.btree.nSkip = 0; pNew->u.btree.pIndex = 0; pNew->nLTerm = 1; pNew->aLTerm[0] = pTerm; @@ -4181,6 +4228,7 @@ static int whereLoopAddBtree( continue; /* Partial index inappropriate for this query */ } pNew->u.btree.nEq = 0; + pNew->u.btree.nSkip = 0; pNew->nLTerm = 0; pNew->iSortIdx = 0; pNew->rSetup = 0; @@ -4715,6 +4763,7 @@ static int wherePathSatisfiesOrderBy( /* Skip over == and IS NULL terms */ if( ju.btree.nEq + && pLoop->u.btree.nSkip==0 && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 ){ if( i & WO_ISNULL ){ @@ -5140,6 +5189,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pWC = &pWInfo->sWC; pLoop = pBuilder->pNew; pLoop->wsFlags = 0; + pLoop->u.btree.nSkip = 0; pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0); if( pTerm ){ pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; @@ -5713,6 +5763,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeNoopComment((v, "End WHERE-core")); sqlite3ExprCacheClear(pParse); for(i=pWInfo->nLevel-1; i>=0; i--){ + int addr; pLevel = &pWInfo->a[i]; pLoop = pLevel->pWLoop; sqlite3VdbeResolveLabel(v, pLevel->addrCont); @@ -5732,8 +5783,14 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3DbFree(db, pLevel->u.in.aInLoop); } sqlite3VdbeResolveLabel(v, pLevel->addrBrk); + if( pLevel->addrSkip ){ + addr = sqlite3VdbeAddOp4Int(v, pLevel->opSkip, pLevel->iIdxCur, 0, + pLevel->p3Skip, pLevel->p4Skip); + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrSkip); + sqlite3VdbeJumpHere(v, pLevel->addrSkip-1); + sqlite3VdbeJumpHere(v, addr); + } if( pLevel->iLeftJoin ){ - int addr; addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || (pLoop->wsFlags & WHERE_INDEXED)!=0 ); diff --git a/src/whereInt.h b/src/whereInt.h index b73353f732..62c93471e5 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -65,12 +65,16 @@ struct WhereLevel { int iIdxCur; /* The VDBE cursor used to access pIdx */ int addrBrk; /* Jump here to break out of the loop */ int addrNxt; /* Jump here to start the next IN combination */ + int addrSkip; /* Jump here for next iteration of skip-scan */ int addrCont; /* Jump here to continue with the next loop cycle */ int addrFirst; /* First instruction of interior of the loop */ int addrBody; /* Beginning of the body of this loop */ u8 iFrom; /* Which entry in the FROM clause */ u8 op, p5; /* Opcode and P5 of the opcode that ends the loop */ + u8 opSkip; /* Opcode to terminate the skip-scan */ int p1, p2; /* Operands of the opcode used to ends the loop */ + int p3Skip; /* P3 operand for the skip-scan terminator */ + u16 p4Skip; /* P4 operand for the skip-scan terminator */ union { /* Information that depends on pWLoop->wsFlags */ struct { int nIn; /* Number of entries in aInLoop[] */ @@ -455,3 +459,4 @@ struct WhereInfo { #define WHERE_ONEROW 0x00001000 /* Selects no more than one row */ #define WHERE_MULTI_OR 0x00002000 /* OR using multiple indices */ #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ +#define WHERE_SKIP_SCAN 0x00008000 /* Uses the skip-scan algorithm */ From c2b23e7a9832fdeb2658171f39a5cbdaf5ccdb67 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 13 Nov 2013 15:32:15 +0000 Subject: [PATCH 2/4] Add test cases for skip-scan. Enhance "do_test" so that if the expected result is of the form "/*..*/" or "~/*..*/" it treats the expected result as a glob pattern rather than as a regular expression. Fix a bug in ANALYZE result loading associated with WITHOUT ROWID tables. FossilOrigin-Name: d3e6e9b2a74074c05429d3c341c23525504351ab --- manifest | 18 +++--- manifest.uuid | 2 +- src/analyze.c | 8 ++- test/skipscan1.test | 148 ++++++++++++++++++++++++++++++++++++++++++++ test/tester.tcl | 21 +++++-- 5 files changed, 179 insertions(+), 18 deletions(-) create mode 100644 test/skipscan1.test diff --git a/manifest b/manifest index 5d4c908bea..7158caf1b6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sability\sto\suse\san\sindex\seven\sif\sthe\sleft-most\scolumns\sof\sthe\sindex\nare\sunconstrainted,\sprovided\sthat\sthe\sleft-most\scolumns\shave\sfew\sdistinct\nvalues. -D 2013-11-13T12:27:25.442 +C Add\stest\scases\sfor\sskip-scan.\s\sEnhance\s"do_test"\sso\sthat\sif\sthe\sexpected\sresult\nis\sof\sthe\sform\s"/*..*/"\sor\s"~/*..*/"\sit\streats\sthe\sexpected\sresult\sas\sa\sglob\npattern\srather\sthan\sas\sa\sregular\sexpression.\s\sFix\sa\sbug\sin\sANALYZE\sresult\nloading\sassociated\swith\sWITHOUT\sROWID\stables. +D 2013-11-13T15:32:15.331 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c 27f0c132aa0679189837e0addf8762e7fd6831b6 +F src/analyze.c 581d5c18ce89c6f45d4dca65914d0de5b4dad41f F src/attach.c 0a17c9364895316ca4f52d06a97a72c0af1ae8b3 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 2f1987981139bd2f6d8c728d64bf09fb387443c3 @@ -805,6 +805,7 @@ F test/shell5.test 46c8c18d62732415c4fe084816c13d559831705e F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 +F test/skipscan1.test 63af32c300be545417410ea2ce44e78c5b2e34b1 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 @@ -835,7 +836,7 @@ F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 0a48d94222d50e6e50d72ac103606c4f8e7cbb81 -F test/tester.tcl 3f675f00d22de3595be25dc2d2cba7ff623ef058 +F test/tester.tcl bce6b929932498383ce92431da6a96432c690bf7 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1138,10 +1139,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5196000930600d0cd931b87e864507791b9dab08 -R d1d503c972952b4e29bfacf424d1532a -T *branch * skip-scan -T *sym-skip-scan * -T -sym-trunk * +P 27dd5993d1ae5625eb94bf406421eb390d001be9 +R 56958c69a5559324c31384471b37b6dc U drh -Z c64da1e6d225931e6198f9f21b69efb9 +Z 2ff45f7229fb8bf180b3b10c0362f0e0 diff --git a/manifest.uuid b/manifest.uuid index 309dd4f350..1dae906437 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27dd5993d1ae5625eb94bf406421eb390d001be9 \ No newline at end of file +d3e6e9b2a74074c05429d3c341c23525504351ab \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index f1094f79f9..3d5c4f6bec 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1428,10 +1428,12 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ if( pTable==0 ){ return 0; } - if( argv[1] ){ - pIndex = sqlite3FindIndex(pInfo->db, argv[1], pInfo->zDatabase); - }else{ + if( argv[1]==0 ){ pIndex = 0; + }else if( sqlite3_stricmp(argv[0],argv[1])==0 ){ + pIndex = sqlite3PrimaryKeyIndex(pTable); + }else{ + pIndex = sqlite3FindIndex(pInfo->db, argv[1], pInfo->zDatabase); } z = argv[2]; diff --git a/test/skipscan1.test b/test/skipscan1.test new file mode 100644 index 0000000000..0931592f94 --- /dev/null +++ b/test/skipscan1.test @@ -0,0 +1,148 @@ +# 2013-11-13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests of the "skip-scan" query strategy. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test skipscan1-1.1 { + CREATE TABLE t1(a TEXT, b INT, c INT, d INT); + CREATE INDEX t1abc ON t1(a,b,c); + INSERT INTO t1 VALUES('abc',123,4,5); + INSERT INTO t1 VALUES('abc',234,5,6); + INSERT INTO t1 VALUES('abc',234,6,7); + INSERT INTO t1 VALUES('abc',345,7,8); + INSERT INTO t1 VALUES('def',567,8,9); + INSERT INTO t1 VALUES('def',345,9,10); + INSERT INTO t1 VALUES('bcd',100,6,11); + + /* Fake the sqlite_stat1 table so that the query planner believes + ** the table contains thousands of rows and that the first few + ** columns are not selective. */ + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1 VALUES('t1','t1abc','10000 5000 2000 10'); + ANALYZE sqlite_master; +} {} + +# Simple queries that leave the first one or two columns of the +# index unconstrainted. +# +do_execsql_test skipscan1-1.2 { + SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a; +} {abc 345 7 8 | def 345 9 10 |} +do_execsql_test skipscan1-1.2eqp { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a; +} {/* USING INDEX t1abc (ANY(a) AND b=?)*/} +do_execsql_test skipscan1-1.2sort { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a; +} {~/*ORDER BY*/} + +do_execsql_test skipscan1-1.3 { + SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a DESC; +} {def 345 9 10 | abc 345 7 8 |} +do_execsql_test skipscan1-1.3eqp { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a; +} {/* USING INDEX t1abc (ANY(a) AND b=?)*/} +do_execsql_test skipscan1-1.3sort { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a; +} {~/*ORDER BY*/} + +do_execsql_test skipscan1-1.4 { + SELECT a,b,c,d,'|' FROM t1 WHERE c=6 ORDER BY a, b, c; +} {abc 234 6 7 | bcd 100 6 11 |} +do_execsql_test skipscan1-1.4eqp { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE c=6 ORDER BY a, b, c; +} {/* USING INDEX t1abc (ANY(a) AND ANY(b) AND c=?)*/} +do_execsql_test skipscan1-1.4sort { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE c=6 ORDER BY a, b, c; +} {~/*ORDER BY*/} + +# Joins +# +do_execsql_test skipscan1-1.5 { + CREATE TABLE t1j(x TEXT, y INTEGER); + INSERT INTO t1j VALUES('one',1),('six',6),('ninty-nine',99); + SELECT x, a, b, c, d, '|' FROM t1j, t1 WHERE c=y ORDER BY +a; +} {six abc 234 6 7 | six bcd 100 6 11 |} +do_execsql_test skipscan1-1.5eqp { + EXPLAIN QUERY PLAN + SELECT x, a, b, c, d, '|' FROM t1j, t1 WHERE c=y ORDER BY +a; +} {/* INDEX t1abc (ANY(a) AND ANY(b) AND c=?)*/} + +do_execsql_test skipscan1-1.6 { + SELECT x, a, b, c, d, '|' FROM t1j LEFT JOIN t1 ON c=y ORDER BY +y, +a; +} {one {} {} {} {} | six abc 234 6 7 | six bcd 100 6 11 | ninty-nine {} {} {} {} |} +do_execsql_test skipscan1-1.6eqp { + EXPLAIN QUERY PLAN + SELECT x, a, b, c, d, '|' FROM t1j LEFT JOIN t1 ON c=y ORDER BY +y, +a; +} {/* INDEX t1abc (ANY(a) AND ANY(b) AND c=?)*/} + +do_execsql_test skipscan1-2.1 { + CREATE TABLE t2(a TEXT, b INT, c INT, d INT, + PRIMARY KEY(a,b,c)); + INSERT INTO t2 SELECT * FROM t1; + + /* Fake the sqlite_stat1 table so that the query planner believes + ** the table contains thousands of rows and that the first few + ** columns are not selective. */ + ANALYZE; + UPDATE sqlite_stat1 SET stat='10000 5000 2000 10' WHERE idx NOT NULL; + ANALYZE sqlite_master; +} {} + +do_execsql_test skipscan1-2.2 { + SELECT a,b,c,d,'|' FROM t2 WHERE b=345 ORDER BY a; +} {abc 345 7 8 | def 345 9 10 |} +do_execsql_test skipscan1-2.2eqp { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t2 WHERE b=345 ORDER BY a; +} {/* USING INDEX sqlite_autoindex_t2_1 (ANY(a) AND b=?)*/} +do_execsql_test skipscan1-2.2sort { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t2 WHERE b=345 ORDER BY a; +} {~/*ORDER BY*/} + + +do_execsql_test skipscan1-3.1 { + CREATE TABLE t3(a TEXT, b INT, c INT, d INT, + PRIMARY KEY(a,b,c)) WITHOUT ROWID; + INSERT INTO t3 SELECT * FROM t1; + + /* Fake the sqlite_stat1 table so that the query planner believes + ** the table contains thousands of rows and that the first few + ** columns are not selective. */ + ANALYZE; + UPDATE sqlite_stat1 SET stat='10000 5000 2000 10' WHERE idx NOT NULL; + ANALYZE sqlite_master; +} {} + +do_execsql_test skipscan1-3.2 { + SELECT a,b,c,d,'|' FROM t3 WHERE b=345 ORDER BY a; +} {abc 345 7 8 | def 345 9 10 |} +do_execsql_test skipscan1-3.2eqp { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t3 WHERE b=345 ORDER BY a; +} {/* INDEX sqlite_autoindex_t3_1 (ANY(a) AND b=?)*/} +do_execsql_test skipscan1-3.2sort { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t3 WHERE b=345 ORDER BY a; +} {~/*ORDER BY*/} + +finish_test diff --git a/test/tester.tcl b/test/tester.tcl index 2d047be5f4..c1272210d0 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -616,11 +616,24 @@ proc do_test {name cmd expected} { # regular expression PATTERN matches the result. "~/PATTERN/" means # the regular expression must not match. if {[string index $expected 0]=="~"} { - set re [string map {# {[-0-9.]+}} [string range $expected 2 end-1]] - set ok [expr {![regexp $re $result]}] + set re [string range $expected 2 end-1] + if {[string index $re 0]=="*"} { + # If the regular expression begins with * then treat it as a glob instead + set ok [string match $re $result] + } else { + set re [string map {# {[-0-9.]+}} $re] + set ok [regexp $re $result] + } + set ok [expr {!$ok}] } else { - set re [string map {# {[-0-9.]+}} [string range $expected 1 end-1]] - set ok [regexp $re $result] + set re [string range $expected 1 end-1] + if {[string index $re 0]=="*"} { + # If the regular expression begins with * then treat it as a glob instead + set ok [string match $re $result] + } else { + set re [string map {# {[-0-9.]+}} $re] + set ok [regexp $re $result] + } } } elseif {[regexp {^~?\*.*\*$} $expected]} { # "expected" is of the form "*GLOB*" then the result if correct if From 2e5ef4ed77dbace183fff5add3bb6e017f4a8417 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 13 Nov 2013 16:58:54 +0000 Subject: [PATCH 3/4] Improve the way that skip-scan loops are constructued. Add test cases. Improved the scoring of skip-scan loops. FossilOrigin-Name: 5e75ab93881b85801cb4ebf70f2063ff7c51ac19 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 28 +++++++++++++++------------- src/whereInt.h | 5 +---- test/skipscan1.test | 35 +++++++++++++++++++++++++++++++---- 5 files changed, 56 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 7158caf1b6..7626e0a77f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sfor\sskip-scan.\s\sEnhance\s"do_test"\sso\sthat\sif\sthe\sexpected\sresult\nis\sof\sthe\sform\s"/*..*/"\sor\s"~/*..*/"\sit\streats\sthe\sexpected\sresult\sas\sa\sglob\npattern\srather\sthan\sas\sa\sregular\sexpression.\s\sFix\sa\sbug\sin\sANALYZE\sresult\nloading\sassociated\swith\sWITHOUT\sROWID\stables. -D 2013-11-13T15:32:15.331 +C Improve\sthe\sway\sthat\sskip-scan\sloops\sare\sconstructued.\s\sAdd\stest\scases.\nImproved\sthe\sscoring\sof\sskip-scan\sloops. +D 2013-11-13T16:58:54.547 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -293,8 +293,8 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 15170b152697e8819de5909031d72ad52a28a7ef -F src/whereInt.h a0e8fa5364122c6cb27c6fa340b257a05c592bfb +F src/where.c f50428d2e93ca3e5c541cc1c2084833b48db947c +F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -805,7 +805,7 @@ F test/shell5.test 46c8c18d62732415c4fe084816c13d559831705e F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 -F test/skipscan1.test 63af32c300be545417410ea2ce44e78c5b2e34b1 +F test/skipscan1.test bc65ecb228eba396c404aa038f49798340f0677f F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 @@ -1139,7 +1139,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 27dd5993d1ae5625eb94bf406421eb390d001be9 -R 56958c69a5559324c31384471b37b6dc +P d3e6e9b2a74074c05429d3c341c23525504351ab +R 2b49a424145c7566c72f185654c1a259 U drh -Z 2ff45f7229fb8bf180b3b10c0362f0e0 +Z 03bb67f899f79fcc8cd9bd00c60852e3 diff --git a/manifest.uuid b/manifest.uuid index 1dae906437..514af577f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d3e6e9b2a74074c05429d3c341c23525504351ab \ No newline at end of file +5e75ab93881b85801cb4ebf70f2063ff7c51ac19 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 021cab187d..c2824862ea 100644 --- a/src/where.c +++ b/src/where.c @@ -2504,11 +2504,11 @@ static int codeAllEqualityTerms( if( nSkip ){ int iIdxCur = pLevel->iIdxCur; - sqlite3VdbeAddOp2(v, (bRev?OP_Last:OP_Rewind), iIdxCur, pLevel->addrNxt); - pLevel->addrSkip = sqlite3VdbeCurrentAddr(v); - pLevel->opSkip = bRev ? OP_SeekLt : OP_SeekGt; - pLevel->p3Skip = regBase; - pLevel->p4Skip = nSkip; + sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); + j = sqlite3VdbeAddOp0(v, OP_Goto); + pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLt:OP_SeekGt), + iIdxCur, 0, regBase, nSkip); + sqlite3VdbeJumpHere(v, j); for(j=0; jaiColumn[j]>=0 ); @@ -3928,11 +3928,13 @@ static int whereLoopAddBtreeIndex( && saved_nEq+1nKeyCol && pProbe->aiRowEst[saved_nEq+1]>50 ){ + LogEst nIter; pNew->u.btree.nEq++; pNew->u.btree.nSkip++; pNew->aLTerm[pNew->nLTerm++] = 0; - pNew->wsFlags |= WHERE_SKIP_SCAN; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul); + pNew->wsFlags |= WHERE_SKIPSCAN; + nIter = sqlite3LogEst(pProbe->aiRowEst[0]/pProbe->aiRowEst[saved_nEq+1]); + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter); } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 0; @@ -3969,8 +3971,10 @@ static int whereLoopAddBtreeIndex( pNew->u.btree.nEq++; pNew->nOut = nRowEst + nInMul + nIn; }else if( pTerm->eOperator & (WO_EQ) ){ - assert( (pNew->wsFlags & (WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 - || nInMul==0 ); + assert( + (pNew->wsFlags & (WHERE_COLUMN_NULL|WHERE_COLUMN_IN|WHERE_SKIPSCAN))!=0 + || nInMul==0 + ); pNew->wsFlags |= WHERE_COLUMN_EQ; if( iCol<0 || (pProbe->onError!=OE_None && nInMul==0 @@ -5784,11 +5788,9 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ } sqlite3VdbeResolveLabel(v, pLevel->addrBrk); if( pLevel->addrSkip ){ - addr = sqlite3VdbeAddOp4Int(v, pLevel->opSkip, pLevel->iIdxCur, 0, - pLevel->p3Skip, pLevel->p4Skip); sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrSkip); - sqlite3VdbeJumpHere(v, pLevel->addrSkip-1); - sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeJumpHere(v, pLevel->addrSkip); + sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); } if( pLevel->iLeftJoin ){ addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); diff --git a/src/whereInt.h b/src/whereInt.h index 62c93471e5..56646c55e6 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -71,10 +71,7 @@ struct WhereLevel { int addrBody; /* Beginning of the body of this loop */ u8 iFrom; /* Which entry in the FROM clause */ u8 op, p5; /* Opcode and P5 of the opcode that ends the loop */ - u8 opSkip; /* Opcode to terminate the skip-scan */ int p1, p2; /* Operands of the opcode used to ends the loop */ - int p3Skip; /* P3 operand for the skip-scan terminator */ - u16 p4Skip; /* P4 operand for the skip-scan terminator */ union { /* Information that depends on pWLoop->wsFlags */ struct { int nIn; /* Number of entries in aInLoop[] */ @@ -459,4 +456,4 @@ struct WhereInfo { #define WHERE_ONEROW 0x00001000 /* Selects no more than one row */ #define WHERE_MULTI_OR 0x00002000 /* OR using multiple indices */ #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ -#define WHERE_SKIP_SCAN 0x00008000 /* Uses the skip-scan algorithm */ +#define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ diff --git a/test/skipscan1.test b/test/skipscan1.test index 0931592f94..eba7383eaa 100644 --- a/test/skipscan1.test +++ b/test/skipscan1.test @@ -74,22 +74,49 @@ do_execsql_test skipscan1-1.4sort { SELECT a,b,c,d,'|' FROM t1 WHERE c=6 ORDER BY a, b, c; } {~/*ORDER BY*/} +do_execsql_test skipscan1-1.5 { + SELECT a,b,c,d,'|' FROM t1 WHERE c IN (6,7) ORDER BY a, b, c; +} {abc 234 6 7 | abc 345 7 8 | bcd 100 6 11 |} +do_execsql_test skipscan1-1.5eqp { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE c IN (6,7) ORDER BY a, b, c; +} {/* USING INDEX t1abc (ANY(a) AND ANY(b) AND c=?)*/} +do_execsql_test skipscan1-1.5sort { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE c IN (6,7) ORDER BY a, b, c; +} {~/*ORDER BY*/} + +do_execsql_test skipscan1-1.6 { + SELECT a,b,c,d,'|' FROM t1 WHERE c BETWEEN 6 AND 7 ORDER BY a, b, c; +} {abc 234 6 7 | abc 345 7 8 | bcd 100 6 11 |} +do_execsql_test skipscan1-1.6eqp { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE c BETWEEN 6 AND 7 ORDER BY a, b, c; +} {/* USING INDEX t1abc (ANY(a) AND ANY(b) AND c>? AND c Date: Wed, 13 Nov 2013 17:24:38 +0000 Subject: [PATCH 4/4] Add VDBE comments to the beginning and end of skip-scan loops. FossilOrigin-Name: 0c85d93b52311dee7980d977be6ed0dc70b060c1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 7626e0a77f..390483589c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\sway\sthat\sskip-scan\sloops\sare\sconstructued.\s\sAdd\stest\scases.\nImproved\sthe\sscoring\sof\sskip-scan\sloops. -D 2013-11-13T16:58:54.547 +C Add\sVDBE\scomments\sto\sthe\sbeginning\sand\send\sof\sskip-scan\sloops. +D 2013-11-13T17:24:39.000 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -293,7 +293,7 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c f50428d2e93ca3e5c541cc1c2084833b48db947c +F src/where.c 1d19a1d49a608441fa697ae32627399073be8dcf F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1139,7 +1139,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d3e6e9b2a74074c05429d3c341c23525504351ab -R 2b49a424145c7566c72f185654c1a259 +P 5e75ab93881b85801cb4ebf70f2063ff7c51ac19 +R 324d2d53e7ac48858b72f1751f7e7610 U drh -Z 03bb67f899f79fcc8cd9bd00c60852e3 +Z 7cfb6326eedc7a21a0bd15f40c4f97e2 diff --git a/manifest.uuid b/manifest.uuid index 514af577f1..ac2ea40722 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5e75ab93881b85801cb4ebf70f2063ff7c51ac19 \ No newline at end of file +0c85d93b52311dee7980d977be6ed0dc70b060c1 \ No newline at end of file diff --git a/src/where.c b/src/where.c index c2824862ea..2fd7c600c8 100644 --- a/src/where.c +++ b/src/where.c @@ -2505,6 +2505,7 @@ static int codeAllEqualityTerms( if( nSkip ){ int iIdxCur = pLevel->iIdxCur; sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); + VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); j = sqlite3VdbeAddOp0(v, OP_Goto); pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLt:OP_SeekGt), iIdxCur, 0, regBase, nSkip); @@ -5789,6 +5790,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeResolveLabel(v, pLevel->addrBrk); if( pLevel->addrSkip ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrSkip); + VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName)); sqlite3VdbeJumpHere(v, pLevel->addrSkip); sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); }