From d7d71470226bcc59799713b95ad9e10f9eff63cc Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Oct 2014 19:57:16 +0000 Subject: [PATCH 1/8] Change the 0x800 bit of SQLITE_TESTCTRL_OPTIMIZATIONS so that it disables the loading of STAT3 and STAT4 content, not just the using of that content. Change the internal name of that bit to SQLITE_Stat34. FossilOrigin-Name: ca3b00c44ec52d209642a5ba9ef82e085fac39db --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/analyze.c | 2 +- src/sqliteInt.h | 2 +- src/test1.c | 3 ++- src/where.c | 4 +--- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 8290b14522..0f41135502 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Take\ssteps\sto\savoid\smisestimating\srange\squery\scosts\sbased\son\sSTAT4\sdata\ndue\sto\sthe\sroundoff\serror\sof\sconverting\sfrom\sintegers\sto\sLogEst\sand\sback\nto\sintegers. -D 2014-10-22T15:27:05.734 +C Change\sthe\s0x800\sbit\sof\sSQLITE_TESTCTRL_OPTIMIZATIONS\sso\sthat\sit\sdisables\nthe\sloading\sof\sSTAT3\sand\sSTAT4\scontent,\snot\sjust\sthe\susing\sof\sthat\scontent.\nChange\sthe\sinternal\sname\sof\sthat\sbit\sto\sSQLITE_Stat34. +D 2014-10-22T19:57:16.520 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c 567c94b763b67f7abda06dbf0ba34b0343ed9447 +F src/analyze.c afbcca663c3f3625340b8e30d440cd7a97ded6bc F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e @@ -232,12 +232,12 @@ F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h d6d423b0f62846eb441236bc15417aeede2ebbdc +F src/sqliteInt.h 6e9e125698c1e5c78a51050ea61f179a281c766d F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb F src/tclsqlite.c c67d310c833046cccc192125d64ad422ab882684 -F src/test1.c 518db4305d76b29dd9da3f022ca899c8fcdf9fc7 +F src/test1.c 63d4b1707c4052cf9c05c1cbb4a62666d70a0b48 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 5099c42e24c63969b3cf3b52e18c1a36cb841a34 +F src/where.c 2cd9e0af718d736459ae9d3b0f4532b4a80640d0 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3e922208b68563489c7766abb9afb4885113e7b8 -R 5c508733aebac553de79291a7eabd709 +P 3c933bf95f291f7957580d823dce92c981375a5c +R 3efbbff265eea57d45c364e4606463ea U drh -Z a3d7104c7f173585d7e788d6453aa065 +Z b4247f0ab9b19770a6354621b6169ee4 diff --git a/manifest.uuid b/manifest.uuid index b84af28cee..fff2e6a3fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3c933bf95f291f7957580d823dce92c981375a5c \ No newline at end of file +ca3b00c44ec52d209642a5ba9ef82e085fac39db \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 67bba1a28a..597885237c 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1861,7 +1861,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load the statistics from the sqlite_stat4 table. */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && OptimizationEnabled(db, SQLITE_Stat34) ){ int lookasideEnabled = db->lookaside.bEnabled; db->lookaside.bEnabled = 0; rc = loadStat4(db, sInfo.zDatabase); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5409f7086f..1b3138be44 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1214,7 +1214,7 @@ struct sqlite3 { #define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ #define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ -#define SQLITE_Stat3 0x0800 /* Use the SQLITE_STAT3 table */ +#define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* diff --git a/src/test1.c b/src/test1.c index 85a16488ba..802aa99b5b 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6293,7 +6293,8 @@ static int optimization_control( { "transitive", SQLITE_Transitive }, { "subquery-coroutine", SQLITE_SubqCoroutine }, { "omit-noop-join", SQLITE_OmitNoopJoin }, - { "stat3", SQLITE_Stat3 }, + { "stat3", SQLITE_Stat34 }, + { "stat4", SQLITE_Stat34 }, }; if( objc!=4 ){ diff --git a/src/where.c b/src/where.c index f708d7fad7..198a6c3fdd 100644 --- a/src/where.c +++ b/src/where.c @@ -1350,7 +1350,7 @@ static void exprAnalyze( if( pExpr->op==TK_NOTNULL && pExpr->pLeft->op==TK_COLUMN && pExpr->pLeft->iColumn>=0 - && OptimizationEnabled(db, SQLITE_Stat3) + && OptimizationEnabled(db, SQLITE_Stat34) ){ Expr *pNewExpr; Expr *pLeft = pExpr->pLeft; @@ -2159,7 +2159,6 @@ static int whereRangeScanEst( if( p->nSample>0 && nEqnSampleCol - && OptimizationEnabled(pParse->db, SQLITE_Stat3) ){ if( nEq==pBuilder->nRecValid ){ UnpackedRecord *pRec = pBuilder->pRec; @@ -4422,7 +4421,6 @@ static int whereLoopAddBtreeIndex( if( nInMul==0 && pProbe->nSample && pNew->u.btree.nEq<=pProbe->nSampleCol - && OptimizationEnabled(db, SQLITE_Stat3) && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) ){ Expr *pExpr = pTerm->pExpr; From b7288e28e48235e8f7671ae7b613f4c5c7251c44 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Oct 2014 20:07:19 +0000 Subject: [PATCH 2/8] Disable the use of strchrnul() unless specifically enabled by compile-time options. FossilOrigin-Name: e580470db77d6da970c755102790e603fb26b3c6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 6 +----- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 0f41135502..ae5d008c78 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\s0x800\sbit\sof\sSQLITE_TESTCTRL_OPTIMIZATIONS\sso\sthat\sit\sdisables\nthe\sloading\sof\sSTAT3\sand\sSTAT4\scontent,\snot\sjust\sthe\susing\sof\sthat\scontent.\nChange\sthe\sinternal\sname\sof\sthat\sbit\sto\sSQLITE_Stat34. -D 2014-10-22T19:57:16.520 +C Disable\sthe\suse\sof\sstrchrnul()\sunless\sspecifically\senabled\sby\scompile-time\noptions. +D 2014-10-22T20:07:19.558 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 -F src/printf.c 6b79bbd063dcbadca4cf617a4cde255bcc13ea64 +F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3c933bf95f291f7957580d823dce92c981375a5c -R 3efbbff265eea57d45c364e4606463ea +P ca3b00c44ec52d209642a5ba9ef82e085fac39db +R 8aff7924cf3a3f93f4ee99fb545d1f5f U drh -Z b4247f0ab9b19770a6354621b6169ee4 +Z c6210337ac863b7dfe5cbe19c541a0aa diff --git a/manifest.uuid b/manifest.uuid index fff2e6a3fa..857c7b0e10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca3b00c44ec52d209642a5ba9ef82e085fac39db \ No newline at end of file +e580470db77d6da970c755102790e603fb26b3c6 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index c0b3c70f6b..1df287fbb6 100644 --- a/src/printf.c +++ b/src/printf.c @@ -21,11 +21,7 @@ ** the glibc version so the glibc version is definitely preferred. */ #if !defined(HAVE_STRCHRNUL) -# if defined(linux) -# define HAVE_STRCHRNUL 1 -# else -# define HAVE_STRCHRNUL 0 -# endif +# define HAVE_STRCHRNUL 0 #endif From 4f81bbb5289cdd248c21775b9e4cdb92e110e139 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Oct 2014 01:01:26 +0000 Subject: [PATCH 3/8] Implement sqlite3_create_collation() by invoking sqlite3_create_collation_v2() with a NULL destructor argument. This saves a little space. FossilOrigin-Name: 9762ad0639cca2fc1ef0573113fb613ce9e7e83e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/main.c | 8 +------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index ae5d008c78..5b87a53662 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\suse\sof\sstrchrnul()\sunless\sspecifically\senabled\sby\scompile-time\noptions. -D 2014-10-22T20:07:19.558 +C Implement\ssqlite3_create_collation()\sby\sinvoking\s\nsqlite3_create_collation_v2()\swith\sa\sNULL\sdestructor\sargument.\s\sThis\ssaves\na\slittle\sspace. +D 2014-10-23T01:01:26.636 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c bbe872b0ac0007bed0ebe1936fc493b039ad4f51 +F src/main.c 8207ece1b4005b1efab55d1505d8ffb45bfced05 F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ca3b00c44ec52d209642a5ba9ef82e085fac39db -R 8aff7924cf3a3f93f4ee99fb545d1f5f +P e580470db77d6da970c755102790e603fb26b3c6 +R f159389d59e4350424d5f438676d787b U drh -Z c6210337ac863b7dfe5cbe19c541a0aa +Z 177632e852e703fce9e831bfbe27f418 diff --git a/manifest.uuid b/manifest.uuid index 857c7b0e10..b0a7f1f074 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e580470db77d6da970c755102790e603fb26b3c6 \ No newline at end of file +9762ad0639cca2fc1ef0573113fb613ce9e7e83e \ No newline at end of file diff --git a/src/main.c b/src/main.c index ea03f2639f..e6cb4cd3df 100644 --- a/src/main.c +++ b/src/main.c @@ -2751,13 +2751,7 @@ int sqlite3_create_collation( void* pCtx, int(*xCompare)(void*,int,const void*,int,const void*) ){ - int rc; - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, 0); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; + return sqlite3_create_collation_v2(db, zName, enc, pCtx, xCompare, 0); } /* From 9769efcc4c785dc7552a2e22782f12e3c56e0169 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 14:32:21 +0000 Subject: [PATCH 4/8] Get the likelihood() functions working on operators like BETWEEN that create virtual terms in the WHERE-clause analysis. FossilOrigin-Name: 03d0498d0f24bec2383d5d79edf25069effecd59 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 29 ++++++++++++++++------------- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 5b87a53662..67132856cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Implement\ssqlite3_create_collation()\sby\sinvoking\s\nsqlite3_create_collation_v2()\swith\sa\sNULL\sdestructor\sargument.\s\sThis\ssaves\na\slittle\sspace. -D 2014-10-23T01:01:26.636 +C Get\sthe\slikelihood()\sfunctions\sworking\son\soperators\slike\sBETWEEN\sthat\ncreate\svirtual\sterms\sin\sthe\sWHERE-clause\sanalysis. +D 2014-10-24T14:32:21.050 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 2cd9e0af718d736459ae9d3b0f4532b4a80640d0 +F src/where.c 746d4f22d75c1f000d6dae251dc65bf755c395a8 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e580470db77d6da970c755102790e603fb26b3c6 -R f159389d59e4350424d5f438676d787b +P 9762ad0639cca2fc1ef0573113fb613ce9e7e83e +R f460b87a81ee1fb775b7700e4252ece6 U drh -Z 177632e852e703fce9e831bfbe27f418 +Z 1428b6fe81396b49c1c6b7022fde286d diff --git a/manifest.uuid b/manifest.uuid index b0a7f1f074..59bf5ae4e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9762ad0639cca2fc1ef0573113fb613ce9e7e83e \ No newline at end of file +03d0498d0f24bec2383d5d79edf25069effecd59 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 198a6c3fdd..d4969fad1e 100644 --- a/src/where.c +++ b/src/where.c @@ -756,6 +756,15 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ } } +/* +** Mark term iChild as being a child of term iParent +*/ +static void markTermAsChild(WhereClause *pWC, int iChild, int iParent){ + pWC->a[iChild].iParent = iParent; + pWC->a[iChild].truthProb = pWC->a[iParent].truthProb; + pWC->a[iParent].nChild++; +} + #if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) /* ** Analyze a term that consists of two or more OR-connected @@ -1053,8 +1062,7 @@ static void exprAnalyzeOrTerm( testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); pTerm = &pWC->a[idxTerm]; - pWC->a[idxNew].iParent = idxTerm; - pTerm->nChild = 1; + markTermAsChild(pWC, idxNew, idxTerm); }else{ sqlite3ExprListDelete(db, pList); } @@ -1156,9 +1164,8 @@ static void exprAnalyze( idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); if( idxNew==0 ) return; pNew = &pWC->a[idxNew]; - pNew->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; if( pExpr->op==TK_EQ && !ExprHasProperty(pExpr, EP_FromJoin) @@ -1215,9 +1222,8 @@ static void exprAnalyze( testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); pTerm = &pWC->a[idxTerm]; - pWC->a[idxNew].iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); } - pTerm->nChild = 2; } #endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ @@ -1292,9 +1298,8 @@ static void exprAnalyze( exprAnalyze(pSrc, pWC, idxNew2); pTerm = &pWC->a[idxTerm]; if( isComplete ){ - pWC->a[idxNew1].iParent = idxTerm; - pWC->a[idxNew2].iParent = idxTerm; - pTerm->nChild = 2; + markTermAsChild(pWC, idxNew1, idxTerm); + markTermAsChild(pWC, idxNew2, idxTerm); } } #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ @@ -1327,9 +1332,8 @@ static void exprAnalyze( pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_MATCH; - pNewTerm->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; pNewTerm->prereqAll = pTerm->prereqAll; } @@ -1369,9 +1373,8 @@ static void exprAnalyze( pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_GT; - pNewTerm->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; pNewTerm->prereqAll = pTerm->prereqAll; } From 4dd96a8315b3f3b2a516513b06754e1887e426eb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 15:26:29 +0000 Subject: [PATCH 5/8] Honor a high likelihood() on range constraints. FossilOrigin-Name: 401235edf40fcd665eaf426cf5155ac6855e8537 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 67132856cf..e98be71834 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sthe\slikelihood()\sfunctions\sworking\son\soperators\slike\sBETWEEN\sthat\ncreate\svirtual\sterms\sin\sthe\sWHERE-clause\sanalysis. -D 2014-10-24T14:32:21.050 +C Honor\sa\shigh\slikelihood()\son\srange\sconstraints. +D 2014-10-24T15:26:29.800 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 746d4f22d75c1f000d6dae251dc65bf755c395a8 +F src/where.c f5c13d9c1929bcc9d571f1e7bf7bfeb8b872ef99 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9762ad0639cca2fc1ef0573113fb613ce9e7e83e -R f460b87a81ee1fb775b7700e4252ece6 +P 03d0498d0f24bec2383d5d79edf25069effecd59 +R eb013b99b9ff12a41d5ffc947e4a6227 U drh -Z 1428b6fe81396b49c1c6b7022fde286d +Z 3215dd92b9608b7b9136fb91eb8178e8 diff --git a/manifest.uuid b/manifest.uuid index 59bf5ae4e6..4ee03d34ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03d0498d0f24bec2383d5d79edf25069effecd59 \ No newline at end of file +401235edf40fcd665eaf426cf5155ac6855e8537 \ No newline at end of file diff --git a/src/where.c b/src/where.c index d4969fad1e..e13b223366 100644 --- a/src/where.c +++ b/src/where.c @@ -2275,12 +2275,15 @@ static int whereRangeScanEst( nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); - /* TUNING: If there is both an upper and lower limit, assume the range is + /* TUNING: If there is both an upper and lower limit and neither limit + ** has an application-defined likelihood(), assume the range is ** reduced by an additional 75%. This means that, by default, an open-ended ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to ** match 1/64 of the index. */ - if( pLower && pUpper ) nNew -= 20; + if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){ + nNew -= 20; + } nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; From 059b2d50e1c6a57ca301f3c9639f92f7e16ff96e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 19:28:09 +0000 Subject: [PATCH 6/8] Enhance the automatic index logic so that it creates a partial index when doing so gives the same answer for less work. FossilOrigin-Name: d95d0313c447f5baeabdb17284d8606331ab7d49 --- manifest | 19 +++++----- manifest.uuid | 2 +- src/expr.c | 85 +++++++++++++++++++++++++++----------------- src/resolve.c | 4 +-- src/sqliteInt.h | 5 ++- src/where.c | 23 ++++++++++-- test/autoindex4.test | 52 +++++++++++++++++++++++++++ 7 files changed, 143 insertions(+), 47 deletions(-) create mode 100644 test/autoindex4.test diff --git a/manifest b/manifest index e98be71834..e831ecb526 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Honor\sa\shigh\slikelihood()\son\srange\sconstraints. -D 2014-10-24T15:26:29.800 +C Enhance\sthe\sautomatic\sindex\slogic\sso\sthat\sit\screates\sa\spartial\sindex\swhen\ndoing\sso\sgives\sthe\ssame\sanswer\sfor\sless\swork. +D 2014-10-24T19:28:09.216 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f -F src/expr.c fc204d08af06437ddaffe5a1b1f1f6f9e1a55d6d +F src/expr.c 0391a657df4959eaf2a2fd7d77de5ebe750686ee F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee @@ -225,14 +225,14 @@ F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 +F src/resolve.c 57d5ad93913beb43ad9b8ade435a54e5fb8ccd40 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 6e9e125698c1e5c78a51050ea61f179a281c766d +F src/sqliteInt.h 123b28f3552d4ffdd3e53707fe8120a069df69e4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c f5c13d9c1929bcc9d571f1e7bf7bfeb8b872ef99 +F src/where.c cc0733c59bd8bf6027d28b7d24b1887f4a870215 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -346,6 +346,7 @@ F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 F test/autoindex1.test 6ff78b94f43a59616c06c11c55b12935173506d7 F test/autoindex2.test 60d2fc6f38364308ce73a9beb01b47ded38697de F test/autoindex3.test 8254f689c3241081fad52b7bea18ba53e07e14a2 +F test/autoindex4.test fc807f9efd158bec60f5dfdf34ebe46fb274612d F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 @@ -1205,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 03d0498d0f24bec2383d5d79edf25069effecd59 -R eb013b99b9ff12a41d5ffc947e4a6227 +P 401235edf40fcd665eaf426cf5155ac6855e8537 +R def8f7d87d0f5784d9eaab2c56b2a690 U drh -Z 3215dd92b9608b7b9136fb91eb8178e8 +Z cb611c4b71e8fc233ea4e1005f56a9e0 diff --git a/manifest.uuid b/manifest.uuid index 4ee03d34ab..7417ef4fc1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -401235edf40fcd665eaf426cf5155ac6855e8537 \ No newline at end of file +d95d0313c447f5baeabdb17284d8606331ab7d49 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 1ad9a879a3..13a9cb46fd 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1210,20 +1210,24 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ } /* -** These routines are Walker callbacks. Walker.u.pi is a pointer -** to an integer. These routines are checking an expression to see -** if it is a constant. Set *Walker.u.i to 0 if the expression is -** not constant. +** These routines are Walker callbacks used to check expressions to +** see if they are "constant" for some definition of constant. The +** Walker.eCode value determines the type of "constant" we are looking +** for. ** ** These callback routines are used to implement the following: ** -** sqlite3ExprIsConstant() pWalker->u.i==1 -** sqlite3ExprIsConstantNotJoin() pWalker->u.i==2 -** sqlite3ExprIsConstantOrFunction() pWalker->u.i==3 or 4 +** sqlite3ExprIsConstant() pWalker->eCode==1 +** sqlite3ExprIsConstantNotJoin() pWalker->eCode==2 +** sqlite3ExprRefOneTableOnly() pWalker->eCode==3 +** sqlite3ExprIsConstantOrFunction() pWalker->eCode==4 or 5 +** +** In all cases, the callbacks set Walker.eCode=0 and abort if the expression +** is found to not be a constant. ** ** The sqlite3ExprIsConstantOrFunction() is used for evaluating expressions -** in a CREATE TABLE statement. The Walker.u.i value is 4 when parsing -** an existing schema and 3 when processing a new statement. A bound +** in a CREATE TABLE statement. The Walker.eCode value is 5 when parsing +** an existing schema and 4 when processing a new statement. A bound ** parameter raises an error for new statements, but is silently converted ** to NULL for existing schemas. This allows sqlite_master tables that ** contain a bound parameter because they were generated by older versions @@ -1232,23 +1236,25 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ */ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ - /* If pWalker->u.i is 2 then any term of the expression that comes from - ** the ON or USING clauses of a join disqualifies the expression + /* If pWalker->eCode is 2 then any term of the expression that comes from + ** the ON or USING clauses of a left join disqualifies the expression ** from being considered constant. */ - if( pWalker->u.i==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ - pWalker->u.i = 0; + if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ + pWalker->eCode = 0; return WRC_Abort; } switch( pExpr->op ){ /* Consider functions to be constant if all their arguments are constant - ** and either pWalker->u.i==3 or 4 or the function as the SQLITE_FUNC_CONST - ** flag. */ + ** and either pWalker->eCode==4 or 5 or the function has the + ** SQLITE_FUNC_CONST flag. */ case TK_FUNCTION: - if( pWalker->u.i>=3 || ExprHasProperty(pExpr,EP_Constant) ){ + if( pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_Constant) ){ return WRC_Continue; + }else{ + pWalker->eCode = 0; + return WRC_Abort; } - /* Fall through */ case TK_ID: case TK_COLUMN: case TK_AGG_FUNCTION: @@ -1257,18 +1263,22 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_COLUMN ); testcase( pExpr->op==TK_AGG_FUNCTION ); testcase( pExpr->op==TK_AGG_COLUMN ); - pWalker->u.i = 0; - return WRC_Abort; + if( pWalker->eCode==3 && pExpr->iTable==pWalker->u.iCur ){ + return WRC_Continue; + }else{ + pWalker->eCode = 0; + return WRC_Abort; + } case TK_VARIABLE: - if( pWalker->u.i==4 ){ + if( pWalker->eCode==5 ){ /* Silently convert bound parameters that appear inside of CREATE ** statements into a NULL when parsing the CREATE statement text out ** of the sqlite_master table */ pExpr->op = TK_NULL; - }else if( pWalker->u.i==3 ){ + }else if( pWalker->eCode==4 ){ /* A bound parameter in a CREATE statement that originates from ** sqlite3_prepare() causes an error */ - pWalker->u.i = 0; + pWalker->eCode = 0; return WRC_Abort; } /* Fall through */ @@ -1280,21 +1290,22 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ } static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ UNUSED_PARAMETER(NotUsed); - pWalker->u.i = 0; + pWalker->eCode = 0; return WRC_Abort; } -static int exprIsConst(Expr *p, int initFlag){ +static int exprIsConst(Expr *p, int initFlag, int iCur){ Walker w; memset(&w, 0, sizeof(w)); - w.u.i = initFlag; + w.eCode = initFlag; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = selectNodeIsConstant; + w.u.iCur = iCur; sqlite3WalkExpr(&w, p); - return w.u.i; + return w.eCode; } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression is constant ** and 0 if it involves variables or function calls. ** ** For the purposes of this function, a double-quoted string (ex: "abc") @@ -1302,21 +1313,31 @@ static int exprIsConst(Expr *p, int initFlag){ ** a constant. */ int sqlite3ExprIsConstant(Expr *p){ - return exprIsConst(p, 1); + return exprIsConst(p, 1, 0); } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression is constant ** that does no originate from the ON or USING clauses of a join. ** Return 0 if it involves variables or function calls or terms from ** an ON or USING clause. */ int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 2); + return exprIsConst(p, 2, 0); } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression constant +** for any single row of the table with cursor iCur. In other words, the +** expression must not refer to any non-deterministic function nor any +** table other than iCur. +*/ +int sqlite3ExprIsTableConstant(Expr *p, int iCur){ + return exprIsConst(p, 3, iCur); +} + +/* +** Walk an expression tree. Return non-zero if the expression is constant ** or a function call with constant arguments. Return and 0 if there ** are any variables. ** @@ -1326,7 +1347,7 @@ int sqlite3ExprIsConstantNotJoin(Expr *p){ */ int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ assert( isInit==0 || isInit==1 ); - return exprIsConst(p, 3+isInit); + return exprIsConst(p, 4+isInit, 0); } /* diff --git a/src/resolve.c b/src/resolve.c index d6a865caef..e507ccb810 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -28,7 +28,7 @@ ** is a helper function - a callback for the tree walker. */ static int incrAggDepth(Walker *pWalker, Expr *pExpr){ - if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.i; + if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.n; return WRC_Continue; } static void incrAggFunctionDepth(Expr *pExpr, int N){ @@ -36,7 +36,7 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ Walker w; memset(&w, 0, sizeof(w)); w.xExprCallback = incrAggDepth; - w.u.i = N; + w.u.n = N; sqlite3WalkExpr(&w, pExpr); } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 1b3138be44..b7f992c34d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2892,9 +2892,11 @@ struct Walker { void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ + u8 eCode; /* A small processing code */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ - int i; /* Integer value */ + int n; /* A counter */ + int iCur; /* A cursor number */ SrcList *pSrcList; /* FROM clause */ struct SrcCount *pSrcCount; /* Counting column references */ } u; @@ -3295,6 +3297,7 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3*); int sqlite3ExprIsConstant(Expr*); int sqlite3ExprIsConstantNotJoin(Expr*); int sqlite3ExprIsConstantOrFunction(Expr*, u8); +int sqlite3ExprIsTableConstant(Expr*,int); int sqlite3ExprIsInteger(Expr*, int*); int sqlite3ExprCanBeNull(const Expr*); int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); diff --git a/src/where.c b/src/where.c index e13b223366..8d2a891870 100644 --- a/src/where.c +++ b/src/where.c @@ -1594,6 +1594,8 @@ static void constructAutomaticIndex( Bitmask idxCols; /* Bitmap of columns used for indexing */ Bitmask extraCols; /* Bitmap of additional columns */ u8 sentWarning = 0; /* True if a warnning has been issued */ + Expr *pPartial = 0; /* Partial Index Expression */ + int iContinue = 0; /* Jump here to skip excluded rows */ /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ @@ -1609,6 +1611,11 @@ static void constructAutomaticIndex( pLoop = pLevel->pWLoop; idxCols = 0; for(pTerm=pWC->a; pTermprereq==0 + && sqlite3ExprIsTableConstant(pTerm->pExpr, pSrc->iCursor) ){ + pPartial = sqlite3ExprAnd(pParse->db, pPartial, + sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); + } if( termCanDriveIndex(pTerm, pSrc, notReady) ){ int iCol = pTerm->u.leftColumn; Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); @@ -1621,7 +1628,9 @@ static void constructAutomaticIndex( sentWarning = 1; } if( (idxCols & cMask)==0 ){ - if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ) return; + if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){ + goto end_auto_index_create; + } pLoop->aLTerm[nKeyCol++] = pTerm; idxCols |= cMask; } @@ -1654,7 +1663,7 @@ static void constructAutomaticIndex( /* Construct the Index object to describe this index */ pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); - if( pIdx==0 ) return; + if( pIdx==0 ) goto end_auto_index_create; pLoop->u.btree.pIndex = pIdx; pIdx->zName = "auto-index"; pIdx->pTable = pTable; @@ -1706,18 +1715,28 @@ static void constructAutomaticIndex( VdbeComment((v, "for %s", pTable->zName)); /* Fill the automatic index with content */ + sqlite3ExprCachePush(pParse); addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); + if( pPartial ){ + iContinue = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); + } regRecord = sqlite3GetTempReg(pParse); sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); sqlite3VdbeJumpHere(v, addrTop); sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ExprCachePop(pParse); /* Jump here when skipping the initialization */ sqlite3VdbeJumpHere(v, addrInit); + +end_auto_index_create: + sqlite3ExprDelete(pParse->db, pPartial); } #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ diff --git a/test/autoindex4.test b/test/autoindex4.test new file mode 100644 index 0000000000..6d0865bf72 --- /dev/null +++ b/test/autoindex4.test @@ -0,0 +1,52 @@ +# 2014-10-24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file implements regression tests for SQLite library. The +# focus of this script is testing automatic index creation logic, +# and specifically creation of automatic partial indexes. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test autoindex4-1.0 { + CREATE TABLE t1(a,b); + INSERT INTO t1 VALUES(123,'abc'),(234,'def'),(234,'ghi'),(345,'jkl'); + CREATE TABLE t2(x,y); + INSERT INTO t2 VALUES(987,'zyx'),(654,'wvu'),(987,'rqp'); + + SELECT *, '|' FROM t1, t2 WHERE a=234 AND x=987 ORDER BY +b; +} {234 def 987 rqp | 234 def 987 zyx | 234 ghi 987 rqp | 234 ghi 987 zyx |} +do_execsql_test autoindex4-1.1 { + SELECT *, '|' FROM t1, t2 WHERE a=234 AND x=555; +} {} + +do_execsql_test autoindex4-1.2 { + SELECT *, '|' FROM t1 LEFT JOIN t2 ON a=234 AND x=555; +} {123 abc {} {} | 234 def {} {} | 234 ghi {} {} | 345 jkl {} {} |} +do_execsql_test autoindex4-1.3 { + SELECT *, '|' FROM t1 LEFT JOIN t2 ON x=555 WHERE a=234; +} {234 def {} {} | 234 ghi {} {} |} +do_execsql_test autoindex4-1.4 { + SELECT *, '|' FROM t1 LEFT JOIN t2 WHERE a=234 AND x=555; +} {} + + +do_execsql_test autoindex4-2.0 { + CREATE TABLE t3(e,f); + INSERT INTO t3 VALUES(123,654),(555,444),(234,987); + + SELECT (SELECT count(*) FROM t1, t2 WHERE a=e AND x=f), e, f, '|' + FROM t3 + ORDER BY rowid; +} {1 123 654 | 0 555 444 | 4 234 987 |} + +finish_test From 051575cbf4c47fc472f53146e30d6ddddc9054b3 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 25 Oct 2014 12:28:25 +0000 Subject: [PATCH 7/8] Do not use virtual (and hence redundant) WHERE-clause terms to restrict the content of a automatic partial index. Show when an automatic partial index is used in the EXPLAIN QUERY PLAN output. FossilOrigin-Name: b9ad601eab1d7298d369267eb697c7fa1bc16985 --- ext/rtree/rtree6.test | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 4 ++++ src/whereInt.h | 1 + 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test index cec3a8da41..c9c87e8ad9 100644 --- a/ext/rtree/rtree6.test +++ b/ext/rtree/rtree6.test @@ -102,7 +102,7 @@ do_eqp_test rtree6.2.4.2 { SELECT * FROM t1,t2 WHERE v=10 and x1<10 and x2>10 } { 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1} - 0 1 1 {SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)} + 0 1 1 {SEARCH TABLE t2 USING AUTOMATIC PARTIAL COVERING INDEX (v=?)} } do_eqp_test rtree6.2.5 { diff --git a/manifest b/manifest index e831ecb526..28fa8e1387 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sautomatic\sindex\slogic\sso\sthat\sit\screates\sa\spartial\sindex\swhen\ndoing\sso\sgives\sthe\ssame\sanswer\sfor\sless\swork. -D 2014-10-24T19:28:09.216 +C Do\snot\suse\svirtual\s(and\shence\sredundant)\sWHERE-clause\sterms\sto\srestrict\sthe\ncontent\sof\sa\sautomatic\spartial\sindex.\s\sShow\swhen\san\sautomatic\spartial\sindex\nis\sused\sin\sthe\sEXPLAIN\sQUERY\sPLAN\soutput. +D 2014-10-25T12:28:25.871 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -130,7 +130,7 @@ F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba F ext/rtree/rtree3.test a494da55c30ee0bc9b01a91c80c81b387b22d2dc F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0 F ext/rtree/rtree5.test 6a510494f12454bf57ef28f45bc7764ea279431e -F ext/rtree/rtree6.test 0cfbdf27ee086bf16a3da2c6f2d5b3d6473cb27e +F ext/rtree/rtree6.test 773a90db2dce6a8353dd0d5b64bca69b29761196 F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971 F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a F ext/rtree/rtree9.test d86ebf08ff6328895613ed577dd8a2a37c472c34 @@ -302,8 +302,8 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c cc0733c59bd8bf6027d28b7d24b1887f4a870215 -F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 +F src/where.c 3312adfda33a9ca85c8a380a642a1a5905398b06 +F src/whereInt.h 19279cd0664ce1d90b9ad3ef0108cb494acfe455 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 @@ -1206,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 401235edf40fcd665eaf426cf5155ac6855e8537 -R def8f7d87d0f5784d9eaab2c56b2a690 +P d95d0313c447f5baeabdb17284d8606331ab7d49 +R 9499438a1fccf595ef80fbd4c31b20b8 U drh -Z cb611c4b71e8fc233ea4e1005f56a9e0 +Z cdff8327924b867b12e561826d5fe961 diff --git a/manifest.uuid b/manifest.uuid index 7417ef4fc1..8b94233ae7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d95d0313c447f5baeabdb17284d8606331ab7d49 \ No newline at end of file +b9ad601eab1d7298d369267eb697c7fa1bc16985 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 8d2a891870..2e42b8fc32 100644 --- a/src/where.c +++ b/src/where.c @@ -1612,6 +1612,7 @@ static void constructAutomaticIndex( idxCols = 0; for(pTerm=pWC->a; pTermprereq==0 + && (pTerm->wtFlags & TERM_VIRTUAL)==0 && sqlite3ExprIsTableConstant(pTerm->pExpr, pSrc->iCursor) ){ pPartial = sqlite3ExprAnd(pParse->db, pPartial, sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); @@ -1720,6 +1721,7 @@ static void constructAutomaticIndex( if( pPartial ){ iContinue = sqlite3VdbeMakeLabel(v); sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); + pLoop->wsFlags |= WHERE_PARTIALIDX; } regRecord = sqlite3GetTempReg(pParse); sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); @@ -2865,6 +2867,8 @@ static void explainOneScan( if( isSearch ){ zFmt = "PRIMARY KEY"; } + }else if( flags & WHERE_PARTIALIDX ){ + zFmt = "AUTOMATIC PARTIAL COVERING INDEX"; }else if( flags & WHERE_AUTO_INDEX ){ zFmt = "AUTOMATIC COVERING INDEX"; }else if( flags & WHERE_IDX_ONLY ){ diff --git a/src/whereInt.h b/src/whereInt.h index e9eb8b7dd2..fd4cfdf88a 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -459,3 +459,4 @@ struct WhereInfo { #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ +#define WHERE_PARTIALIDX 0x00020000 /* The automatic index is partial */ From d05ab6aacf08a1fd7af581cae4b28390ecd5c44c Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 25 Oct 2014 13:42:16 +0000 Subject: [PATCH 8/8] Increase the resolution of the second parameter to the likelihood() SQL function (the probability value) so that it can handle probabilities as small as 0.00000001. Formerly, it ran out of precision at 0.001. FossilOrigin-Name: 0f08924fe0c52a85a103f67bee9809e0f8f884b0 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/resolve.c | 4 ++-- src/sqliteInt.h | 2 +- src/where.c | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 28fa8e1387..6d8395af5e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\svirtual\s(and\shence\sredundant)\sWHERE-clause\sterms\sto\srestrict\sthe\ncontent\sof\sa\sautomatic\spartial\sindex.\s\sShow\swhen\san\sautomatic\spartial\sindex\nis\sused\sin\sthe\sEXPLAIN\sQUERY\sPLAN\soutput. -D 2014-10-25T12:28:25.871 +C Increase\sthe\sresolution\sof\sthe\ssecond\sparameter\sto\sthe\slikelihood()\sSQL\nfunction\s(the\sprobability\svalue)\sso\sthat\sit\scan\shandle\sprobabilities\nas\ssmall\sas\s0.00000001.\s\sFormerly,\sit\sran\sout\sof\sprecision\sat\s0.001. +D 2014-10-25T13:42:16.126 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -225,14 +225,14 @@ F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c 57d5ad93913beb43ad9b8ade435a54e5fb8ccd40 +F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 123b28f3552d4ffdd3e53707fe8120a069df69e4 +F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 3312adfda33a9ca85c8a380a642a1a5905398b06 +F src/where.c 4ce8c4826b7f86d080f0ed4e7a9045bb5014be77 F src/whereInt.h 19279cd0664ce1d90b9ad3ef0108cb494acfe455 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1206,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d95d0313c447f5baeabdb17284d8606331ab7d49 -R 9499438a1fccf595ef80fbd4c31b20b8 +P b9ad601eab1d7298d369267eb697c7fa1bc16985 +R 06fd31d9e4c300e561fccc4cf28a702b U drh -Z cdff8327924b867b12e561826d5fe961 +Z 628a0c5751868817bd40f1740c611aef diff --git a/manifest.uuid b/manifest.uuid index 8b94233ae7..3804da23b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9ad601eab1d7298d369267eb697c7fa1bc16985 \ No newline at end of file +0f08924fe0c52a85a103f67bee9809e0f8f884b0 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index e507ccb810..8fb580b3a1 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -584,7 +584,7 @@ static int exprProbability(Expr *p){ sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8); assert( r>=0.0 ); if( r>1.0 ) return -1; - return (int)(r*1000.0); + return (int)(r*134217728.0); } /* @@ -716,7 +716,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent to ** likelihood(X,0.9375). */ /* TUNING: unlikely() probability is 0.0625. likely() is 0.9375 */ - pExpr->iTable = pDef->zName[0]=='u' ? 62 : 938; + pExpr->iTable = pDef->zName[0]=='u' ? 8388608 : 125829120; } } #ifndef SQLITE_OMIT_AUTHORIZATION diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b7f992c34d..5114ccccf4 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2000,7 +2000,7 @@ struct Expr { int iTable; /* TK_COLUMN: cursor number of table holding column ** TK_REGISTER: register number ** TK_TRIGGER: 1 -> new, 0 -> old - ** EP_Unlikely: 1000 times likelihood */ + ** EP_Unlikely: 134217728 times likelihood */ ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. ** TK_VARIABLE: variable number (always >= 1). */ i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ diff --git a/src/where.c b/src/where.c index 2e42b8fc32..358d508260 100644 --- a/src/where.c +++ b/src/where.c @@ -225,7 +225,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ } pTerm = &pWC->a[idx = pWC->nTerm++]; if( p && ExprHasProperty(p, EP_Unlikely) ){ - pTerm->truthProb = sqlite3LogEst(p->iTable) - 99; + pTerm->truthProb = sqlite3LogEst(p->iTable) - 270; }else{ pTerm->truthProb = 1; }