From 6e711869042b45d84fd6c8fe8243db3849287137 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 26 Nov 2022 14:19:47 +0000 Subject: [PATCH 1/4] Experimental changes to help the query planner detect when an expression index is coverting. Works somewhat, but there are tests that fail. FossilOrigin-Name: 968c189bcf29a9b25305251a58d09b7d52ab9dd08f5057dc3ab1f7ad1a5316a0 --- manifest | 15 ++++++----- manifest.uuid | 2 +- src/where.c | 69 +++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 60 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index 3a6ab9396c..a56bd219e4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Relax\srestriction\s(8)\son\sthe\spush-down\soptimization\sso\sthat\sit\sonly\sapplies\nif\sone\sor\smore\scolumns\suses\sa\scollating\ssequence\sother\sthan\sBINARY.\nSee\s[forum:/forumpost/3824ced748baa808|forum\spost\s3824ced748baa808]\sand\ncheck-in\s[346a3b12b861ce7b]. -D 2022-11-25T17:05:55.692 +C Experimental\schanges\sto\shelp\sthe\squery\splanner\sdetect\swhen\san\sexpression\nindex\sis\scoverting.\s\sWorks\ssomewhat,\sbut\sthere\sare\stests\sthat\sfail. +D 2022-11-26T14:19:47.899 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -728,7 +728,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c ea0f518df9e00aa44013a1d384090b4b3a499ee11d4daa0a7d99c4eb9f7ab4ba +F src/where.c 550388b75a29c2af2c2182e56e2a51bc6b92aef302eb072f177b3526394d6020 F src/whereInt.h df0c79388c0b71b4a91f480d02791679fe0345d40410435c541c8893e95a4d3f F src/wherecode.c 133a94f82858787217d073143617df19e4a6a7d0b771a1519f957608109ad5a5 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae @@ -2060,8 +2060,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 09e1e42e0ff26f9a71cbd128169f060a66425828d637bf8f781490ca38d99103 -R 051f7bebeb695c93dfdb45030ad5b532 +P adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf +R b02bf662fd2aa7710f36a7d19197692a +T *branch * covering-indexed-expr +T *sym-covering-indexed-expr * +T -sym-trunk * U drh -Z 0a4a218169578e69e7b7469760b002aa +Z e78db49817bfc2647feb4d11db6b69e2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 76e9e654c1..036305084f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf \ No newline at end of file +968c189bcf29a9b25305251a58d09b7d52ab9dd08f5057dc3ab1f7ad1a5316a0 \ No newline at end of file diff --git a/src/where.c b/src/where.c index ae3d513cae..5f35c1e791 100644 --- a/src/where.c +++ b/src/where.c @@ -3247,6 +3247,26 @@ static int whereUsablePartialIndex( return 0; } +/* +** pIdx is an index containing expressions. Check it see if any of the +** expressions in the index match the pExpr expression. +*/ +static int exprIsCoveredByIndex( + const Expr *pExpr, + const Index *pIdx, + int iTabCur +){ + int i; + for(i=0; inColumn; i++){ + if( pIdx->aiColumn[i]==XN_EXPR + && sqlite3ExprCompare(0, pExpr, pIdx->aColExpr->a[i].pExpr, iTabCur)==0 + ){ + return 1; + } + } + return 0; +} + /* ** Structure passed to the whereIsCoveringIndex Walker callback. */ @@ -3256,7 +3276,7 @@ struct CoveringIndexCheck { }; /* -** Information passed in is pWalk->u.pCovIdxCk. Call is pCk. +** Information passed in is pWalk->u.pCovIdxCk. Call it pCk. ** ** If the Expr node references the table with cursor pCk->iTabCur, then ** make sure that column is covered by the index pCk->pIdx. We know that @@ -3268,23 +3288,32 @@ struct CoveringIndexCheck { ** ** If this node does not disprove that the index can be a covering index, ** then just return WRC_Continue, to continue the search. +** +** If pCk->pIdx contains indexed expressions and one of those expressions +** matches pExpr, then prune the search. */ static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ int i; /* Loop counter */ const Index *pIdx; /* The index of interest */ const i16 *aiColumn; /* Columns contained in the index */ u16 nColumn; /* Number of columns in the index */ - if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_AGG_COLUMN ) return WRC_Continue; - if( pExpr->iColumn<(BMS-1) ) return WRC_Continue; - if( pExpr->iTable!=pWalk->u.pCovIdxCk->iTabCur ) return WRC_Continue; pIdx = pWalk->u.pCovIdxCk->pIdx; - aiColumn = pIdx->aiColumn; - nColumn = pIdx->nColumn; - for(i=0; iiColumn ) return WRC_Continue; + if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){ + /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/ + if( pExpr->iTable!=pWalk->u.pCovIdxCk->iTabCur ) return WRC_Continue; + pIdx = pWalk->u.pCovIdxCk->pIdx; + aiColumn = pIdx->aiColumn; + nColumn = pIdx->nColumn; + for(i=0; iiColumn ) return WRC_Continue; + } + pWalk->eCode = 1; + return WRC_Abort; + }else if( pIdx->bHasExpr + && exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){ + return WRC_Prune; } - pWalk->eCode = 1; - return WRC_Abort; + return WRC_Continue; } @@ -3315,14 +3344,16 @@ static SQLITE_NOINLINE u32 whereIsCoveringIndex( ** if pIdx is covering. Assume it is not. */ return 1; } - for(i=0; inColumn; i++){ - if( pIdx->aiColumn[i]>=BMS-1 ) break; - } - if( i>=pIdx->nColumn ){ - /* pIdx does not index any columns greater than 62, but we know from - ** colMask that columns greater than 62 are used, so this is not a - ** covering index */ - return 1; + if( pIdx->bHasExpr==0 ){ + for(i=0; inColumn; i++){ + if( pIdx->aiColumn[i]>=BMS-1 ) break; + } + if( i>=pIdx->nColumn ){ + /* pIdx does not index any columns greater than 62, but we know from + ** colMask that columns greater than 62 are used, so this is not a + ** covering index */ + return 1; + } } ck.pIdx = pIdx; ck.iTabCur = iTabCur; @@ -3552,7 +3583,7 @@ static int whereLoopAddBtree( m = 0; }else{ m = pSrc->colUsed & pProbe->colNotIdxed; - if( m==TOPBIT ){ + if( m==TOPBIT || pProbe->bHasExpr ){ m = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); } pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; From 9d34152bcf72c20ed260d2c124dfde292cf4d5f0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 26 Nov 2022 20:52:38 +0000 Subject: [PATCH 2/4] Better reporting of when the WHERE clause analysis thinks that an index is covering. FossilOrigin-Name: 17ebcf316b91042c823eea7bb6f1309325023cb5c70538cdb2ce932caee2ef05 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/treeview.c | 5 ++++- src/where.c | 9 ++++++++- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 1e9ca78250..e31ab14280 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\sfixes\sinto\sthe\scovering-indexed-expr\sbranch. -D 2022-11-26T20:13:54.129 +C Better\sreporting\sof\swhen\sthe\sWHERE\sclause\sanalysis\sthinks\sthat\san\sindex\nis\scovering. +D 2022-11-26T20:52:38.117 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -706,7 +706,7 @@ F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b -F src/treeview.c 07787f67cd297a6d09d04b8d70c06769c60c9c1d9080378f93929c16f8fd3298 +F src/treeview.c 29b1dc7e0f84ba090734febe27393d4719682af0cae1b902d5ebf0236ecebea4 F src/trigger.c 4163ada044af89d51caba1cb713a73165347b2ec05fe84a283737c134d61fcd5 F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 @@ -728,7 +728,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 550388b75a29c2af2c2182e56e2a51bc6b92aef302eb072f177b3526394d6020 +F src/where.c 1f64631ab2c94b3c11de00ad1d08be9d51ee73acd613c54c4367526f5e2ac241 F src/whereInt.h df0c79388c0b71b4a91f480d02791679fe0345d40410435c541c8893e95a4d3f F src/wherecode.c 133a94f82858787217d073143617df19e4a6a7d0b771a1519f957608109ad5a5 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 968c189bcf29a9b25305251a58d09b7d52ab9dd08f5057dc3ab1f7ad1a5316a0 f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e -R b11982e41348628501c0b91820bd7cd7 +P 9ac67ff968e874cf955e46e3993e3215c766feec3d5bdd38d77884eedd86b59e +R 1dffa8ce2c066e6a4b41c4abd6aac427 U drh -Z 5e6dcb1170e4740f335f77683444a169 +Z 2a27a64568a31340c97915e2beb3adb7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 25771bfcb7..7f494549c9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ac67ff968e874cf955e46e3993e3215c766feec3d5bdd38d77884eedd86b59e \ No newline at end of file +17ebcf316b91042c823eea7bb6f1309325023cb5c70538cdb2ce932caee2ef05 \ No newline at end of file diff --git a/src/treeview.c b/src/treeview.c index 89a128dff6..9df16a12bf 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -487,7 +487,7 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ sqlite3TreeViewPop(&pView); return; } - if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags ){ + if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags || pExpr->pAggInfo ){ StrAccum x; sqlite3StrAccumInit(&x, 0, zFlgs, sizeof(zFlgs), 0); sqlite3_str_appendf(&x, " fg.af=%x.%c", @@ -504,6 +504,9 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ if( ExprHasVVAProperty(pExpr, EP_Immutable) ){ sqlite3_str_appendf(&x, " IMMUTABLE"); } + if( pExpr->pAggInfo!=0 ){ + sqlite3_str_appendf(&x, " agg-column[%d]", pExpr->iAgg); + } sqlite3StrAccumFinish(&x); }else{ zFlgs[0] = 0; diff --git a/src/where.c b/src/where.c index 5f35c1e791..6ab9230d83 100644 --- a/src/where.c +++ b/src/where.c @@ -3583,8 +3583,15 @@ static int whereLoopAddBtree( m = 0; }else{ m = pSrc->colUsed & pProbe->colNotIdxed; - if( m==TOPBIT || pProbe->bHasExpr ){ + if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol) ){ m = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); + WHERETRACE(0xff, + ("-> %s %s a covering index according to whereIsCoveringIndex()\n", + pProbe->zName, m==0 ? "is" : "is not")); + }else{ + WHERETRACE(0xff, + ("-> %s %s a covering index according to bitmasks\n", + pProbe->zName, m==0 ? "is" : "is not")); } pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; } From 84606c973757cbb5a276b54c5bfad67943288722 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 28 Nov 2022 15:23:53 +0000 Subject: [PATCH 3/4] Rework the covering index checking routine, whereIsCoveringIndex(), so that it can return a "maybe" result for aggregate queries where we are not exactly sure. The index is scored as if it is covering, but the main table is still opened. FossilOrigin-Name: b8eec4214363192e6f3e12b3faa5810d8269a5fdaecab3ec09b02e5002cf798a --- manifest | 14 +++---- manifest.uuid | 2 +- src/where.c | 103 ++++++++++++++++++++++++++++++++++--------------- src/whereInt.h | 1 + 4 files changed, 81 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index e31ab14280..f0f6a8af9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Better\sreporting\sof\swhen\sthe\sWHERE\sclause\sanalysis\sthinks\sthat\san\sindex\nis\scovering. -D 2022-11-26T20:52:38.117 +C Rework\sthe\scovering\sindex\schecking\sroutine,\swhereIsCoveringIndex(),\sso\sthat\nit\scan\sreturn\sa\s"maybe"\sresult\sfor\saggregate\squeries\swhere\swe\sare\snot\sexactly\nsure.\s\sThe\sindex\sis\sscored\sas\sif\sit\sis\scovering,\sbut\sthe\smain\stable\sis\nstill\sopened. +D 2022-11-28T15:23:53.775 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -728,8 +728,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 1f64631ab2c94b3c11de00ad1d08be9d51ee73acd613c54c4367526f5e2ac241 -F src/whereInt.h df0c79388c0b71b4a91f480d02791679fe0345d40410435c541c8893e95a4d3f +F src/where.c 81422870d17fdf3880fdb9d3b3965e1aa5efb7a72c6933cec3f1bc8081c463db +F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 133a94f82858787217d073143617df19e4a6a7d0b771a1519f957608109ad5a5 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae F src/window.c 14836767adb26573b50f528eb37f8b1336f2c430ab38de7cead1e5c546bb4d8c @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9ac67ff968e874cf955e46e3993e3215c766feec3d5bdd38d77884eedd86b59e -R 1dffa8ce2c066e6a4b41c4abd6aac427 +P 17ebcf316b91042c823eea7bb6f1309325023cb5c70538cdb2ce932caee2ef05 +R 254ab4db9b1e239ec16eb2e1a31dc657 U drh -Z 2a27a64568a31340c97915e2beb3adb7 +Z 489a805b1f3c740affbc27a0febf881c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7f494549c9..d0a052a41d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -17ebcf316b91042c823eea7bb6f1309325023cb5c70538cdb2ce932caee2ef05 \ No newline at end of file +b8eec4214363192e6f3e12b3faa5810d8269a5fdaecab3ec09b02e5002cf798a \ No newline at end of file diff --git a/src/where.c b/src/where.c index 6ab9230d83..589300c6ba 100644 --- a/src/where.c +++ b/src/where.c @@ -3094,7 +3094,7 @@ static int whereLoopAddBtreeIndex( assert( pSrc->pTab->szTabRow>0 ); rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); - if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ + if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); @@ -3270,9 +3270,12 @@ static int exprIsCoveredByIndex( /* ** Structure passed to the whereIsCoveringIndex Walker callback. */ +typedef struct CoveringIndexCheck CoveringIndexCheck; struct CoveringIndexCheck { Index *pIdx; /* The index */ int iTabCur; /* Cursor number for the corresponding table */ + u8 bExpr; /* Uses an indexed expression */ + u8 bUnidx; /* Uses an unindexed column not within an indexed expr */ }; /* @@ -3293,24 +3296,28 @@ struct CoveringIndexCheck { ** matches pExpr, then prune the search. */ static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ - int i; /* Loop counter */ - const Index *pIdx; /* The index of interest */ - const i16 *aiColumn; /* Columns contained in the index */ - u16 nColumn; /* Number of columns in the index */ - pIdx = pWalk->u.pCovIdxCk->pIdx; + int i; /* Loop counter */ + const Index *pIdx; /* The index of interest */ + const i16 *aiColumn; /* Columns contained in the index */ + u16 nColumn; /* Number of columns in the index */ + CoveringIndexCheck *pCk; /* Info about this search */ + + pCk = pWalk->u.pCovIdxCk; + pIdx = pCk->pIdx; if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){ /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/ - if( pExpr->iTable!=pWalk->u.pCovIdxCk->iTabCur ) return WRC_Continue; + if( pExpr->iTable!=pCk->iTabCur ) return WRC_Continue; pIdx = pWalk->u.pCovIdxCk->pIdx; aiColumn = pIdx->aiColumn; nColumn = pIdx->nColumn; for(i=0; iiColumn ) return WRC_Continue; } - pWalk->eCode = 1; + pCk->bUnidx = 1; return WRC_Abort; }else if( pIdx->bHasExpr && exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){ + pCk->bExpr = 1; return WRC_Prune; } return WRC_Continue; @@ -3319,30 +3326,39 @@ static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ /* ** pIdx is an index that covers all of the low-number columns used by -** pWInfo->pSelect (columns from 0 through 62). But there are columns -** in pWInfo->pSelect beyond 62. This routine tries to answer the question -** of whether pIdx covers *all* columns in the query. +** pWInfo->pSelect (columns from 0 through 62) or an index that has +** expressions terms. Hence, we cannot determine whether or not it is +** a covering index by using the colUsed bitmasks. We have to do a search +** to see if the index is covering. This routine does that search. ** -** Return 0 if pIdx is a covering index. Return non-zero if pIdx is -** not a covering index or if we are unable to determine if pIdx is a -** covering index. +** The return value is one of these: ** -** This routine is an optimization. It is always safe to return non-zero. -** But returning zero when non-zero should have been returned can lead to -** incorrect bytecode and assertion faults. +** 0 The index is definitely not a covering index +** +** WHERE_IDX_ONLY The index is definitely a covering index +** +** WHERE_EXPRIDX The index is likely a covering index, but it is +** difficult to determine precisely because of the +** expressions that are indexed. Score it as a +** covering index, but still keep the main table open +** just in case we need it. +** +** This routine is an optimization. It is always safe to return zero. +** But returning one of the other two values when zero should have been +** returned can lead to incorrect bytecode and assertion faults. */ static SQLITE_NOINLINE u32 whereIsCoveringIndex( WhereInfo *pWInfo, /* The WHERE clause context */ Index *pIdx, /* Index that is being tested */ int iTabCur /* Cursor for the table being indexed */ ){ - int i; + int i, rc; struct CoveringIndexCheck ck; Walker w; if( pWInfo->pSelect==0 ){ /* We don't have access to the full query, so we cannot check to see ** if pIdx is covering. Assume it is not. */ - return 1; + return 0; } if( pIdx->bHasExpr==0 ){ for(i=0; inColumn; i++){ @@ -3352,18 +3368,26 @@ static SQLITE_NOINLINE u32 whereIsCoveringIndex( /* pIdx does not index any columns greater than 62, but we know from ** colMask that columns greater than 62 are used, so this is not a ** covering index */ - return 1; + return 0; } } ck.pIdx = pIdx; ck.iTabCur = iTabCur; + ck.bExpr = 0; + ck.bUnidx = 0; memset(&w, 0, sizeof(w)); w.xExprCallback = whereIsCoveringIndexWalkCallback; w.xSelectCallback = sqlite3SelectWalkNoop; w.u.pCovIdxCk = &ck; - w.eCode = 0; sqlite3WalkSelect(&w, pWInfo->pSelect); - return w.eCode; + if( ck.bUnidx ){ + rc = 0; + }else if( ck.bExpr ){ + rc = WHERE_EXPRIDX; + }else{ + rc = WHERE_IDX_ONLY; + } + return rc; } /* @@ -3579,21 +3603,38 @@ static int whereLoopAddBtree( }else{ Bitmask m; if( pProbe->isCovering ){ - pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; m = 0; + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; }else{ m = pSrc->colUsed & pProbe->colNotIdxed; - if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol) ){ - m = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); + pNew->wsFlags = WHERE_INDEXED; + if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){ + u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); + if( isCov==0 ){ + WHERETRACE(0xff, + ("-> %s is not a covering index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + assert( m!=0 ); + }else{ + m = 0; + pNew->wsFlags |= isCov; + if( isCov & WHERE_IDX_ONLY ){ + WHERETRACE(0xff, + ("-> %s is a covering expression index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + }else{ + assert( isCov==WHERE_EXPRIDX ); + WHERETRACE(0xff, + ("-> %s might be a covering expression index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + } + } + }else if( m==0 ){ WHERETRACE(0xff, - ("-> %s %s a covering index according to whereIsCoveringIndex()\n", - pProbe->zName, m==0 ? "is" : "is not")); - }else{ - WHERETRACE(0xff, - ("-> %s %s a covering index according to bitmasks\n", + ("-> %s a covering index according to bitmasks\n", pProbe->zName, m==0 ? "is" : "is not")); + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; } - pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; } /* Full scan via index */ diff --git a/src/whereInt.h b/src/whereInt.h index 28ede5be66..b89a4513e3 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -634,5 +634,6 @@ void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*); #define WHERE_SELFCULL 0x00800000 /* nOut reduced by extra WHERE terms */ #define WHERE_OMIT_OFFSET 0x01000000 /* Set offset counter to zero */ #define WHERE_VIEWSCAN 0x02000000 /* A full-scan of a VIEW or subquery */ +#define WHERE_EXPRIDX 0x04000000 /* Uses an index-on-expressions */ #endif /* !defined(SQLITE_WHEREINT_H) */ From cbb958580575c4f4c57dc2019993d44653e04f41 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 28 Nov 2022 18:17:42 +0000 Subject: [PATCH 4/4] Test cases added derived from the enhancment ticket. FossilOrigin-Name: c022c0152ad61a4f56e36f4062609073e2273fbf6f826c20652159be229c2d46 --- manifest | 12 +++---- manifest.uuid | 2 +- test/indexexpr1.test | 83 ++++++++++++++++++++++++++++++++++++++------ 3 files changed, 79 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index c83d4173b2..c76ad7a312 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\scovering-index-expr\sbranch. -D 2022-11-28T16:11:10.036 +C Test\scases\sadded\sderived\sfrom\sthe\senhancment\sticket. +D 2022-11-28T18:17:42.151 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1213,7 +1213,7 @@ F test/index7.test b238344318e0b4e42126717f6554f0e7dfd0b39cecad4b736039b43e1e3b6 F test/index8.test caa097735c91dbc23d8a402f5e63a2a03c83840ba3928733ed7f9a03f8a912a3 F test/index9.test 2ac891806a4136ef3e91280477e23114e67575207dc331e6797fa0ed9379f997 F test/indexedby.test f21eca4f7a6ffe14c8500a7ad6cd53166666c99e5ccd311842a28bc94a195fe0 -F test/indexexpr1.test 3360c2a29a8844e7c4b13293567025281257f9e13a31854cfff6959cede11502 +F test/indexexpr1.test 1cd460113e3999a1477754fd0e3ac0d7bcbbbc742b427b418945f28af708c695 F test/indexexpr2.test 2c7abe3c48f8aaa5a448615ab4d13df3662185d28419c00999670834a3f0b484 F test/indexfault.test 98d78a8ff1f5335628b62f886a1cb7c7dac1ef6d48fa39c51ec871c87dce9811 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b8eec4214363192e6f3e12b3faa5810d8269a5fdaecab3ec09b02e5002cf798a b44ab10c49bc2895483a9d40813be3798710ee713cc4bf04e449dce55a68452a -R 5944640ae5b4a3237e1d8a4c46c48db4 +P 89d775ada39aed4dc532374ace02156d07dc06e8ae54a194608af0c5a582d20f +R e73156e3b8e8d088eb2bbd902f358640 U drh -Z 9aa95e661b7edb77eed478b829e7796d +Z b96271db8dd85961a10ccefd57c768e1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e7c719444e..a270d0f31b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -89d775ada39aed4dc532374ace02156d07dc06e8ae54a194608af0c5a582d20f \ No newline at end of file +c022c0152ad61a4f56e36f4062609073e2273fbf6f826c20652159be229c2d46 \ No newline at end of file diff --git a/test/indexexpr1.test b/test/indexexpr1.test index 042132b81d..a5d1514bcd 100644 --- a/test/indexexpr1.test +++ b/test/indexexpr1.test @@ -420,7 +420,7 @@ do_execsql_test indexexpr1-1500 { # 2018-01-03 OSSFuzz discovers another test case for the same problem # above. # -do_execsql_test indexexpr-1510 { +do_execsql_test indexexpr1-1510 { DROP TABLE IF EXISTS t1; CREATE TABLE t1(a PRIMARY KEY,b UNIQUE); REPLACE INTO t1 VALUES(2, 1); @@ -434,18 +434,18 @@ do_execsql_test indexexpr-1510 { # a numeric table column, trouble can arise since there are multiple # string that can map to the same numeric value. (Ex: 123, 0123, 000123). # -do_execsql_test indexexpr-1600 { +do_execsql_test indexexpr1-1600 { DROP TABLE IF EXISTS t1; CREATE TABLE t1 (a INTEGER, b); CREATE INDEX idx1 ON t1 (lower(a)); INSERT INTO t1 VALUES('0001234',3); PRAGMA integrity_check; } {ok} -do_execsql_test indexexpr-1610 { +do_execsql_test indexexpr1-1610 { INSERT INTO t1 VALUES('1234',0),('001234',2),('01234',1); SELECT b FROM t1 WHERE lower(a)='1234' ORDER BY +b; } {0 1 2 3} -do_execsql_test indexexpr-1620 { +do_execsql_test indexexpr1-1620 { SELECT b FROM t1 WHERE lower(a)='01234' ORDER BY +b; } {} @@ -453,7 +453,7 @@ do_execsql_test indexexpr-1620 { # ExprImpliesExpr theorem prover bug: # "(NULL IS FALSE) IS FALSE" does not imply "NULL IS NULL" # -do_execsql_test indexexpr-1700 { +do_execsql_test indexexpr1-1700 { DROP TABLE IF EXISTS t0; CREATE TABLE t0(c0); INSERT INTO t0(c0) VALUES (0); @@ -468,17 +468,17 @@ do_execsql_test indexexpr-1700 { # computing the expression. # ifcapable like_match_blobs { - do_execsql_test indexexpr-1800 { + do_execsql_test indexexpr1-1800 { DROP TABLE IF EXISTS t0; CREATE TABLE t0(c0 REAL, c1 TEXT); CREATE INDEX i0 ON t0(+c0, c0); INSERT INTO t0(c0) VALUES(0); SELECT CAST(+ t0.c0 AS BLOB) LIKE 0 FROM t0; } {0} - do_execsql_test indexexpr-1810 { + do_execsql_test indexexpr1-1810 { SELECT CAST(+ t0.c0 AS BLOB) LIKE '0.0' FROM t0; } {1} - do_execsql_test indexexpr-1820 { + do_execsql_test indexexpr1-1820 { DROP TABLE IF EXISTS t1; CREATE TABLE t1(x REAL); CREATE INDEX t1x ON t1(x, +x); @@ -491,19 +491,80 @@ ifcapable like_match_blobs { # Assertion fault during a DELETE INDEXED BY. # reset_db -do_execsql_test indexexpr-1900 { +do_execsql_test indexexpr1-1900 { CREATE TABLE t1(x TEXT PRIMARY KEY, y TEXT, z INT); INSERT INTO t1(x,y,z) VALUES('alpha','ALPHA',1),('bravo','charlie',1); CREATE INDEX i1 ON t1(+y COLLATE NOCASE); SELECT * FROM t1; } {alpha ALPHA 1 bravo charlie 1} -do_execsql_test indexexpr-1910 { +do_execsql_test indexexpr1-1910 { DELETE FROM t1 INDEXED BY i1 WHERE x IS +y COLLATE NOCASE IN (SELECT z FROM t1) RETURNING *; } {alpha ALPHA 1} -do_execsql_test indexexpr-1920 { +do_execsql_test indexexpr1-1920 { SELECT * FROM t1; } {bravo charlie 1} +# 2022-11-28 Ticket 695a1a53de +# Improved ability to recognize that an index on an expression is a +# covering index. +# +reset_db +do_execsql_test indexexpr1-2000 { + CREATE TABLE t1(a INT, b TEXT); + INSERT INTO t1(a,b) VALUES + (10, '{"one":5,"two":6}'), + (10, '{"one":50,"two":60}'), + (10, '{"three":99}'), + (11, '{"one":100,"two":200}'); + CREATE INDEX t1_one ON t1(a, b->>'one'); + CREATE INDEX t1_two ON t1(a, b->>'two'); +} +do_execsql_test indexexpr1-2010 { + EXPLAIN QUERY PLAN + SELECT sum(b->>'one') FROM t1 WHERE a=10; /* Query AA */ +} {/.* t1_one .*/} +do_execsql_test indexexpr1-2011 { + SELECT sum(b->>'one') FROM t1 WHERE a=10; /* Query AA */ +} {55} +do_execsql_test indexexpr1-2020 { + EXPLAIN QUERY PLAN + SELECT sum(b->>'two') FROM t1 WHERE a=10; /* Query BB */ +} {/.* t1_two .*/} +do_execsql_test indexexpr1-2021 { + SELECT sum(b->>'two') FROM t1 WHERE a=10; /* Query BB */ +} {66} +do_execsql_test indexexpr1-2030 { + DROP TABLE t1; + CREATE TABLE t1(a INT, b TEXT, c INT, d INT); + INSERT INTO t1(a,b,c,d) VALUES + (1, '{"x":1}', 12, 3), + (1, '{"x":2}', 4, 5), + (1, '{"x":1}', 6, 11), + (2, '{"x":1}', 22, 3), + (2, '{"x":2}', 4, 5), + (3, '{"x":1}', 6, 7); + CREATE INDEX t1x ON t1(d, a, b->>'x', c); +} +do_execsql_test indexexpr1-2030 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1; +} {1 6 4 54 46} +do_execsql_test indexexpr1-2030 { + explain query plan + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1; +} {/.*SCAN t1 USING INDEX t1x.*/} + + + finish_test