From b9366f8e3250eae8417a46396cefd082c2ab7a59 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 13 Jan 2021 11:44:51 +0000 Subject: [PATCH 01/30] In the .selecttrace output, show the ORDER BY clause added by the Min/Max optimization. FossilOrigin-Name: db0ecfe66433f8915b6eb16d3735a4a0d0f8e0bbc395bc9c1364387506fc4657 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/select.c | 4 ++++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 018db97054..24a5971adf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sParseCleanup\senhancement\sto\strunk. -D 2021-01-12T16:26:36.110 +C In\sthe\s.selecttrace\soutput,\sshow\sthe\sORDER\sBY\sclause\sadded\sby\sthe\sMin/Max\noptimization. +D 2021-01-13T11:44:51.022 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -540,7 +540,7 @@ F src/printf.c 30e92b638fac71dcd85cdea1d12ecfae354c9adee2c71e8e1ae4727cde7c91ed F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 1948a92ca9eab776632816b97e57c61d933474a78aad4f4ef835c916a83dbb1c F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c a9c38abfbaaf1230fa9079b4d1d43694cac335a85efa39684bd4969a5c877a19 +F src/select.c 74655feb1ba3767f0f376029acdf1e488942e6b02aa0e65679a75181c2fbe88f F src/shell.c.in 79bceb990e4bac23a09bb8dd65783ea4867b8bfca9242b5a82b884043e65109a F src/sqlite.h.in 0af968a1fa3c717261e1df0ed105fa7bddb4d82de7e0adb3eab49e6aa81b4de7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1895,8 +1895,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8f3ab5da4c8906b63e2c1a0021a3ba4f60e7199e8640518060f998876a002663 38ef8ab9830e12acd2c710e113939b1f8dced02612c6933c37a3c948a4030d0a -R a2b97b3e175b432108521b3e9366ef42 -T +closed 38ef8ab9830e12acd2c710e113939b1f8dced02612c6933c37a3c948a4030d0a +P 35824c1bcbd89ae4a94acfbe511bfbd888c418b981819e72bc9a991fc82d136c +R 61cf4ac944232eaafcf9775988abf565 U drh -Z 9c0d514193cce7631d3a6adc01edb18c +Z 998402e9038a172631ad95c2c70da3a5 diff --git a/manifest.uuid b/manifest.uuid index 730c7d084c..5458f1df8c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35824c1bcbd89ae4a94acfbe511bfbd888c418b981819e72bc9a991fc82d136c \ No newline at end of file +db0ecfe66433f8915b6eb16d3735a4a0d0f8e0bbc395bc9c1364387506fc4657 \ No newline at end of file diff --git a/src/select.c b/src/select.c index c4d17810e3..f00a3acd8d 100644 --- a/src/select.c +++ b/src/select.c @@ -6580,6 +6580,10 @@ int sqlite3Select( int ii; SELECTTRACE(0x400,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); sqlite3TreeViewSelect(0, p, 0); + if( minMaxFlag ){ + sqlite3DebugPrintf("MIN/MAX Optimization (0x%02x) adds:\n", minMaxFlag); + sqlite3TreeViewExprList(0, pMinMaxOrderBy, 0, "ORDERBY"); + } for(ii=0; iinColumn; ii++){ sqlite3DebugPrintf("agg-column[%d] iMem=%d\n", ii, pAggInfo->aCol[ii].iMem); From aa556b0fa7ae3f96df773f20d86c51186be25f52 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 13 Jan 2021 12:59:20 +0000 Subject: [PATCH 02/30] cli: Omit surplus whitespace at the end of lines in .explain output. FossilOrigin-Name: 11e4eb095746602961a178044809a68a77ba7b367596997bef726e54062423d9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 24a5971adf..b050b37ce0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\s.selecttrace\soutput,\sshow\sthe\sORDER\sBY\sclause\sadded\sby\sthe\sMin/Max\noptimization. -D 2021-01-13T11:44:51.022 +C cli:\sOmit\ssurplus\swhitespace\sat\sthe\send\sof\slines\sin\s.explain\soutput. +D 2021-01-13T12:59:20.834 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -541,7 +541,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 1948a92ca9eab776632816b97e57c61d933474a78aad4f4ef835c916a83dbb1c F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 74655feb1ba3767f0f376029acdf1e488942e6b02aa0e65679a75181c2fbe88f -F src/shell.c.in 79bceb990e4bac23a09bb8dd65783ea4867b8bfca9242b5a82b884043e65109a +F src/shell.c.in 8331e564cd73e1784dd034d88ce626d2867531e9cc38ece980b256b09eb548b3 F src/sqlite.h.in 0af968a1fa3c717261e1df0ed105fa7bddb4d82de7e0adb3eab49e6aa81b4de7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 61b38c073d5e1e96a3d45271b257aef27d0d13da2bea5347692ae579475cd95e @@ -1895,7 +1895,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 35824c1bcbd89ae4a94acfbe511bfbd888c418b981819e72bc9a991fc82d136c -R 61cf4ac944232eaafcf9775988abf565 +P db0ecfe66433f8915b6eb16d3735a4a0d0f8e0bbc395bc9c1364387506fc4657 +R 2ad612629a79dbad03e8b7f173945d59 U drh -Z 998402e9038a172631ad95c2c70da3a5 +Z f407a95e341e109313c0922f7d2a229b diff --git a/manifest.uuid b/manifest.uuid index 5458f1df8c..ccaf3daab6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db0ecfe66433f8915b6eb16d3735a4a0d0f8e0bbc395bc9c1364387506fc4657 \ No newline at end of file +11e4eb095746602961a178044809a68a77ba7b367596997bef726e54062423d9 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 806ca2102b..734a54546e 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -2027,6 +2027,7 @@ static int shell_callback( if( azArg==0 ) break; for(i=0; iw ){ w = strlenChar(azArg[i]); } From d193057ad147fe038d0f85c68cd914fa9d91c40c Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 13 Jan 2021 15:23:17 +0000 Subject: [PATCH 03/30] Further enhancements to the min/max optimization of [/info/b8ba2f17f938c035|check-in b8ba2f17f938c035] to fix the performance regression identified by [forum:/forumpost/623f571482|forum post 623f571482]. FossilOrigin-Name: 188772a1dbaf066fbddd39c718fdd87478b19a920622f4640bcb79d4ef065331 --- manifest | 19 +++++++++++-------- manifest.uuid | 2 +- src/select.c | 8 +++----- src/sqliteInt.h | 1 + src/where.c | 23 +++++++++++++++++++++++ 5 files changed, 39 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index b050b37ce0..d0beeb67d4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C cli:\sOmit\ssurplus\swhitespace\sat\sthe\send\sof\slines\sin\s.explain\soutput. -D 2021-01-13T12:59:20.834 +C Further\senhancements\sto\sthe\smin/max\soptimization\sof\n[/info/b8ba2f17f938c035|check-in\sb8ba2f17f938c035]\sto\sfix\sthe\sperformance\nregression\sidentified\sby\n[forum:/forumpost/623f571482|forum\spost\s623f571482]. +D 2021-01-13T15:23:17.568 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -540,12 +540,12 @@ F src/printf.c 30e92b638fac71dcd85cdea1d12ecfae354c9adee2c71e8e1ae4727cde7c91ed F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 1948a92ca9eab776632816b97e57c61d933474a78aad4f4ef835c916a83dbb1c F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 74655feb1ba3767f0f376029acdf1e488942e6b02aa0e65679a75181c2fbe88f +F src/select.c 083c9bcc6b2ec80f172d414f8129679dddfb25e1ad57364cf3677bb504d62158 F src/shell.c.in 8331e564cd73e1784dd034d88ce626d2867531e9cc38ece980b256b09eb548b3 F src/sqlite.h.in 0af968a1fa3c717261e1df0ed105fa7bddb4d82de7e0adb3eab49e6aa81b4de7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 61b38c073d5e1e96a3d45271b257aef27d0d13da2bea5347692ae579475cd95e -F src/sqliteInt.h c7c7e0e79769885a1c7fa519299bdb222e83e2d7d4843ed6ca1cd9a585016386 +F src/sqliteInt.h cecaa4eea2af66d36e4b1453d08da83b4d9ad2de2510d58d518ebd51fbd2692b F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -627,7 +627,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 69e770e96fd56cc21608992bf2c6f1f3dc5cf2572d0495c6a643b06c3a679f14 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c -F src/where.c 3d31871d03906312d7d71a9c0b28c97bcbaead7606dfc15f9b3d080b18702385 +F src/where.c dee929890fe399993d4e78a5a89d4b289f7ab015446e157d2e4c25477fdfa060 F src/whereInt.h 9a3f577619f07700d16d89eeb2f3d94d6b7ed7f109c2dacf0ce8844921549506 F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee F src/whereexpr.c 3a463e156ea388083c501502229c2c7f4f5c6b5330ea59bdf40d6eb6e155a25f @@ -1895,7 +1895,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P db0ecfe66433f8915b6eb16d3735a4a0d0f8e0bbc395bc9c1364387506fc4657 -R 2ad612629a79dbad03e8b7f173945d59 +P 11e4eb095746602961a178044809a68a77ba7b367596997bef726e54062423d9 +R 14dcd4c79aadaa323e7c1244a63358d2 +T *branch * minmax-opt-exp +T *sym-minmax-opt-exp * +T -sym-trunk * U drh -Z f407a95e341e109313c0922f7d2a229b +Z 525d6c9785240d33b80adadc935f19d6 diff --git a/manifest.uuid b/manifest.uuid index ccaf3daab6..a15faf20cd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -11e4eb095746602961a178044809a68a77ba7b367596997bef726e54062423d9 \ No newline at end of file +188772a1dbaf066fbddd39c718fdd87478b19a920622f4640bcb79d4ef065331 \ No newline at end of file diff --git a/src/select.c b/src/select.c index f00a3acd8d..76806b66ea 100644 --- a/src/select.c +++ b/src/select.c @@ -6773,7 +6773,7 @@ int sqlite3Select( /* End of the loop */ if( groupBySort ){ - sqlite3VdbeAddOp2(v, OP_SorterNext, pAggInfo->sortingIdx, addrTopOfLoop); + sqlite3VdbeAddOp2(v, OP_SorterNext, pAggInfo->sortingIdx,addrTopOfLoop); VdbeCoverage(v); }else{ sqlite3WhereEnd(pWInfo); @@ -6885,7 +6885,6 @@ int sqlite3Select( explainSimpleCount(pParse, pTab, pBest); }else{ int regAcc = 0; /* "populate accumulators" flag */ - int addrSkip; /* If there are accumulator registers but no min() or max() functions ** without FILTER clauses, allocate register regAcc. Register regAcc @@ -6934,9 +6933,8 @@ int sqlite3Select( } updateAccumulator(pParse, regAcc, pAggInfo); if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc); - addrSkip = sqlite3WhereOrderByLimitOptLabel(pWInfo); - if( addrSkip!=sqlite3WhereContinueLabel(pWInfo) ){ - sqlite3VdbeGoto(v, addrSkip); + if( minMaxFlag ){ + sqlite3WhereMinMaxOptEarlyOut(v, pWInfo); } sqlite3WhereEnd(pWInfo); finalizeAggFunctions(pParse, pAggInfo); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 6d1e598d94..8432fe20cd 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4341,6 +4341,7 @@ LogEst sqlite3WhereOutputRowCount(WhereInfo*); int sqlite3WhereIsDistinct(WhereInfo*); int sqlite3WhereIsOrdered(WhereInfo*); int sqlite3WhereOrderByLimitOptLabel(WhereInfo*); +void sqlite3WhereMinMaxOptEarlyOut(Vdbe*,WhereInfo*); int sqlite3WhereIsSorted(WhereInfo*); int sqlite3WhereContinueLabel(WhereInfo*); int sqlite3WhereBreakLabel(WhereInfo*); diff --git a/src/where.c b/src/where.c index 0f219d4391..adc688b4ba 100644 --- a/src/where.c +++ b/src/where.c @@ -99,6 +99,29 @@ int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){ return pInner->addrNxt; } +/* +** While generating code for the min/max optimization, after handling +** the aggregate-step call to min() or max(), check to see if any +** additional looping is required. If the output order is such that +** we are certain that the correct answer has already been found, then +** code an OP_Goto to by pass subsequent processing. +** +** Any extra OP_Goto that is coded here is an optimization. The +** correct answer should be obtained regardless. This OP_Goto just +** makes the answer appear faster. +*/ +void sqlite3WhereMinMaxOptEarlyOut(Vdbe *v, WhereInfo *pWInfo){ + WhereLevel *pInner; + if( !pWInfo->bOrderedInnerLoop ) return; + if( pWInfo->nOBSat==0 ) return; + pInner = &pWInfo->a[pWInfo->nLevel-1]; + if( (pInner->pWLoop->wsFlags & WHERE_COLUMN_IN)==0 ){ + sqlite3VdbeGoto(v, pWInfo->iBreak); + }else if( pInner->addrNxt!=pWInfo->iContinue ){ + sqlite3VdbeGoto(v, pInner->addrNxt); + } +} + /* ** Return the VDBE address or label to jump to in order to continue ** immediately with the next row of a WHERE clause. From af7b76534b3680d712fadf5d5a038c75d4b99a90 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 13 Jan 2021 19:28:17 +0000 Subject: [PATCH 04/30] Expand the number of optimization-disable bits from 16 to 32. Use one of the new bits to disable the min/max optimization, so that we can more easily verify that we get the same answer both with and within that optimization. FossilOrigin-Name: fd0c9a123b58b7b134ed67f26dbb4196b61e56227f078422cc7e9a3497054c2d --- manifest | 21 +++++++++------------ manifest.uuid | 2 +- src/main.c | 2 +- src/select.c | 6 +++++- src/shell.c.in | 2 +- src/sqliteInt.h | 39 ++++++++++++++++++++------------------- 6 files changed, 37 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index d0beeb67d4..761cd8ddc4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\senhancements\sto\sthe\smin/max\soptimization\sof\n[/info/b8ba2f17f938c035|check-in\sb8ba2f17f938c035]\sto\sfix\sthe\sperformance\nregression\sidentified\sby\n[forum:/forumpost/623f571482|forum\spost\s623f571482]. -D 2021-01-13T15:23:17.568 +C Expand\sthe\snumber\sof\soptimization-disable\sbits\sfrom\s16\sto\s32.\s\sUse\sone\sof\nthe\snew\sbits\sto\sdisable\sthe\smin/max\soptimization,\sso\sthat\swe\scan\smore\seasily\nverify\sthat\swe\sget\sthe\ssame\sanswer\sboth\swith\sand\swithin\sthat\soptimization. +D 2021-01-13T19:28:17.446 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -504,7 +504,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c c5e0c25cfb9960d9b7d49043de6adc12748853bc6dea76f5adef059e366f2f70 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 8c9c8cd2bd8eecdb06d9b6e89de7e9e65bae45cc8fc33609cc74023a5c296067 -F src/main.c 97e9f137354bc1f76dc9bb60a0a24f8c45cf73b33e80d3ee4c64155336fb820d +F src/main.c 1c5de7b3fabcdf05f4fe563aab5d81d175b89c67a8678a12ba86629356afa356 F src/malloc.c c1af4ac5a463648cd2953fd4ac679b3ba9022ce5ec794a60806150ad69dfd33a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -540,12 +540,12 @@ F src/printf.c 30e92b638fac71dcd85cdea1d12ecfae354c9adee2c71e8e1ae4727cde7c91ed F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 1948a92ca9eab776632816b97e57c61d933474a78aad4f4ef835c916a83dbb1c F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 083c9bcc6b2ec80f172d414f8129679dddfb25e1ad57364cf3677bb504d62158 -F src/shell.c.in 8331e564cd73e1784dd034d88ce626d2867531e9cc38ece980b256b09eb548b3 +F src/select.c 738cb746189f721f59972993c13085fa2975c4cbfd04ba26445f3b42c81237dc +F src/shell.c.in 9ebc74e4f05cfbd0f4a36060fdaeff1da4e9af4458358722bc08c5a1ab9a0879 F src/sqlite.h.in 0af968a1fa3c717261e1df0ed105fa7bddb4d82de7e0adb3eab49e6aa81b4de7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 61b38c073d5e1e96a3d45271b257aef27d0d13da2bea5347692ae579475cd95e -F src/sqliteInt.h cecaa4eea2af66d36e4b1453d08da83b4d9ad2de2510d58d518ebd51fbd2692b +F src/sqliteInt.h 6aad58a5ae1374e18ea53d0c3ea71f047b67313426767783bd7fa14ee786725a F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -1895,10 +1895,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 11e4eb095746602961a178044809a68a77ba7b367596997bef726e54062423d9 -R 14dcd4c79aadaa323e7c1244a63358d2 -T *branch * minmax-opt-exp -T *sym-minmax-opt-exp * -T -sym-trunk * +P 188772a1dbaf066fbddd39c718fdd87478b19a920622f4640bcb79d4ef065331 +R fee2bff3fc3f4ab0866b8b2d20b15a63 U drh -Z 525d6c9785240d33b80adadc935f19d6 +Z b04c23448445719d4a662625f97c5411 diff --git a/manifest.uuid b/manifest.uuid index a15faf20cd..834b730385 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -188772a1dbaf066fbddd39c718fdd87478b19a920622f4640bcb79d4ef065331 \ No newline at end of file +fd0c9a123b58b7b134ed67f26dbb4196b61e56227f078422cc7e9a3497054c2d \ No newline at end of file diff --git a/src/main.c b/src/main.c index bbf81e778e..0f3989d9df 100644 --- a/src/main.c +++ b/src/main.c @@ -4080,7 +4080,7 @@ int sqlite3_test_control(int op, ...){ */ case SQLITE_TESTCTRL_OPTIMIZATIONS: { sqlite3 *db = va_arg(ap, sqlite3*); - db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); + db->dbOptFlags = va_arg(ap, u32); break; } diff --git a/src/select.c b/src/select.c index 76806b66ea..7c3019890c 100644 --- a/src/select.c +++ b/src/select.c @@ -4640,7 +4640,11 @@ static u8 minMaxQuery(sqlite3 *db, Expr *pFunc, ExprList **ppMinMax){ assert( *ppMinMax==0 ); assert( pFunc->op==TK_AGG_FUNCTION ); assert( !IsWindowFunc(pFunc) ); - if( pEList==0 || pEList->nExpr!=1 || ExprHasProperty(pFunc, EP_WinFunc) ){ + if( pEList==0 + || pEList->nExpr!=1 + || ExprHasProperty(pFunc, EP_WinFunc) + || OptimizationDisabled(db, SQLITE_MinMaxOpt) + ){ return eRet; } zFunc = pFunc->u.zToken; diff --git a/src/shell.c.in b/src/shell.c.in index 734a54546e..bfb9648128 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -9995,7 +9995,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* sqlite3_test_control(int, db, int) */ case SQLITE_TESTCTRL_OPTIMIZATIONS: if( nArg==3 ){ - int opt = (int)strtol(azArg[2], 0, 0); + unsigned int opt = (unsigned int)strtol(azArg[2], 0, 0); rc2 = sqlite3_test_control(testctrl, p->db, opt); isOk = 3; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8432fe20cd..0581e907da 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1496,7 +1496,7 @@ struct sqlite3 { int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ int iSysErrno; /* Errno value from last system error */ - u16 dbOptFlags; /* Flags to enable/disable optimizations */ + u32 dbOptFlags; /* Flags to enable/disable optimizations */ u8 enc; /* Text encoding */ u8 autoCommit; /* The auto-commit flag. */ u8 temp_store; /* 1: file 2: memory 0: default */ @@ -1703,24 +1703,25 @@ struct sqlite3 { ** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to ** selectively disable various optimizations. */ -#define SQLITE_QueryFlattener 0x0001 /* Query flattening */ -#define SQLITE_WindowFunc 0x0002 /* Use xInverse for window functions */ -#define SQLITE_GroupByOrder 0x0004 /* GROUPBY cover of ORDERBY */ -#define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ -#define SQLITE_DistinctOpt 0x0010 /* DISTINCT using indexes */ -#define SQLITE_CoverIdxScan 0x0020 /* Covering index scans */ -#define SQLITE_OrderByIdxJoin 0x0040 /* ORDER BY of joins via index */ -#define SQLITE_Transitive 0x0080 /* Transitive constraints */ -#define SQLITE_OmitNoopJoin 0x0100 /* Omit unused tables in joins */ -#define SQLITE_CountOfView 0x0200 /* The count-of-view optimization */ -#define SQLITE_CursorHints 0x0400 /* Add OP_CursorHint opcodes */ -#define SQLITE_Stat4 0x0800 /* Use STAT4 data */ - /* TH3 expects the Stat4 ^^^^^^ value to be 0x0800. Don't change it */ -#define SQLITE_PushDown 0x1000 /* The push-down optimization */ -#define SQLITE_SimplifyJoin 0x2000 /* Convert LEFT JOIN to JOIN */ -#define SQLITE_SkipScan 0x4000 /* Skip-scans */ -#define SQLITE_PropagateConst 0x8000 /* The constant propagation opt */ -#define SQLITE_AllOpts 0xffff /* All optimizations */ +#define SQLITE_QueryFlattener 0x00000001 /* Query flattening */ +#define SQLITE_WindowFunc 0x00000002 /* Use xInverse for window functions */ +#define SQLITE_GroupByOrder 0x00000004 /* GROUPBY cover of ORDERBY */ +#define SQLITE_FactorOutConst 0x00000008 /* Constant factoring */ +#define SQLITE_DistinctOpt 0x00000010 /* DISTINCT using indexes */ +#define SQLITE_CoverIdxScan 0x00000020 /* Covering index scans */ +#define SQLITE_OrderByIdxJoin 0x00000040 /* ORDER BY of joins via index */ +#define SQLITE_Transitive 0x00000080 /* Transitive constraints */ +#define SQLITE_OmitNoopJoin 0x00000100 /* Omit unused tables in joins */ +#define SQLITE_CountOfView 0x00000200 /* The count-of-view optimization */ +#define SQLITE_CursorHints 0x00000400 /* Add OP_CursorHint opcodes */ +#define SQLITE_Stat4 0x00000800 /* Use STAT4 data */ + /* TH3 expects this value ^^^^^^^^^^ to be 0x0000800. Don't change it */ +#define SQLITE_PushDown 0x00001000 /* The push-down optimization */ +#define SQLITE_SimplifyJoin 0x00002000 /* Convert LEFT JOIN to JOIN */ +#define SQLITE_SkipScan 0x00004000 /* Skip-scans */ +#define SQLITE_PropagateConst 0x00008000 /* The constant propagation opt */ +#define SQLITE_MinMaxOpt 0x00010000 /* The min/max optimization */ +#define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* ** Macros for testing whether or not optimizations are enabled or disabled. From 5e5683ae466667c61aa4cd5127850f080508eab9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 13 Jan 2021 21:05:07 +0000 Subject: [PATCH 05/30] Fix a harmless compiler warning. FossilOrigin-Name: 83ec01e38cbd22147ba544e15eae32c72e0523a55b54851e483dc2effc64f206 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/prepare.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 761cd8ddc4..f632558422 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Expand\sthe\snumber\sof\soptimization-disable\sbits\sfrom\s16\sto\s32.\s\sUse\sone\sof\nthe\snew\sbits\sto\sdisable\sthe\smin/max\soptimization,\sso\sthat\swe\scan\smore\seasily\nverify\sthat\swe\sget\sthe\ssame\sanswer\sboth\swith\sand\swithin\sthat\soptimization. -D 2021-01-13T19:28:17.446 +C Fix\sa\sharmless\scompiler\swarning. +D 2021-01-13T21:05:07.314 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -535,7 +535,7 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c 6596e10baf3d8f84cc1585d226cf1ab26564a5f5caf85a15757a281ff977d51a F src/pragma.c 6daaaecc26a4b09481d21722525b079ce756751a43a79cc1d8f122d686806193 F src/pragma.h 8dc78ab7e9ec6ce3ded8332810a2066f1ef6267e2e03cd7356ee00276125c6cf -F src/prepare.c cfe5a0ec9fd612c89b4d50acfd4b796bd5fe850441cab0d866d72fa3aa982c45 +F src/prepare.c f288cbc35f79eb32e162de7e80a63ebe00d80e639dcfac071bee11570cbdb16f F src/printf.c 30e92b638fac71dcd85cdea1d12ecfae354c9adee2c71e8e1ae4727cde7c91ed F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 1948a92ca9eab776632816b97e57c61d933474a78aad4f4ef835c916a83dbb1c @@ -1895,7 +1895,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 188772a1dbaf066fbddd39c718fdd87478b19a920622f4640bcb79d4ef065331 -R fee2bff3fc3f4ab0866b8b2d20b15a63 +P fd0c9a123b58b7b134ed67f26dbb4196b61e56227f078422cc7e9a3497054c2d +R 066b14d85f5f9812b58b6c0444ced87b U drh -Z b04c23448445719d4a662625f97c5411 +Z 1cdf44f5345d423ffb8022710ae292a6 diff --git a/manifest.uuid b/manifest.uuid index 834b730385..72d1bbda5b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fd0c9a123b58b7b134ed67f26dbb4196b61e56227f078422cc7e9a3497054c2d \ No newline at end of file +83ec01e38cbd22147ba544e15eae32c72e0523a55b54851e483dc2effc64f206 \ No newline at end of file diff --git a/src/prepare.c b/src/prepare.c index f93a6f07ad..87c1ab9368 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -571,10 +571,10 @@ void sqlite3ParserReset(Parse *pParse){ pThis = pNext; } while( pParse->pCleanup ){ - ParseCleanup *pThis = pParse->pCleanup; - pParse->pCleanup = pThis->pNext; - pThis->xCleanup(db, pThis->pPtr); - sqlite3DbFree(db, pThis); + ParseCleanup *pCleanup = pParse->pCleanup; + pParse->pCleanup = pCleanup->pNext; + pCleanup->xCleanup(db, pCleanup->pPtr); + sqlite3DbFree(db, pCleanup); } sqlite3DbFree(db, pParse->aLabel); if( pParse->pConstExpr ){ From 5870dc80f906c5e841d330db095ec2dae6db4ce9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 14 Jan 2021 00:53:14 +0000 Subject: [PATCH 06/30] The early-out of the inner loop on the min/max optimization was overly aggressive for the cases where there is a join and outer loops contain IN operators. Fix this. Test case in TH3. FossilOrigin-Name: ccd3bae14b6b47bb0f9622700c04db989f76ce65e10e0709964cfd0675eca762 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 13 ++++++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index f632558422..a0e67a5b46 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\scompiler\swarning. -D 2021-01-13T21:05:07.314 +C The\searly-out\sof\sthe\sinner\sloop\son\sthe\smin/max\soptimization\swas\soverly\naggressive\sfor\sthe\scases\swhere\sthere\sis\sa\sjoin\sand\souter\sloops\scontain\nIN\soperators.\s\sFix\sthis.\s\sTest\scase\sin\sTH3. +D 2021-01-14T00:53:14.879 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -627,7 +627,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 69e770e96fd56cc21608992bf2c6f1f3dc5cf2572d0495c6a643b06c3a679f14 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c -F src/where.c dee929890fe399993d4e78a5a89d4b289f7ab015446e157d2e4c25477fdfa060 +F src/where.c 0e6abb22a2323fec80b450825593c26a2ad8f4815d1ee3af9969d8f6144bf681 F src/whereInt.h 9a3f577619f07700d16d89eeb2f3d94d6b7ed7f109c2dacf0ce8844921549506 F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee F src/whereexpr.c 3a463e156ea388083c501502229c2c7f4f5c6b5330ea59bdf40d6eb6e155a25f @@ -1895,7 +1895,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fd0c9a123b58b7b134ed67f26dbb4196b61e56227f078422cc7e9a3497054c2d -R 066b14d85f5f9812b58b6c0444ced87b +P 83ec01e38cbd22147ba544e15eae32c72e0523a55b54851e483dc2effc64f206 +R 3dc0d948d76c5e90d01be3a954f85921 U drh -Z 1cdf44f5345d423ffb8022710ae292a6 +Z 4b60af58a6a03c91f6ab835dbd142db0 diff --git a/manifest.uuid b/manifest.uuid index 72d1bbda5b..7f27c3343a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83ec01e38cbd22147ba544e15eae32c72e0523a55b54851e483dc2effc64f206 \ No newline at end of file +ccd3bae14b6b47bb0f9622700c04db989f76ce65e10e0709964cfd0675eca762 \ No newline at end of file diff --git a/src/where.c b/src/where.c index adc688b4ba..8128c2ed0b 100644 --- a/src/where.c +++ b/src/where.c @@ -112,14 +112,17 @@ int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){ */ void sqlite3WhereMinMaxOptEarlyOut(Vdbe *v, WhereInfo *pWInfo){ WhereLevel *pInner; + int i; if( !pWInfo->bOrderedInnerLoop ) return; if( pWInfo->nOBSat==0 ) return; - pInner = &pWInfo->a[pWInfo->nLevel-1]; - if( (pInner->pWLoop->wsFlags & WHERE_COLUMN_IN)==0 ){ - sqlite3VdbeGoto(v, pWInfo->iBreak); - }else if( pInner->addrNxt!=pWInfo->iContinue ){ - sqlite3VdbeGoto(v, pInner->addrNxt); + for(i=pWInfo->nLevel-1; i>=0; i--){ + pInner = &pWInfo->a[i]; + if( (pInner->pWLoop->wsFlags & WHERE_COLUMN_IN)!=0 ){ + sqlite3VdbeGoto(v, pInner->addrNxt); + return; + } } + sqlite3VdbeGoto(v, pWInfo->iBreak); } /* From 6bfc167a67586d465ed995ae6dd3216967fc83c6 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 14 Jan 2021 20:50:40 +0000 Subject: [PATCH 07/30] Allow the planner to convert an EXISTS(SELECT...) expression in a WHERE clause to the equivalent IN(...) expression in situations where this is possible and advantageous. FossilOrigin-Name: 9f90a88221d0694951c353e58efce342eb0b868b8ca6a4469c8205e5c7855b24 --- manifest | 17 ++-- manifest.uuid | 2 +- src/whereexpr.c | 200 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b050b37ce0..a99b42e9cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C cli:\sOmit\ssurplus\swhitespace\sat\sthe\send\sof\slines\sin\s.explain\soutput. -D 2021-01-13T12:59:20.834 +C Allow\sthe\splanner\sto\sconvert\san\sEXISTS(SELECT...)\sexpression\sin\sa\sWHERE\sclause\sto\sthe\sequivalent\sIN(...)\sexpression\sin\ssituations\swhere\sthis\sis\spossible\sand\sadvantageous. +D 2021-01-14T20:50:40.571 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -630,7 +630,7 @@ F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c F src/where.c 3d31871d03906312d7d71a9c0b28c97bcbaead7606dfc15f9b3d080b18702385 F src/whereInt.h 9a3f577619f07700d16d89eeb2f3d94d6b7ed7f109c2dacf0ce8844921549506 F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee -F src/whereexpr.c 3a463e156ea388083c501502229c2c7f4f5c6b5330ea59bdf40d6eb6e155a25f +F src/whereexpr.c e48e3edea45b4afabbf6f6ece9647734c828a20c49a8cc780ff4d69b42f55fa4 F src/window.c edd6f5e25a1e8f2b6f5305b7f5f7da7bb35f07f0d432b255b1d4c2fcab4205aa F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -1895,7 +1895,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P db0ecfe66433f8915b6eb16d3735a4a0d0f8e0bbc395bc9c1364387506fc4657 -R 2ad612629a79dbad03e8b7f173945d59 -U drh -Z f407a95e341e109313c0922f7d2a229b +P 11e4eb095746602961a178044809a68a77ba7b367596997bef726e54062423d9 +R 1a9ea60bc9bb7dab69ccea9e0b8c40d8 +T *branch * exists-to-in +T *sym-exists-to-in * +T -sym-trunk * +U dan +Z d4d95c9878f460cd53a2ce8ada6afd2a diff --git a/manifest.uuid b/manifest.uuid index ccaf3daab6..e8520659ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -11e4eb095746602961a178044809a68a77ba7b367596997bef726e54062423d9 \ No newline at end of file +9f90a88221d0694951c353e58efce342eb0b868b8ca6a4469c8205e5c7855b24 \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index a77eb36106..9326a57fcb 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1007,6 +1007,202 @@ static int exprMightBeIndexed( return exprMightBeIndexed2(pFrom,mPrereq,aiCurCol,pExpr); } +/* +** Expression callback for exprUsesSrclist(). +*/ +static int exprUsesSrclistCb(Walker *p, Expr *pExpr){ + if( pExpr->op==TK_COLUMN ){ + SrcList *pSrc = p->u.pSrcList; + int iCsr = pExpr->iTable; + int ii; + for(ii=0; iinSrc; ii++){ + if( pSrc->a[ii].iCursor==iCsr ){ + return p->eCode ? WRC_Abort : WRC_Continue; + } + } + return p->eCode ? WRC_Continue : WRC_Abort; + } + return WRC_Continue; +} + +/* +** Select callback for exprUsesSrclist(). +*/ +static int exprUsesSrclistSelectCb(Walker *p, Select *pSelect){ + return WRC_Abort; +} + +/* +** This function always returns true if expression pExpr contains +** a sub-select. +** +** If there is no sub-select and bUses is 1, then true is returned +** if the expression contains at least one TK_COLUMN node that refers +** to a table in pSrc. +** +** Or, if there is no sub-select and bUses is 0, then true is returned +** if the expression contains at least one TK_COLUMN node that refers +** to a table that is not in pSrc. +*/ +static int exprUsesSrclist(SrcList *pSrc, Expr *pExpr, int bUses){ + Walker sWalker; + memset(&sWalker, 0, sizeof(Walker)); + sWalker.eCode = bUses; + sWalker.u.pSrcList = pSrc; + sWalker.xExprCallback = exprUsesSrclistCb; + sWalker.xSelectCallback = exprUsesSrclistSelectCb; + return (sqlite3WalkExpr(&sWalker, pExpr)==WRC_Abort); +} + +struct ExistsToInCtx { + SrcList *pSrc; + Expr *pInLhs; + Expr *pEq; + Expr **ppAnd; + Expr **ppParent; +}; + +static int exprExistsToInIter( + struct ExistsToInCtx *p, + Expr *pExpr, + Expr **ppExpr +){ + assert( ppExpr==0 || *ppExpr==pExpr ); + switch( pExpr->op ){ + case TK_AND: + p->ppParent = ppExpr; + if( exprExistsToInIter(p, pExpr->pLeft, &pExpr->pLeft) ) return 1; + p->ppParent = ppExpr; + if( exprExistsToInIter(p, pExpr->pRight, &pExpr->pRight) ) return 1; + break; + case TK_EQ: { + int bLeft = exprUsesSrclist(p->pSrc, pExpr->pLeft, 0); + int bRight = exprUsesSrclist(p->pSrc, pExpr->pRight, 0); + if( bLeft || bRight ){ + if( (bLeft && bRight) || p->pInLhs ) return 1; + p->pInLhs = bLeft ? pExpr->pLeft : pExpr->pRight; + p->pEq = pExpr; + p->ppAnd = p->ppParent; + if( exprUsesSrclist(p->pSrc, p->pInLhs, 1) ) return 1; + } + break; + } + default: + if( exprUsesSrclist(p->pSrc, pExpr, 0) ){ + return 1; + } + break; + } + + return 0; +} + +static Expr *exprAnalyzeExistsFindEq( + SrcList *pSrc, + Expr *pWhere, /* WHERE clause to traverse */ + Expr **ppEq, /* OUT: == node from WHERE clause */ + Expr ***pppAnd /* OUT: Pointer to parent of ==, if any */ +){ + struct ExistsToInCtx ctx; + memset(&ctx, 0, sizeof(ctx)); + ctx.pSrc = pSrc; + if( exprExistsToInIter(&ctx, pWhere, 0) ){ + return 0; + } + if( ppEq ) *ppEq = ctx.pEq; + if( pppAnd ) *pppAnd = ctx.ppAnd; + return ctx.pInLhs; +} + +/* +** Term idxTerm of the WHERE clause passed as the second argument is an +** EXISTS expression with a correlated SELECT statement on the RHS. +** This function analyzes the SELECT statement, and if possible adds an +** equivalent "? IN(SELECT...)" virtual term to the WHERE clause. +** +** For an EXISTS term such as the following: +** +** EXISTS (SELECT ... FROM WHERE = AND ) +** +** The virtual IN() term added is: +** +** IN (SELECT FROM WHERE ) +** +** The virtual term is only added if the following conditions are met: +** +** 1. The sub-select must not be an aggregate or use window functions, +** +** 2. The sub-select must not be a compound SELECT, +** +** 3. Expression must refer to at least one column from the outer +** query, and must not refer to any column from the inner query +** (i.e. from ). +** +** 4. and must not refer to any values from the outer query. +** In other words, once has been removed, the inner query +** must not be correlated. +** +*/ +static void exprAnalyzeExists( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* the WHERE clause */ + int idxTerm /* Index of the term to be analyzed */ +){ + Parse *pParse = pWC->pWInfo->pParse; + WhereTerm *pTerm = &pWC->a[idxTerm]; + Expr *pExpr = pTerm->pExpr; + Select *pSel = pExpr->x.pSelect; + Expr *pDup = 0; + Expr *pEq = 0; + Expr *pRet = 0; + Expr *pInLhs = 0; + Expr **ppAnd = 0; + int idxNew; + + assert( pExpr->op==TK_EXISTS ); + assert( (pExpr->flags & EP_VarSelect) && (pExpr->flags & EP_xIsSelect) ); + + if( (pSel->selFlags & SF_Aggregate) || pSel->pWin ) return; + if( pSel->pPrior ) return; + if( pSel->pWhere==0 ) return; + if( 0==exprAnalyzeExistsFindEq(pSel->pSrc, pSel->pWhere, 0, 0) ) return; + + pDup = sqlite3ExprDup(pParse->db, pExpr, 0); + if( pDup==0 ) return; + pSel = pDup->x.pSelect; + sqlite3ExprListDelete(pParse->db, pSel->pEList); + pSel->pEList = 0; + + pInLhs = exprAnalyzeExistsFindEq(pSel->pSrc, pSel->pWhere, &pEq, &ppAnd); + + assert( pDup->pLeft==0 ); + pDup->op = TK_IN; + pDup->pLeft = pInLhs; + pDup->flags &= ~EP_VarSelect; + pRet = (pInLhs==pEq->pLeft) ? pEq->pRight : pEq->pLeft; + pSel->pEList = sqlite3ExprListAppend(pParse, 0, pRet); + pEq->pLeft = 0; + pEq->pRight = 0; + if( ppAnd ){ + Expr *pAnd = *ppAnd; + Expr *pOther = (pAnd->pLeft==pEq) ? pAnd->pRight : pAnd->pLeft; + pAnd->pLeft = pAnd->pRight = 0; + sqlite3ExprDelete(pParse->db, pAnd); + *ppAnd = pOther; + }else{ + assert( pSel->pWhere==pEq ); + pSel->pWhere = 0; + } + sqlite3ExprDelete(pParse->db, pEq); + + idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); + if( idxNew ){ + exprAnalyze(pSrc, pWC, idxNew); + markTermAsChild(pWC, idxNew, idxTerm); + pWC->a[idxTerm].wtFlags |= TERM_COPIED; + } +} + /* ** The input to this routine is an WhereTerm structure with only the ** "pExpr" field filled in. The job of this routine is to analyze the @@ -1418,6 +1614,10 @@ static void exprAnalyze( } #endif /* SQLITE_ENABLE_STAT4 */ + if( pExpr->op==TK_EXISTS && (pExpr->flags & EP_VarSelect) ){ + exprAnalyzeExists(pSrc, pWC, idxTerm); + } + /* Prevent ON clause terms of a LEFT JOIN from being used to drive ** an index for tables to the left of the join. */ From 76cac6ef0663c097c9ba5195556963567cece3b5 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Jan 2021 11:39:46 +0000 Subject: [PATCH 08/30] Add simple tests (and a fix) for the change on this branch. FossilOrigin-Name: 897f3f40267dc922f0fda287484435e1fd8709bade3e87c3829e2f945bb5e4aa --- manifest | 16 ++++----- manifest.uuid | 2 +- src/whereexpr.c | 27 +++++++-------- test/exists2.test | 87 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 107 insertions(+), 25 deletions(-) create mode 100644 test/exists2.test diff --git a/manifest b/manifest index a99b42e9cc..93a625e98a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\splanner\sto\sconvert\san\sEXISTS(SELECT...)\sexpression\sin\sa\sWHERE\sclause\sto\sthe\sequivalent\sIN(...)\sexpression\sin\ssituations\swhere\sthis\sis\spossible\sand\sadvantageous. -D 2021-01-14T20:50:40.571 +C Add\ssimple\stests\s(and\sa\sfix)\sfor\sthe\schange\son\sthis\sbranch. +D 2021-01-15T11:39:46.131 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -630,7 +630,7 @@ F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c F src/where.c 3d31871d03906312d7d71a9c0b28c97bcbaead7606dfc15f9b3d080b18702385 F src/whereInt.h 9a3f577619f07700d16d89eeb2f3d94d6b7ed7f109c2dacf0ce8844921549506 F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee -F src/whereexpr.c e48e3edea45b4afabbf6f6ece9647734c828a20c49a8cc780ff4d69b42f55fa4 +F src/whereexpr.c 1f4fac8ea8eac6facb9e80107c659094a56a8ad0b35108717ac99ea57bbc71bc F src/window.c edd6f5e25a1e8f2b6f5305b7f5f7da7bb35f07f0d432b255b1d4c2fcab4205aa F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -874,6 +874,7 @@ F test/exclusive.test 7ff63be7503990921838d5c9f77f6e33e68e48ed1a9d48cd28745bf650 F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac +F test/exists2.test 44e18a6aaadefa8e1c68a8453fb22a57a0270a673d9b256790d142ea257459e3 F test/expr.test 26cd01e8485bc48c8aa6a1add598e9ce1e706b4eb4f3f554e0b0223022e8c2cf F test/expr2.test c27327ae9c017a7ff6280123f67aff496f912da74d78c888926d68b46ec75fd8 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9 @@ -1895,10 +1896,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 11e4eb095746602961a178044809a68a77ba7b367596997bef726e54062423d9 -R 1a9ea60bc9bb7dab69ccea9e0b8c40d8 -T *branch * exists-to-in -T *sym-exists-to-in * -T -sym-trunk * +P 9f90a88221d0694951c353e58efce342eb0b868b8ca6a4469c8205e5c7855b24 +R 9984162db1d3035db01dde79c4fce484 U dan -Z d4d95c9878f460cd53a2ce8ada6afd2a +Z 278fef5940bb689562d7ebf0ab17cbb2 diff --git a/manifest.uuid b/manifest.uuid index e8520659ef..ace34aaabc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9f90a88221d0694951c353e58efce342eb0b868b8ca6a4469c8205e5c7855b24 \ No newline at end of file +897f3f40267dc922f0fda287484435e1fd8709bade3e87c3829e2f945bb5e4aa \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 9326a57fcb..e5e566bbfe 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1062,18 +1062,14 @@ struct ExistsToInCtx { Expr **ppParent; }; -static int exprExistsToInIter( - struct ExistsToInCtx *p, - Expr *pExpr, - Expr **ppExpr -){ - assert( ppExpr==0 || *ppExpr==pExpr ); +static int exprExistsToInIter(struct ExistsToInCtx *p, Expr **ppExpr){ + Expr *pExpr = *ppExpr; switch( pExpr->op ){ case TK_AND: p->ppParent = ppExpr; - if( exprExistsToInIter(p, pExpr->pLeft, &pExpr->pLeft) ) return 1; + if( exprExistsToInIter(p, &pExpr->pLeft) ) return 1; p->ppParent = ppExpr; - if( exprExistsToInIter(p, pExpr->pRight, &pExpr->pRight) ) return 1; + if( exprExistsToInIter(p, &pExpr->pRight) ) return 1; break; case TK_EQ: { int bLeft = exprUsesSrclist(p->pSrc, pExpr->pLeft, 0); @@ -1081,9 +1077,9 @@ static int exprExistsToInIter( if( bLeft || bRight ){ if( (bLeft && bRight) || p->pInLhs ) return 1; p->pInLhs = bLeft ? pExpr->pLeft : pExpr->pRight; + if( exprUsesSrclist(p->pSrc, p->pInLhs, 1) ) return 1; p->pEq = pExpr; p->ppAnd = p->ppParent; - if( exprUsesSrclist(p->pSrc, p->pInLhs, 1) ) return 1; } break; } @@ -1098,15 +1094,14 @@ static int exprExistsToInIter( } static Expr *exprAnalyzeExistsFindEq( - SrcList *pSrc, - Expr *pWhere, /* WHERE clause to traverse */ + Select *pSel, Expr **ppEq, /* OUT: == node from WHERE clause */ Expr ***pppAnd /* OUT: Pointer to parent of ==, if any */ ){ struct ExistsToInCtx ctx; memset(&ctx, 0, sizeof(ctx)); - ctx.pSrc = pSrc; - if( exprExistsToInIter(&ctx, pWhere, 0) ){ + ctx.pSrc = pSel->pSrc; + if( exprExistsToInIter(&ctx, &pSel->pWhere) ){ return 0; } if( ppEq ) *ppEq = ctx.pEq; @@ -1165,7 +1160,7 @@ static void exprAnalyzeExists( if( (pSel->selFlags & SF_Aggregate) || pSel->pWin ) return; if( pSel->pPrior ) return; if( pSel->pWhere==0 ) return; - if( 0==exprAnalyzeExistsFindEq(pSel->pSrc, pSel->pWhere, 0, 0) ) return; + if( 0==exprAnalyzeExistsFindEq(pSel, 0, 0) ) return; pDup = sqlite3ExprDup(pParse->db, pExpr, 0); if( pDup==0 ) return; @@ -1173,7 +1168,9 @@ static void exprAnalyzeExists( sqlite3ExprListDelete(pParse->db, pSel->pEList); pSel->pEList = 0; - pInLhs = exprAnalyzeExistsFindEq(pSel->pSrc, pSel->pWhere, &pEq, &ppAnd); + pInLhs = exprAnalyzeExistsFindEq(pSel, &pEq, &ppAnd); + assert( pInLhs && pEq ); + assert( pEq==pSel->pWhere || ppAnd ); assert( pDup->pLeft==0 ); pDup->op = TK_IN; diff --git a/test/exists2.test b/test/exists2.test new file mode 100644 index 0000000000..55369f8f2f --- /dev/null +++ b/test/exists2.test @@ -0,0 +1,87 @@ +# 2021 January 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing cases where EXISTS expressions are +# transformed to IN() expressions by where.c +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix exists2 + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + INSERT INTO t1 VALUES(3, 'three'); + INSERT INTO t1 VALUES(4, 'four'); + INSERT INTO t1 VALUES(5, 'five'); + INSERT INTO t1 VALUES(6, 'six'); + INSERT INTO t1 VALUES(7, 'seven'); + + CREATE TABLE t2(c INTEGER, d INTEGER); + INSERT INTO t2 VALUES(1, 1); + INSERT INTO t2 VALUES(3, 2); + INSERT INTO t2 VALUES(5, 3); + INSERT INTO t2 VALUES(7, 4); +} + +proc do_execsql_eqp_test {tn sql eqp res} { + uplevel [list do_eqp_test $tn.1 $sql [string trim $eqp]] + uplevel [list do_execsql_test $tn.2 $sql $res] +} + +do_execsql_eqp_test 1.1 { + SELECT t1.* FROM t1 WHERE EXISTS(SELECT * FROM t2 WHERE t1.a=t2.c); +} { + USING INTEGER PRIMARY KEY +} { + 1 one 3 three 5 five 7 seven +} + +do_execsql_eqp_test 1.2 { + SELECT t1.* FROM t1 WHERE EXISTS(SELECT * FROM t2 WHERE t2.c=t1.a); +} { + SEARCH TABLE t1 USING INTEGER PRIMARY KEY +} { + 1 one 3 three 5 five 7 seven +} + +do_execsql_eqp_test 1.3 { + SELECT t1.* FROM t1 WHERE EXISTS(SELECT * FROM t2 WHERE t2.c+1=t1.a); +} { + SEARCH TABLE t1 USING INTEGER PRIMARY KEY +} { + 2 two 4 four 6 six +} + +do_execsql_eqp_test 1.4 { + SELECT t1.* FROM t1 WHERE EXISTS(SELECT * FROM t2 WHERE t2.c+1=t1.a+1); +} { + SCAN TABLE t1 +} { + 1 one 3 three 5 five 7 seven +} + +breakpoint +do_execsql_eqp_test 1.5 { + SELECT t1.* FROM t1 WHERE EXISTS( + SELECT * FROM t2 WHERE t1.a=t2.c AND d IN (1, 2, 3) + ); +} { + SEARCH TABLE t1 USING INTEGER PRIMARY KEY +} { + 1 one 3 three 5 five +} + + + +finish_test From 10c9ef65fa7d453bfd8b1d82f434f29938d39342 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 15 Jan 2021 14:25:06 +0000 Subject: [PATCH 09/30] Small performance improvement in the EXISTS-to-IN translator for the common case where the EXISTS operator is not found in the WHERE clause. FossilOrigin-Name: dcb7772d7695ddbc0fe89e06c07ff4a6ae4fa05de914e2ec10b5cc07a62ed49f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/whereexpr.c | 12 ++++++++---- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index afcc0dca9c..c7f52277ae 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\senhancements\sinto\sthe\sexists-to-in\sbranch. -D 2021-01-15T14:15:31.281 +C Small\sperformance\simprovement\sin\sthe\sEXISTS-to-IN\stranslator\sfor\sthe\ncommon\scase\swhere\sthe\sEXISTS\soperator\sis\snot\sfound\sin\sthe\sWHERE\sclause. +D 2021-01-15T14:25:06.079 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -630,7 +630,7 @@ F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c F src/where.c 0e6abb22a2323fec80b450825593c26a2ad8f4815d1ee3af9969d8f6144bf681 F src/whereInt.h 9a3f577619f07700d16d89eeb2f3d94d6b7ed7f109c2dacf0ce8844921549506 F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee -F src/whereexpr.c 1f4fac8ea8eac6facb9e80107c659094a56a8ad0b35108717ac99ea57bbc71bc +F src/whereexpr.c 0ba5c743ee34255658051179441a13297dc2d6b21ef3c41120659ae3dd896ae8 F src/window.c edd6f5e25a1e8f2b6f5305b7f5f7da7bb35f07f0d432b255b1d4c2fcab4205aa F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -1896,7 +1896,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 897f3f40267dc922f0fda287484435e1fd8709bade3e87c3829e2f945bb5e4aa 249a71cc6822d6bdd5bb9e727aac81c6549693b418e9c0987b96850ee332c940 -R 6fb3b1c5ffde39ec234334a7c5ffcb56 +P 13c4c9088cc8a2426e30a2ad1e9b9969407249281c6ed16653d43a0e6852a2e4 +R c79f76fc898939a9f2cc931eb9a2c482 U drh -Z 90ae5651a54d924d6159985985e1584c +Z e1fe560394403461c8187fdaaf345432 diff --git a/manifest.uuid b/manifest.uuid index 88f1bf8bca..13001fa395 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -13c4c9088cc8a2426e30a2ad1e9b9969407249281c6ed16653d43a0e6852a2e4 \ No newline at end of file +dcb7772d7695ddbc0fe89e06c07ff4a6ae4fa05de914e2ec10b5cc07a62ed49f \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index e5e566bbfe..e360001931 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1385,6 +1385,14 @@ static void exprAnalyze( } #endif /* SQLITE_OMIT_OR_OPTIMIZATION */ + else if( pExpr->op==TK_EXISTS ){ + /* Perhaps treat an EXISTS operator as an IN operator */ + if( (pExpr->flags & EP_VarSelect)!=0 ){ + exprAnalyzeExists(pSrc, pWC, idxTerm); + } + } + + #ifndef SQLITE_OMIT_LIKE_OPTIMIZATION /* Add constraints to reduce the search space on a LIKE or GLOB ** operator. @@ -1611,10 +1619,6 @@ static void exprAnalyze( } #endif /* SQLITE_ENABLE_STAT4 */ - if( pExpr->op==TK_EXISTS && (pExpr->flags & EP_VarSelect) ){ - exprAnalyzeExists(pSrc, pWC, idxTerm); - } - /* Prevent ON clause terms of a LEFT JOIN from being used to drive ** an index for tables to the left of the join. */ From 19ef211d85972c998ce8baa834efe0797f1ab492 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 15 Jan 2021 15:17:14 +0000 Subject: [PATCH 10/30] Add a new optimizer disabling bit to close off the exists-to-in optimization, for testing purposes. FossilOrigin-Name: a80c9a076d31729282004ca372913c9fdbfb6e74711fbb8c5dc12ee0ecba2b87 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 1 + src/whereexpr.c | 4 +++- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c7f52277ae..0e289c8966 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\simprovement\sin\sthe\sEXISTS-to-IN\stranslator\sfor\sthe\ncommon\scase\swhere\sthe\sEXISTS\soperator\sis\snot\sfound\sin\sthe\sWHERE\sclause. -D 2021-01-15T14:25:06.079 +C Add\sa\snew\soptimizer\sdisabling\sbit\sto\sclose\soff\sthe\sexists-to-in\soptimization,\nfor\stesting\spurposes. +D 2021-01-15T15:17:14.152 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -545,7 +545,7 @@ F src/shell.c.in 9ebc74e4f05cfbd0f4a36060fdaeff1da4e9af4458358722bc08c5a1ab9a087 F src/sqlite.h.in 0af968a1fa3c717261e1df0ed105fa7bddb4d82de7e0adb3eab49e6aa81b4de7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 61b38c073d5e1e96a3d45271b257aef27d0d13da2bea5347692ae579475cd95e -F src/sqliteInt.h 6aad58a5ae1374e18ea53d0c3ea71f047b67313426767783bd7fa14ee786725a +F src/sqliteInt.h d921214bba203960de10ea7a9183735ec362a5def5910d468282a8f82e8b4c26 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -630,7 +630,7 @@ F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c F src/where.c 0e6abb22a2323fec80b450825593c26a2ad8f4815d1ee3af9969d8f6144bf681 F src/whereInt.h 9a3f577619f07700d16d89eeb2f3d94d6b7ed7f109c2dacf0ce8844921549506 F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee -F src/whereexpr.c 0ba5c743ee34255658051179441a13297dc2d6b21ef3c41120659ae3dd896ae8 +F src/whereexpr.c 2d42217961cf8da8280779df88bcfb7cb3ee719369cafb44ac2b376fdecf9db7 F src/window.c edd6f5e25a1e8f2b6f5305b7f5f7da7bb35f07f0d432b255b1d4c2fcab4205aa F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -1896,7 +1896,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 13c4c9088cc8a2426e30a2ad1e9b9969407249281c6ed16653d43a0e6852a2e4 -R c79f76fc898939a9f2cc931eb9a2c482 +P dcb7772d7695ddbc0fe89e06c07ff4a6ae4fa05de914e2ec10b5cc07a62ed49f +R 6fb63e2f60db07c2af9d4dfa7cc52f88 U drh -Z e1fe560394403461c8187fdaaf345432 +Z 5ba57a27c0ec24c2bc90f07584644072 diff --git a/manifest.uuid b/manifest.uuid index 13001fa395..1f56d238fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dcb7772d7695ddbc0fe89e06c07ff4a6ae4fa05de914e2ec10b5cc07a62ed49f \ No newline at end of file +a80c9a076d31729282004ca372913c9fdbfb6e74711fbb8c5dc12ee0ecba2b87 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 0581e907da..e1e0707bf3 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1721,6 +1721,7 @@ struct sqlite3 { #define SQLITE_SkipScan 0x00004000 /* Skip-scans */ #define SQLITE_PropagateConst 0x00008000 /* The constant propagation opt */ #define SQLITE_MinMaxOpt 0x00010000 /* The min/max optimization */ +#define SQLITE_ExistsToIN 0x00020000 /* The EXISTS-to-IN optimization */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* diff --git a/src/whereexpr.c b/src/whereexpr.c index e360001931..4de5af1730 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1387,7 +1387,9 @@ static void exprAnalyze( else if( pExpr->op==TK_EXISTS ){ /* Perhaps treat an EXISTS operator as an IN operator */ - if( (pExpr->flags & EP_VarSelect)!=0 ){ + if( (pExpr->flags & EP_VarSelect)!=0 + && OptimizationEnabled(db, SQLITE_ExistsToIN) + ){ exprAnalyzeExists(pSrc, pWC, idxTerm); } } From 9d326d67937b23217718cc64e67d214351691a2a Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 15 Jan 2021 15:21:27 +0000 Subject: [PATCH 11/30] Fix a potential NULL pointer dereference following OOM. FossilOrigin-Name: 8ce3cb90965771530c0021173d98720fc4c76bb99e69f7a879f80471dea0aace --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/whereexpr.c | 14 +++++++++----- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 0e289c8966..235abe4634 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\snew\soptimizer\sdisabling\sbit\sto\sclose\soff\sthe\sexists-to-in\soptimization,\nfor\stesting\spurposes. -D 2021-01-15T15:17:14.152 +C Fix\sa\spotential\sNULL\spointer\sdereference\sfollowing\sOOM. +D 2021-01-15T15:21:27.437 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -630,7 +630,7 @@ F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c F src/where.c 0e6abb22a2323fec80b450825593c26a2ad8f4815d1ee3af9969d8f6144bf681 F src/whereInt.h 9a3f577619f07700d16d89eeb2f3d94d6b7ed7f109c2dacf0ce8844921549506 F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee -F src/whereexpr.c 2d42217961cf8da8280779df88bcfb7cb3ee719369cafb44ac2b376fdecf9db7 +F src/whereexpr.c 8ea4f6cd1332fdfbfbe832dc8a9f5194990684870931e7a07c2cafbc544588e7 F src/window.c edd6f5e25a1e8f2b6f5305b7f5f7da7bb35f07f0d432b255b1d4c2fcab4205aa F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -1896,7 +1896,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dcb7772d7695ddbc0fe89e06c07ff4a6ae4fa05de914e2ec10b5cc07a62ed49f -R 6fb63e2f60db07c2af9d4dfa7cc52f88 +P a80c9a076d31729282004ca372913c9fdbfb6e74711fbb8c5dc12ee0ecba2b87 +R 31b39e7a643244cb65bba0cd39985e5a U drh -Z 5ba57a27c0ec24c2bc90f07584644072 +Z 96ac907b935dc11fb79e0a38f40aef27 diff --git a/manifest.uuid b/manifest.uuid index 1f56d238fa..b72b9116a2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a80c9a076d31729282004ca372913c9fdbfb6e74711fbb8c5dc12ee0ecba2b87 \ No newline at end of file +8ce3cb90965771530c0021173d98720fc4c76bb99e69f7a879f80471dea0aace \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 4de5af1730..0359babc50 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1153,6 +1153,7 @@ static void exprAnalyzeExists( Expr *pInLhs = 0; Expr **ppAnd = 0; int idxNew; + sqlite3 *db = pParse->db; assert( pExpr->op==TK_EXISTS ); assert( (pExpr->flags & EP_VarSelect) && (pExpr->flags & EP_xIsSelect) ); @@ -1162,10 +1163,13 @@ static void exprAnalyzeExists( if( pSel->pWhere==0 ) return; if( 0==exprAnalyzeExistsFindEq(pSel, 0, 0) ) return; - pDup = sqlite3ExprDup(pParse->db, pExpr, 0); - if( pDup==0 ) return; + pDup = sqlite3ExprDup(db, pExpr, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDup); + return; + } pSel = pDup->x.pSelect; - sqlite3ExprListDelete(pParse->db, pSel->pEList); + sqlite3ExprListDelete(db, pSel->pEList); pSel->pEList = 0; pInLhs = exprAnalyzeExistsFindEq(pSel, &pEq, &ppAnd); @@ -1184,13 +1188,13 @@ static void exprAnalyzeExists( Expr *pAnd = *ppAnd; Expr *pOther = (pAnd->pLeft==pEq) ? pAnd->pRight : pAnd->pLeft; pAnd->pLeft = pAnd->pRight = 0; - sqlite3ExprDelete(pParse->db, pAnd); + sqlite3ExprDelete(db, pAnd); *ppAnd = pOther; }else{ assert( pSel->pWhere==pEq ); pSel->pWhere = 0; } - sqlite3ExprDelete(pParse->db, pEq); + sqlite3ExprDelete(db, pEq); idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); if( idxNew ){ From e8f7fcf6f4a3655334251ad638db47bcdde37a9c Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Jan 2021 15:32:09 +0000 Subject: [PATCH 12/30] Ensure the EXISTS->IN transformation preserves the collation sequence of the comparison operation. FossilOrigin-Name: a373baae12c914e48fd84de77998e301fdd3da43b06b9d64ac24a14418ed48cd --- manifest | 16 +++++----- manifest.uuid | 2 +- src/whereexpr.c | 8 ++++- test/exists2.test | 78 ++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 93 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 235abe4634..1523531c03 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spotential\sNULL\spointer\sdereference\sfollowing\sOOM. -D 2021-01-15T15:21:27.437 +C Ensure\sthe\sEXISTS->IN\stransformation\spreserves\sthe\scollation\ssequence\sof\sthe\scomparison\soperation. +D 2021-01-15T15:32:09.587 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -630,7 +630,7 @@ F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c F src/where.c 0e6abb22a2323fec80b450825593c26a2ad8f4815d1ee3af9969d8f6144bf681 F src/whereInt.h 9a3f577619f07700d16d89eeb2f3d94d6b7ed7f109c2dacf0ce8844921549506 F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee -F src/whereexpr.c 8ea4f6cd1332fdfbfbe832dc8a9f5194990684870931e7a07c2cafbc544588e7 +F src/whereexpr.c d63dcb44253f7ad67449759bca6da63cac1c352aba7bea46ba3210154ad21e22 F src/window.c edd6f5e25a1e8f2b6f5305b7f5f7da7bb35f07f0d432b255b1d4c2fcab4205aa F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -874,7 +874,7 @@ F test/exclusive.test 7ff63be7503990921838d5c9f77f6e33e68e48ed1a9d48cd28745bf650 F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac -F test/exists2.test 44e18a6aaadefa8e1c68a8453fb22a57a0270a673d9b256790d142ea257459e3 +F test/exists2.test 1022fe1c85ee162ae55efda6c00497b29d1f61ecb21e7d5f1c014eb46245c70a F test/expr.test 26cd01e8485bc48c8aa6a1add598e9ce1e706b4eb4f3f554e0b0223022e8c2cf F test/expr2.test c27327ae9c017a7ff6280123f67aff496f912da74d78c888926d68b46ec75fd8 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9 @@ -1896,7 +1896,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a80c9a076d31729282004ca372913c9fdbfb6e74711fbb8c5dc12ee0ecba2b87 -R 31b39e7a643244cb65bba0cd39985e5a -U drh -Z 96ac907b935dc11fb79e0a38f40aef27 +P 8ce3cb90965771530c0021173d98720fc4c76bb99e69f7a879f80471dea0aace +R bc5d3e0052171a4a3fcfe5fb2a3891b3 +U dan +Z 4e48cccd8bd31a2f422fbd932b326f93 diff --git a/manifest.uuid b/manifest.uuid index b72b9116a2..58184acaa4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8ce3cb90965771530c0021173d98720fc4c76bb99e69f7a879f80471dea0aace \ No newline at end of file +a373baae12c914e48fd84de77998e301fdd3da43b06b9d64ac24a14418ed48cd \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 0359babc50..82a9759f39 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1175,12 +1175,18 @@ static void exprAnalyzeExists( pInLhs = exprAnalyzeExistsFindEq(pSel, &pEq, &ppAnd); assert( pInLhs && pEq ); assert( pEq==pSel->pWhere || ppAnd ); + if( pInLhs==pEq->pLeft ){ + pRet = pEq->pRight; + }else{ + CollSeq *p = sqlite3ExprCompareCollSeq(pParse, pEq); + pInLhs = sqlite3ExprAddCollateString(pParse, pInLhs, p?p->zName:"BINARY"); + pRet = pEq->pLeft; + } assert( pDup->pLeft==0 ); pDup->op = TK_IN; pDup->pLeft = pInLhs; pDup->flags &= ~EP_VarSelect; - pRet = (pInLhs==pEq->pLeft) ? pEq->pRight : pEq->pLeft; pSel->pEList = sqlite3ExprListAppend(pParse, 0, pRet); pEq->pLeft = 0; pEq->pRight = 0; diff --git a/test/exists2.test b/test/exists2.test index 55369f8f2f..3472751a21 100644 --- a/test/exists2.test +++ b/test/exists2.test @@ -71,7 +71,6 @@ do_execsql_eqp_test 1.4 { 1 one 3 three 5 five 7 seven } -breakpoint do_execsql_eqp_test 1.5 { SELECT t1.* FROM t1 WHERE EXISTS( SELECT * FROM t2 WHERE t1.a=t2.c AND d IN (1, 2, 3) @@ -82,6 +81,83 @@ do_execsql_eqp_test 1.5 { 1 one 3 three 5 five } +do_execsql_eqp_test 1.6 { + SELECT t1.* FROM t1 WHERE EXISTS( + SELECT * FROM t2 WHERE d IN (1, 2, 3)AND t1.a=t2.c + ); +} { + SEARCH TABLE t1 USING INTEGER PRIMARY KEY +} { + 1 one 3 three 5 five +} + +do_execsql_eqp_test 1.7 { + SELECT t1.* FROM t1 WHERE EXISTS( + SELECT * FROM t2 WHERE d IN (1, 2, 3)AND t1.a=t2.c + ); +} { + SEARCH TABLE t1 USING INTEGER PRIMARY KEY +} { + 1 one 3 three 5 five +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.0 { + CREATE TABLE t3(a TEXT PRIMARY KEY, b TEXT) WITHOUT ROWID; + CREATE TABLE t4(c TEXT COLLATE nocase); + + INSERT INTO t3 VALUES('one', 'i'); + INSERT INTO t3 VALUES('two', 'ii'); + INSERT INTO t3 VALUES('three', 'iii'); + INSERT INTO t3 VALUES('four', 'iv'); + INSERT INTO t3 VALUES('five', 'v'); + + INSERT INTO t4 VALUES('FIVE'), ('four'), ('TWO'), ('one'); +} + +do_execsql_test 2.1 { SELECT a FROM t3, t4 WHERE a=c } {four one} +do_execsql_test 2.2 { SELECT a FROM t3, t4 WHERE c=a } {five four one two} + +do_execsql_eqp_test 2.3 { + SELECT a FROM t3 WHERE EXISTS (SELECT 1 FROM t4 WHERE a=c) +} { + SEARCH TABLE t3 USING PRIMARY KEY +} { + four one +} + +do_execsql_eqp_test 2.4 { + SELECT a FROM t3 WHERE EXISTS (SELECT 1 FROM t4 WHERE c=a) +} { + SCAN TABLE t3 +} { + five four one two +} + +do_execsql_test 2.5 { + CREATE INDEX t3anc ON t3(a COLLATE nocase); +} + +do_execsql_eqp_test 2.6 { + SELECT a FROM t3 WHERE EXISTS (SELECT 1 FROM t4 WHERE c=a) +} { + SEARCH TABLE t3 USING COVERING INDEX t3anc +} { + five four one two +} + +do_execsql_eqp_test 2.7 { + SELECT a FROM t3 WHERE EXISTS (SELECT 1 FROM t4 WHERE a=c) +} { + SEARCH TABLE t3 USING PRIMARY KEY +} { + four one +} + + finish_test + From a828d565b6b1b0fa6f97175f3d7934100a69b505 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Jan 2021 16:37:32 +0000 Subject: [PATCH 13/30] Update header comments for routines added by this branch. FossilOrigin-Name: 950030d679933f9ccd2b86ee650a4a78d338278a3629f0d289cca720a43e686b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/whereexpr.c | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 1523531c03..7005bdf9f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthe\sEXISTS->IN\stransformation\spreserves\sthe\scollation\ssequence\sof\sthe\scomparison\soperation. -D 2021-01-15T15:32:09.587 +C Update\sheader\scomments\sfor\sroutines\sadded\sby\sthis\sbranch. +D 2021-01-15T16:37:32.495 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -630,7 +630,7 @@ F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c F src/where.c 0e6abb22a2323fec80b450825593c26a2ad8f4815d1ee3af9969d8f6144bf681 F src/whereInt.h 9a3f577619f07700d16d89eeb2f3d94d6b7ed7f109c2dacf0ce8844921549506 F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee -F src/whereexpr.c d63dcb44253f7ad67449759bca6da63cac1c352aba7bea46ba3210154ad21e22 +F src/whereexpr.c 0462a66ccabeea05c2d41a1e650a9c4db5bc696b9a3b42b48a43a89a2ba81474 F src/window.c edd6f5e25a1e8f2b6f5305b7f5f7da7bb35f07f0d432b255b1d4c2fcab4205aa F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -1896,7 +1896,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8ce3cb90965771530c0021173d98720fc4c76bb99e69f7a879f80471dea0aace -R bc5d3e0052171a4a3fcfe5fb2a3891b3 +P a373baae12c914e48fd84de77998e301fdd3da43b06b9d64ac24a14418ed48cd +R e59675eaf8d9cef929f4570c41e8d7b0 U dan -Z 4e48cccd8bd31a2f422fbd932b326f93 +Z 99df7f39e80917b2e7ef225da7aeef0d diff --git a/manifest.uuid b/manifest.uuid index 58184acaa4..a3d3997a3a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a373baae12c914e48fd84de77998e301fdd3da43b06b9d64ac24a14418ed48cd \ No newline at end of file +950030d679933f9ccd2b86ee650a4a78d338278a3629f0d289cca720a43e686b \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 82a9759f39..cdaaa7e894 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1054,6 +1054,10 @@ static int exprUsesSrclist(SrcList *pSrc, Expr *pExpr, int bUses){ return (sqlite3WalkExpr(&sWalker, pExpr)==WRC_Abort); } +/* +** Context object used by exprExistsToInIter() as it iterates through an +** expression tree. +*/ struct ExistsToInCtx { SrcList *pSrc; Expr *pInLhs; @@ -1062,6 +1066,15 @@ struct ExistsToInCtx { Expr **ppParent; }; +/* +** Iterate through all AND connected nodes in the expression tree +** headed by (*ppExpr), populating the structure passed as the first +** argument with the values required by exprAnalyzeExistsFindEq(). +** +** This function returns non-zero if the expression tree does not meet +** the two conditions described by the header comment for +** exprAnalyzeExistsFindEq(), or zero if it does. +*/ static int exprExistsToInIter(struct ExistsToInCtx *p, Expr **ppExpr){ Expr *pExpr = *ppExpr; switch( pExpr->op ){ @@ -1093,6 +1106,30 @@ static int exprExistsToInIter(struct ExistsToInCtx *p, Expr **ppExpr){ return 0; } +/* +** This function is by exprAnalyzeExists() when creating virtual IN(...) +** terms equivalent to user-supplied EXIST(...) clauses. It splits the WHERE +** clause of the Select object passed as the first argument into one or more +** expressions joined by AND operators, and then tests if the following are +** true: +** +** 1. Exactly one of the AND separated terms refers to the outer +** query, and it is an == (TK_EQ) expression. +** +** 2. Only one side of the == expression refers to the outer query, and +** it does not refer to any columns from the inner query. +** +** If both these conditions are true, then a pointer to the side of the == +** expression that refers to the outer query is returned. The caller will +** use this expression as the LHS of the IN(...) virtual term. Or, if one +** or both of the above conditions are not true, NULL is returned. +** +** If non-NULL is returned and ppEq is non-NULL, *ppEq is set to point +** to the == expression node before returning. If pppAnd is non-NULL and +** the == node is not the root of the WHERE clause, then *pppAnd is set +** to point to the pointer to the AND node that is the parent of the == +** node within the WHERE expression tree. +*/ static Expr *exprAnalyzeExistsFindEq( Select *pSel, Expr **ppEq, /* OUT: == node from WHERE clause */ From f7588d4072453c050169f549ce06ac3cc6b259ff Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Jan 2021 17:51:56 +0000 Subject: [PATCH 14/30] Add OOM injection tests for new code on this branch. FossilOrigin-Name: 9a181dbaedcc2117e670e679ca94ed6d1fabd90c835671dee36424dd0646c4e5 --- manifest | 11 ++++----- manifest.uuid | 2 +- test/existsfault.test | 52 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 test/existsfault.test diff --git a/manifest b/manifest index 7005bdf9f6..a50135c18f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sheader\scomments\sfor\sroutines\sadded\sby\sthis\sbranch. -D 2021-01-15T16:37:32.495 +C Add\sOOM\sinjection\stests\sfor\snew\scode\son\sthis\sbranch. +D 2021-01-15T17:51:56.060 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -875,6 +875,7 @@ F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac F test/exists2.test 1022fe1c85ee162ae55efda6c00497b29d1f61ecb21e7d5f1c014eb46245c70a +F test/existsfault.test 74f7edc713f5a335e7ff47adf503067bf05c6f8630f88b2a19c24f0fa5486ab8 F test/expr.test 26cd01e8485bc48c8aa6a1add598e9ce1e706b4eb4f3f554e0b0223022e8c2cf F test/expr2.test c27327ae9c017a7ff6280123f67aff496f912da74d78c888926d68b46ec75fd8 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9 @@ -1896,7 +1897,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a373baae12c914e48fd84de77998e301fdd3da43b06b9d64ac24a14418ed48cd -R e59675eaf8d9cef929f4570c41e8d7b0 +P 950030d679933f9ccd2b86ee650a4a78d338278a3629f0d289cca720a43e686b +R 9a769128390cc86c331540163e8fa29c U dan -Z 99df7f39e80917b2e7ef225da7aeef0d +Z a3f9abfbc145e44a2a1bba02d43a960d diff --git a/manifest.uuid b/manifest.uuid index a3d3997a3a..7877c79580 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -950030d679933f9ccd2b86ee650a4a78d338278a3629f0d289cca720a43e686b \ No newline at end of file +9a181dbaedcc2117e670e679ca94ed6d1fabd90c835671dee36424dd0646c4e5 \ No newline at end of file diff --git a/test/existsfault.test b/test/existsfault.test new file mode 100644 index 0000000000..24e44a7b58 --- /dev/null +++ b/test/existsfault.test @@ -0,0 +1,52 @@ +# 2021 January 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing cases where EXISTS expressions are +# transformed to IN() expressions by where.c +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix existsfault + +do_execsql_test 1 { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + INSERT INTO t1 VALUES(3, 'three'); + INSERT INTO t1 VALUES(4, 'four'); + INSERT INTO t1 VALUES(5, 'five'); + INSERT INTO t1 VALUES(6, 'six'); + INSERT INTO t1 VALUES(7, 'seven'); + + CREATE TABLE t2(c INTEGER, d INTEGER); + INSERT INTO t2 VALUES(1, 1); + INSERT INTO t2 VALUES(3, 2); + INSERT INTO t2 VALUES(5, 3); + INSERT INTO t2 VALUES(7, 4); +} +faultsim_save_and_close + +do_faultsim_test 1 -prep { + faultsim_restore_and_reopen +} -body { + execsql { + SELECT t1.* FROM t1 WHERE EXISTS( + SELECT * FROM t2 WHERE t2.c=t1.a AND d IN (1, 2, 3) + ) + } +} -test { + faultsim_test_result {0 {1 one 3 three 5 five}} +} + + +finish_test + From 1e2896ec6e47c6f5a3bf3920c241014e6c47fcab Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 16 Jan 2021 12:15:41 +0000 Subject: [PATCH 15/30] Fix a hyperlink in the Lemon documentation. FossilOrigin-Name: 2ffb2ffa0ea147edd88632d2bbe29cc1d66d0911ce8e1068c406c81dd5a20242 --- doc/lemon.html | 4 ++-- manifest | 13 ++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/doc/lemon.html b/doc/lemon.html index bd078c8ec7..457004627e 100644 --- a/doc/lemon.html +++ b/doc/lemon.html @@ -1077,7 +1077,7 @@ can choose a different start symbol using the

4.4.19 The %syntax_error directive

-

See Error Processing.

+

See Error Processing.

4.4.20 The %token_class directive

@@ -1176,7 +1176,7 @@ match any input token.

the wildcard token and some other token, the other token is always used. The wildcard token is only matched if there are no alternatives.

- +

5.0 Error Processing

After extensive experimentation over several years, it has been diff --git a/manifest b/manifest index c00d0af4e9..1bbd748b03 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\smin/max\soptimization.\s\sFix\sfor\sa\sperformance\nregression\sintroduced\sat\s[b8ba2f17f938c035]\sreported\sby\n[forum:/forumpost/4050026ab8|forum\spost\s4050026ab8] -D 2021-01-14T20:57:47.228 +C Fix\sa\shyperlink\sin\sthe\sLemon\sdocumentation. +D 2021-01-16T12:15:41.192 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -38,7 +38,7 @@ F configure 91893a81f698778dda4d8fb24bfca606ded31ef02bcfbc2ab072d30fb67138d6 x F configure.ac 412b65c6107e41c098ad7f5f2e6a3f74ac02ffc6e92b9a6264b9f1060c235a04 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd -F doc/lemon.html c5d8ba85ac1daef7be8c2d389899480eb62451ff5c09b0c28ff8157bb8770746 +F doc/lemon.html 1bb72ece6271df0d901d233551dd985f2c6ba30d09382cf2d321ed951ab57491 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 F doc/trusted-schema.md 33625008620e879c7bcfbbfa079587612c434fa094d338b08242288d358c3e8a F doc/vdbesort-memory.md 4da2639c14cd24a31e0af694b1a8dd37eaf277aff3867e9a8cc14046bc49df56 @@ -1895,8 +1895,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 11e4eb095746602961a178044809a68a77ba7b367596997bef726e54062423d9 ccd3bae14b6b47bb0f9622700c04db989f76ce65e10e0709964cfd0675eca762 -R 3dc0d948d76c5e90d01be3a954f85921 -T +closed ccd3bae14b6b47bb0f9622700c04db989f76ce65e10e0709964cfd0675eca762 +P 249a71cc6822d6bdd5bb9e727aac81c6549693b418e9c0987b96850ee332c940 +R 0d3b2c6d616a91b857e305e5fa92f92c U drh -Z 8ad13e36552647d99e513fed14d21def +Z 3e8f8636c650d40fd45e0f7dddeb2d81 diff --git a/manifest.uuid b/manifest.uuid index 6ee0559a9e..ffa68fe850 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -249a71cc6822d6bdd5bb9e727aac81c6549693b418e9c0987b96850ee332c940 \ No newline at end of file +2ffb2ffa0ea147edd88632d2bbe29cc1d66d0911ce8e1068c406c81dd5a20242 \ No newline at end of file From 2a3be742caf8c35330038ce839f9a735e847045e Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 16 Jan 2021 18:22:10 +0000 Subject: [PATCH 16/30] Add debugging output about the EXISTS-to-IN optimization when the ".wheretrace" flag has the 0x20 bit set. FossilOrigin-Name: 0dad5ce34ad8a59200b013453c9334f8898e07f2c0107c8c734ecc34b67de572 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/whereexpr.c | 10 +++++++++- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a50135c18f..f5d05c712c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sOOM\sinjection\stests\sfor\snew\scode\son\sthis\sbranch. -D 2021-01-15T17:51:56.060 +C Add\sdebugging\soutput\sabout\sthe\sEXISTS-to-IN\soptimization\swhen\nthe\s".wheretrace"\sflag\shas\sthe\s0x20\sbit\sset. +D 2021-01-16T18:22:10.163 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -630,7 +630,7 @@ F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c F src/where.c 0e6abb22a2323fec80b450825593c26a2ad8f4815d1ee3af9969d8f6144bf681 F src/whereInt.h 9a3f577619f07700d16d89eeb2f3d94d6b7ed7f109c2dacf0ce8844921549506 F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee -F src/whereexpr.c 0462a66ccabeea05c2d41a1e650a9c4db5bc696b9a3b42b48a43a89a2ba81474 +F src/whereexpr.c 10ed78eb09474c4f4d2ad61c3047900897d48ce969f2fa7ef48ffd28b6d934a6 F src/window.c edd6f5e25a1e8f2b6f5305b7f5f7da7bb35f07f0d432b255b1d4c2fcab4205aa F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -1897,7 +1897,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 950030d679933f9ccd2b86ee650a4a78d338278a3629f0d289cca720a43e686b -R 9a769128390cc86c331540163e8fa29c -U dan -Z a3f9abfbc145e44a2a1bba02d43a960d +P 9a181dbaedcc2117e670e679ca94ed6d1fabd90c835671dee36424dd0646c4e5 +R 37544a92998410ad6bf700343ed335b0 +U drh +Z b455d8286b10f237c30679d2b4a9d690 diff --git a/manifest.uuid b/manifest.uuid index 7877c79580..c0bab29af4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a181dbaedcc2117e670e679ca94ed6d1fabd90c835671dee36424dd0646c4e5 \ No newline at end of file +0dad5ce34ad8a59200b013453c9334f8898e07f2c0107c8c734ecc34b67de572 \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index cdaaa7e894..1772aa7486 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1107,7 +1107,7 @@ static int exprExistsToInIter(struct ExistsToInCtx *p, Expr **ppExpr){ } /* -** This function is by exprAnalyzeExists() when creating virtual IN(...) +** This function is used by exprAnalyzeExists() when creating virtual IN(...) ** terms equivalent to user-supplied EXIST(...) clauses. It splits the WHERE ** clause of the Select object passed as the first argument into one or more ** expressions joined by AND operators, and then tests if the following are @@ -1239,6 +1239,14 @@ static void exprAnalyzeExists( } sqlite3ExprDelete(db, pEq); +#ifdef WHERETRACE_ENABLED /* 0x20 */ + if( sqlite3WhereTrace & 0x20 ){ + sqlite3DebugPrintf("Convert EXISTS:\n"); + sqlite3TreeViewExpr(0, pExpr, 0); + sqlite3DebugPrintf("into IN:\n"); + sqlite3TreeViewExpr(0, pDup, 0); + } +#endif idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); if( idxNew ){ exprAnalyze(pSrc, pWC, idxNew); From 4be8bdccd453440e653f04bb6da4dcecf4bdfa68 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 16 Jan 2021 18:55:10 +0000 Subject: [PATCH 17/30] Give the EXISTS-to-IN optimization the ability to handle some cases that involve vector comparisons, instead of throwing a mysterious error in those cases. FossilOrigin-Name: 87e78a19bb3ae1caf57aeeae53a5ab4efdccb57265f25d5c19b62eae53747aff --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/whereexpr.c | 14 +++++++++++--- test/exists2.test | 19 ++++++++++++++++++- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index f5d05c712c..995f5e28d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sdebugging\soutput\sabout\sthe\sEXISTS-to-IN\soptimization\swhen\nthe\s".wheretrace"\sflag\shas\sthe\s0x20\sbit\sset. -D 2021-01-16T18:22:10.163 +C Give\sthe\sEXISTS-to-IN\soptimization\sthe\sability\sto\shandle\ssome\scases\sthat\ninvolve\svector\scomparisons,\sinstead\sof\sthrowing\sa\smysterious\serror\sin\sthose\ncases. +D 2021-01-16T18:55:10.186 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -630,7 +630,7 @@ F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c F src/where.c 0e6abb22a2323fec80b450825593c26a2ad8f4815d1ee3af9969d8f6144bf681 F src/whereInt.h 9a3f577619f07700d16d89eeb2f3d94d6b7ed7f109c2dacf0ce8844921549506 F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee -F src/whereexpr.c 10ed78eb09474c4f4d2ad61c3047900897d48ce969f2fa7ef48ffd28b6d934a6 +F src/whereexpr.c 0a3fd3667cb78d5ced831d103a9769e9429b69cc06336a557f7436a36298a08d F src/window.c edd6f5e25a1e8f2b6f5305b7f5f7da7bb35f07f0d432b255b1d4c2fcab4205aa F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -874,7 +874,7 @@ F test/exclusive.test 7ff63be7503990921838d5c9f77f6e33e68e48ed1a9d48cd28745bf650 F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac -F test/exists2.test 1022fe1c85ee162ae55efda6c00497b29d1f61ecb21e7d5f1c014eb46245c70a +F test/exists2.test 3e6741849880410a438e1e016878aee518e82fc14c9df7bb8b975414e822960a F test/existsfault.test 74f7edc713f5a335e7ff47adf503067bf05c6f8630f88b2a19c24f0fa5486ab8 F test/expr.test 26cd01e8485bc48c8aa6a1add598e9ce1e706b4eb4f3f554e0b0223022e8c2cf F test/expr2.test c27327ae9c017a7ff6280123f67aff496f912da74d78c888926d68b46ec75fd8 @@ -1897,7 +1897,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9a181dbaedcc2117e670e679ca94ed6d1fabd90c835671dee36424dd0646c4e5 -R 37544a92998410ad6bf700343ed335b0 +P 0dad5ce34ad8a59200b013453c9334f8898e07f2c0107c8c734ecc34b67de572 +R dac9991b639e128a44072e06099f9d5a U drh -Z b455d8286b10f237c30679d2b4a9d690 +Z 4c2faa19ecfc2a7e1d7bc30b27d619e2 diff --git a/manifest.uuid b/manifest.uuid index c0bab29af4..b027e9f246 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0dad5ce34ad8a59200b013453c9334f8898e07f2c0107c8c734ecc34b67de572 \ No newline at end of file +87e78a19bb3ae1caf57aeeae53a5ab4efdccb57265f25d5c19b62eae53747aff \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 1772aa7486..e3c02deacd 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1215,16 +1215,24 @@ static void exprAnalyzeExists( if( pInLhs==pEq->pLeft ){ pRet = pEq->pRight; }else{ - CollSeq *p = sqlite3ExprCompareCollSeq(pParse, pEq); - pInLhs = sqlite3ExprAddCollateString(pParse, pInLhs, p?p->zName:"BINARY"); pRet = pEq->pLeft; + if( pRet->op!=TK_VECTOR ){ + CollSeq *p = sqlite3ExprCompareCollSeq(pParse, pEq); + pInLhs = sqlite3ExprAddCollateString(pParse, pInLhs, p?p->zName:"BINARY"); + } } assert( pDup->pLeft==0 ); pDup->op = TK_IN; pDup->pLeft = pInLhs; pDup->flags &= ~EP_VarSelect; - pSel->pEList = sqlite3ExprListAppend(pParse, 0, pRet); + if( pRet->op==TK_VECTOR ){ + pSel->pEList = pRet->x.pList; + pRet->x.pList = 0; + sqlite3ExprDelete(db, pRet); + }else{ + pSel->pEList = sqlite3ExprListAppend(pParse, 0, pRet); + } pEq->pLeft = 0; pEq->pRight = 0; if( ppAnd ){ diff --git a/test/exists2.test b/test/exists2.test index 3472751a21..a28e5daaef 100644 --- a/test/exists2.test +++ b/test/exists2.test @@ -156,8 +156,25 @@ do_execsql_eqp_test 2.7 { four one } +# EXISTS clauses using vector expressions in the WHERE clause. +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a,b); + INSERT INTO t1(a,b) VALUES(1,111),(2,222),(8,888); + CREATE TABLE t2(x INTEGER PRIMARY KEY, y); + INSERT INTO t2(x,y) VALUES(2,222),(3,333),(7,333); + SELECT y FROM t2 WHERE EXISTS(SELECT 1 FROM t1 WHERE (x,y)=(a,b)); +} {222} +do_execsql_test 3.1 { + SELECT y FROM t2 WHERE EXISTS(SELECT 1 FROM t1 WHERE (a,b)=(x,y)); +} {222} +do_execsql_test 3.2 { + SELECT y FROM t2 WHERE EXISTS(SELECT 1 FROM t1 WHERE (x,b)=(a,y)); +} {222} + + finish_test - From 9ffa258a01ff64cce868f452164a001e702561b5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 16 Jan 2021 20:22:11 +0000 Subject: [PATCH 18/30] Improved handling of vector equalities in the EXISTS-to-IN translator. FossilOrigin-Name: ef49ee4a3766146963bfb6b013472f9836afb9c5b0d21a8533871cf961139e38 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 13 ++++++++++++- src/whereexpr.c | 6 ++---- test/exists2.test | 26 +++++++++++++++++--------- 5 files changed, 40 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 995f5e28d2..2755d47c82 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Give\sthe\sEXISTS-to-IN\soptimization\sthe\sability\sto\shandle\ssome\scases\sthat\ninvolve\svector\scomparisons,\sinstead\sof\sthrowing\sa\smysterious\serror\sin\sthose\ncases. -D 2021-01-16T18:55:10.186 +C Improved\shandling\sof\svector\sequalities\sin\sthe\sEXISTS-to-IN\stranslator. +D 2021-01-16T20:22:11.928 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -492,7 +492,7 @@ F src/date.c dace306a10d9b02ee553d454c8e1cf8d3c9b932e137738a6b15b90253a9bfc10 F src/dbpage.c 8a01e865bf8bc6d7b1844b4314443a6436c07c3efe1d488ed89e81719047833a F src/dbstat.c 3aa79fc3aed7ce906e4ea6c10e85d657299e304f6049861fe300053ac57de36c F src/delete.c 927cf8f900583e79aca8f1a321979e0a8f053babd9a690b44b38f79de2cc09fe -F src/expr.c 0d196ed5a2ebf96be7e8df88add4fabfad0dce16c0fed81a4b8f6a26e259797f +F src/expr.c 47c85263e6d179424e6b09e2c79db5704ab5b8cbc2fae2ee3285faa2566f2e74 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 83372403298e6a7dd989a47aaacdbaa5b4307b5199dbd56e07d4896066b3de72 F src/func.c 251b5953cecd0ce3e282213c5e623134415793d3569d7804d13460559d7e45ff @@ -630,7 +630,7 @@ F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c F src/where.c 0e6abb22a2323fec80b450825593c26a2ad8f4815d1ee3af9969d8f6144bf681 F src/whereInt.h 9a3f577619f07700d16d89eeb2f3d94d6b7ed7f109c2dacf0ce8844921549506 F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee -F src/whereexpr.c 0a3fd3667cb78d5ced831d103a9769e9429b69cc06336a557f7436a36298a08d +F src/whereexpr.c 9fdbed19035077e41ca993ab469114e9b6b0746c18ea4e7675a850713b35fc72 F src/window.c edd6f5e25a1e8f2b6f5305b7f5f7da7bb35f07f0d432b255b1d4c2fcab4205aa F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -874,7 +874,7 @@ F test/exclusive.test 7ff63be7503990921838d5c9f77f6e33e68e48ed1a9d48cd28745bf650 F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac -F test/exists2.test 3e6741849880410a438e1e016878aee518e82fc14c9df7bb8b975414e822960a +F test/exists2.test 3e5726d6a67ebd4bd3466db58be424c09156c1f276c7594abb260cbf6ad494d3 F test/existsfault.test 74f7edc713f5a335e7ff47adf503067bf05c6f8630f88b2a19c24f0fa5486ab8 F test/expr.test 26cd01e8485bc48c8aa6a1add598e9ce1e706b4eb4f3f554e0b0223022e8c2cf F test/expr2.test c27327ae9c017a7ff6280123f67aff496f912da74d78c888926d68b46ec75fd8 @@ -1897,7 +1897,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0dad5ce34ad8a59200b013453c9334f8898e07f2c0107c8c734ecc34b67de572 -R dac9991b639e128a44072e06099f9d5a +P 87e78a19bb3ae1caf57aeeae53a5ab4efdccb57265f25d5c19b62eae53747aff +R 9040bb485a8bcb3beab221382ed76aaa U drh -Z 4c2faa19ecfc2a7e1d7bc30b27d619e2 +Z 15d0a474b0220437a1e81a3ea0fc01b3 diff --git a/manifest.uuid b/manifest.uuid index b027e9f246..773b82c6ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -87e78a19bb3ae1caf57aeeae53a5ab4efdccb57265f25d5c19b62eae53747aff \ No newline at end of file +ef49ee4a3766146963bfb6b013472f9836afb9c5b0d21a8533871cf961139e38 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 685f041752..f225b59bce 100644 --- a/src/expr.c +++ b/src/expr.c @@ -95,7 +95,18 @@ Expr *sqlite3ExprAddCollateToken( const Token *pCollName, /* Name of collating sequence */ int dequote /* True to dequote pCollName */ ){ - if( pCollName->n>0 ){ + assert( pExpr!=0 || pParse->db->mallocFailed ); + if( pExpr==0 ) return 0; + if( pExpr->op==TK_VECTOR ){ + ExprList *pList = pExpr->x.pList; + if( ALWAYS(pList!=0) ){ + int i; + for(i=0; inExpr; i++){ + pList->a[i].pExpr = sqlite3ExprAddCollateToken(pParse,pList->a[i].pExpr, + pCollName, dequote); + } + } + }else if( pCollName->n>0 ){ Expr *pNew = sqlite3ExprAlloc(pParse->db, TK_COLLATE, pCollName, dequote); if( pNew ){ pNew->pLeft = pExpr; diff --git a/src/whereexpr.c b/src/whereexpr.c index e3c02deacd..9029ff1810 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1215,11 +1215,9 @@ static void exprAnalyzeExists( if( pInLhs==pEq->pLeft ){ pRet = pEq->pRight; }else{ + CollSeq *p = sqlite3ExprCompareCollSeq(pParse, pEq); + pInLhs = sqlite3ExprAddCollateString(pParse, pInLhs, p?p->zName:"BINARY"); pRet = pEq->pLeft; - if( pRet->op!=TK_VECTOR ){ - CollSeq *p = sqlite3ExprCompareCollSeq(pParse, pEq); - pInLhs = sqlite3ExprAddCollateString(pParse, pInLhs, p?p->zName:"BINARY"); - } } assert( pDup->pLeft==0 ); diff --git a/test/exists2.test b/test/exists2.test index a28e5daaef..c4fc9dc373 100644 --- a/test/exists2.test +++ b/test/exists2.test @@ -105,16 +105,16 @@ do_execsql_eqp_test 1.7 { # reset_db do_execsql_test 2.0 { - CREATE TABLE t3(a TEXT PRIMARY KEY, b TEXT) WITHOUT ROWID; - CREATE TABLE t4(c TEXT COLLATE nocase); + CREATE TABLE t3(a TEXT PRIMARY KEY, b TEXT, x INT) WITHOUT ROWID; + CREATE TABLE t4(c TEXT COLLATE nocase, y INT); - INSERT INTO t3 VALUES('one', 'i'); - INSERT INTO t3 VALUES('two', 'ii'); - INSERT INTO t3 VALUES('three', 'iii'); - INSERT INTO t3 VALUES('four', 'iv'); - INSERT INTO t3 VALUES('five', 'v'); + INSERT INTO t3 VALUES('one', 'i', 1); + INSERT INTO t3 VALUES('two', 'ii', 2); + INSERT INTO t3 VALUES('three', 'iii', 3); + INSERT INTO t3 VALUES('four', 'iv', 4); + INSERT INTO t3 VALUES('five', 'v', 5); - INSERT INTO t4 VALUES('FIVE'), ('four'), ('TWO'), ('one'); + INSERT INTO t4 VALUES('FIVE',5), ('four',4), ('TWO',2), ('one',1); } do_execsql_test 2.1 { SELECT a FROM t3, t4 WHERE a=c } {four one} @@ -137,7 +137,7 @@ do_execsql_eqp_test 2.4 { } do_execsql_test 2.5 { - CREATE INDEX t3anc ON t3(a COLLATE nocase); + CREATE INDEX t3anc ON t3(a COLLATE nocase, x); } do_execsql_eqp_test 2.6 { @@ -147,6 +147,9 @@ do_execsql_eqp_test 2.6 { } { five four one two } +do_execsql_test 2.6a { + SELECT a FROM t3 WHERE EXISTS (SELECT 1 FROM t4 WHERE (c,y)=(a,x)) +} {five four one two} do_execsql_eqp_test 2.7 { SELECT a FROM t3 WHERE EXISTS (SELECT 1 FROM t4 WHERE a=c) @@ -155,6 +158,11 @@ do_execsql_eqp_test 2.7 { } { four one } +do_execsql_test 2.7a { + SELECT a FROM t3 WHERE EXISTS (SELECT 1 FROM t4 WHERE (a,x)=(c,y)) +} { + four one +} # EXISTS clauses using vector expressions in the WHERE clause. # From 9fcc8c69bbf0af6da7f3e767ddadd1aa7bf28bbe Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 17 Jan 2021 00:13:12 +0000 Subject: [PATCH 19/30] Minor simplification of the EXISTS-to-IN logic. FossilOrigin-Name: cac90a9f4ab0a8f3ff77ee1f8549213c2f97169fc3469e55d57caa564079ce2a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/whereexpr.c | 8 +++----- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 2755d47c82..51ebfdca8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\shandling\sof\svector\sequalities\sin\sthe\sEXISTS-to-IN\stranslator. -D 2021-01-16T20:22:11.928 +C Minor\ssimplification\sof\sthe\sEXISTS-to-IN\slogic. +D 2021-01-17T00:13:12.097 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -630,7 +630,7 @@ F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c F src/where.c 0e6abb22a2323fec80b450825593c26a2ad8f4815d1ee3af9969d8f6144bf681 F src/whereInt.h 9a3f577619f07700d16d89eeb2f3d94d6b7ed7f109c2dacf0ce8844921549506 F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee -F src/whereexpr.c 9fdbed19035077e41ca993ab469114e9b6b0746c18ea4e7675a850713b35fc72 +F src/whereexpr.c 9886a16b52af66e01ded9e69e3e2913f25d8e34bdeaa5e583d7c8c9a1ffe71d2 F src/window.c edd6f5e25a1e8f2b6f5305b7f5f7da7bb35f07f0d432b255b1d4c2fcab4205aa F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -1897,7 +1897,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 87e78a19bb3ae1caf57aeeae53a5ab4efdccb57265f25d5c19b62eae53747aff -R 9040bb485a8bcb3beab221382ed76aaa +P ef49ee4a3766146963bfb6b013472f9836afb9c5b0d21a8533871cf961139e38 +R ed5d5749a7ed1103ef3bdb4ce2a908bc U drh -Z 15d0a474b0220437a1e81a3ea0fc01b3 +Z beb39ea924321eafaa423b5e3b964772 diff --git a/manifest.uuid b/manifest.uuid index 773b82c6ef..7186b19c6e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef49ee4a3766146963bfb6b013472f9836afb9c5b0d21a8533871cf961139e38 \ No newline at end of file +cac90a9f4ab0a8f3ff77ee1f8549213c2f97169fc3469e55d57caa564079ce2a \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 9029ff1810..584d2d35f5 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1254,11 +1254,9 @@ static void exprAnalyzeExists( } #endif idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); - if( idxNew ){ - exprAnalyze(pSrc, pWC, idxNew); - markTermAsChild(pWC, idxNew, idxTerm); - pWC->a[idxTerm].wtFlags |= TERM_COPIED; - } + exprAnalyze(pSrc, pWC, idxNew); + markTermAsChild(pWC, idxNew, idxTerm); + pWC->a[idxTerm].wtFlags |= TERM_COPIED; } /* From 06afa291cfc887c7a5592542c770184db6732814 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 18 Jan 2021 00:11:20 +0000 Subject: [PATCH 20/30] More comments on the EXISTS-to-IN optimization logic. FossilOrigin-Name: 92cc29099f796f5f244dd80ee431c48d36d01eaece6f150119ead5ecd14eaae1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/whereexpr.c | 37 +++++++++++++++++++++++++------------ 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 51ebfdca8d..c51fc6703b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\ssimplification\sof\sthe\sEXISTS-to-IN\slogic. -D 2021-01-17T00:13:12.097 +C More\scomments\son\sthe\sEXISTS-to-IN\soptimization\slogic. +D 2021-01-18T00:11:20.813 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -630,7 +630,7 @@ F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c F src/where.c 0e6abb22a2323fec80b450825593c26a2ad8f4815d1ee3af9969d8f6144bf681 F src/whereInt.h 9a3f577619f07700d16d89eeb2f3d94d6b7ed7f109c2dacf0ce8844921549506 F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee -F src/whereexpr.c 9886a16b52af66e01ded9e69e3e2913f25d8e34bdeaa5e583d7c8c9a1ffe71d2 +F src/whereexpr.c a022b4f447c0fb0674e172a9a303537ea5055df60d579f99cec7ead18e8c453f F src/window.c edd6f5e25a1e8f2b6f5305b7f5f7da7bb35f07f0d432b255b1d4c2fcab4205aa F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -1897,7 +1897,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ef49ee4a3766146963bfb6b013472f9836afb9c5b0d21a8533871cf961139e38 -R ed5d5749a7ed1103ef3bdb4ce2a908bc +P cac90a9f4ab0a8f3ff77ee1f8549213c2f97169fc3469e55d57caa564079ce2a +R f533358ab2d88dffa58abcd3bcdbe4bd U drh -Z beb39ea924321eafaa423b5e3b964772 +Z 7271ac9ea9a27eaec7cf5d0a637bfec4 diff --git a/manifest.uuid b/manifest.uuid index 7186b19c6e..689f037d90 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cac90a9f4ab0a8f3ff77ee1f8549213c2f97169fc3469e55d57caa564079ce2a \ No newline at end of file +92cc29099f796f5f244dd80ee431c48d36d01eaece6f150119ead5ecd14eaae1 \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 584d2d35f5..5cfdad67c1 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1036,13 +1036,26 @@ static int exprUsesSrclistSelectCb(Walker *p, Select *pSelect){ ** This function always returns true if expression pExpr contains ** a sub-select. ** -** If there is no sub-select and bUses is 1, then true is returned -** if the expression contains at least one TK_COLUMN node that refers -** to a table in pSrc. +** If there is no sub-select in pExpr, then return true if pExpr +** contains a TK_COLUMN node for a table that is (bUses==1) +** or is not (bUses==0) in pSrc. ** -** Or, if there is no sub-select and bUses is 0, then true is returned -** if the expression contains at least one TK_COLUMN node that refers -** to a table that is not in pSrc. +** Said another way: +** +** bUses Return Meaning +** -------- ------ ------------------------------------------------ +** +** bUses==1 true pExpr contains either a sub-select or a +** TK_COLUMN referencing pSrc. +** +** bUses==1 false pExpr contains no sub-selects and all TK_COLUMN +** nodes reference tables not found in pSrc +** +** bUses==0 true pExpr contains either a sub-select or a TK_COLUMN +** that references a table not in pSrc. +** +** bUses==0 false pExpr contains no sub-selects and all TK_COLUMN +** nodes reference pSrc */ static int exprUsesSrclist(SrcList *pSrc, Expr *pExpr, int bUses){ Walker sWalker; @@ -1059,11 +1072,11 @@ static int exprUsesSrclist(SrcList *pSrc, Expr *pExpr, int bUses){ ** expression tree. */ struct ExistsToInCtx { - SrcList *pSrc; - Expr *pInLhs; - Expr *pEq; - Expr **ppAnd; - Expr **ppParent; + SrcList *pSrc; /* The tables in an EXISTS(SELECT ... FROM ...) */ + Expr *pInLhs; /* OUT: Use this as the LHS of the IN operator */ + Expr *pEq; /* OUT: The == term that include pInLhs */ + Expr **ppAnd; /* OUT: The AND operator that includes pEq as a child */ + Expr **ppParent; /* The AND operator currently being examined */ }; /* @@ -1131,7 +1144,7 @@ static int exprExistsToInIter(struct ExistsToInCtx *p, Expr **ppExpr){ ** node within the WHERE expression tree. */ static Expr *exprAnalyzeExistsFindEq( - Select *pSel, + Select *pSel, /* The SELECT of the EXISTS */ Expr **ppEq, /* OUT: == node from WHERE clause */ Expr ***pppAnd /* OUT: Pointer to parent of ==, if any */ ){ From f3ebea8114feb2798ffd2f12dd1b9a2a1b0d418f Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 18 Jan 2021 19:27:56 +0000 Subject: [PATCH 21/30] Update test helper procedure 'get_pwd' to handle the ComSpec environment variable being absent. FossilOrigin-Name: fe1979552f43e0526f16481457e01981f29707401f77079f9854a8d91b35b5a4 --- manifest | 17 ++++++++--------- manifest.uuid | 2 +- test/crash5.test | 10 ++++++++-- test/tester.tcl | 8 +++++++- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 36270a6a17..03ea998b51 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\squery\splanner\sso\sthat\sit\sis\sable\sto\scode\sEXISTS\soperators\sin\nthe\sWHERE\sclause\sas\sif\sthey\swere\sIN\soperators,\swhen\sappropriate. -D 2021-01-18T12:35:16.581 +C Update\stest\shelper\sprocedure\s'get_pwd'\sto\shandle\sthe\sComSpec\senvironment\svariable\sbeing\sabsent. +D 2021-01-18T19:27:56.252 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -800,7 +800,7 @@ F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f F test/crash2.test 5b14d4eb58b880e231361d3b609b216acda86651 F test/crash3.test 8f5de9d32ab9ab95475a9efe7f47a940aa889418 F test/crash4.test fe2821baf37168dc59dd733dcf7dba2a401487bc -F test/crash5.test 98f77ad22bceaea9043bf87088d5b61d004a1f40bb7c9dc3b6a7c70bd502c0bb +F test/crash5.test 4aa55e7ac3c4bc511873e457aa65d2827d52da9b51e061511899dadcfe22b1e8 F test/crash6.test 4c56f1e40d0291e1110790a99807aa875b1647ba F test/crash7.test 1a194c4900a255258cf94b7fcbfd29536db572df F test/crash8.test 64366e459c28dd62edfb7ad87253a409c7533b92d16fcc479a6a8131bdcc3100 @@ -1437,7 +1437,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test d2940417496e2b9548e01d09990763fbe88c316504033256d51493e1f1a5ce6a F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 56c059c88c5b96a624f1193ba48b0bac034190b79cd9c75cb4acbfe84baf7ec5 +F test/tester.tcl e5e4f5707fbf791ff8e06438fd0d4d71fe4c1d48753b7dd415efe72e853ef877 F test/thread001.test b61a29dd87cf669f5f6ac96124a7c97d71b0c80d9012746072055877055cf9ef F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1897,8 +1897,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2ffb2ffa0ea147edd88632d2bbe29cc1d66d0911ce8e1068c406c81dd5a20242 92cc29099f796f5f244dd80ee431c48d36d01eaece6f150119ead5ecd14eaae1 -R e9622ae2cc2f8f16e772e641d8b24f1c -T +closed 92cc29099f796f5f244dd80ee431c48d36d01eaece6f150119ead5ecd14eaae1 -U drh -Z f5a7eb36d087a773f1b509e693ef42b1 +P c1862abb44873f06ec0d772469d8a2d128ae4670b1e98c2d97b0e2da18df9a04 +R 3b894a27798e4da33503e4ea0791a039 +U mistachkin +Z 4d9421bedbee9a79d4853a9ab24d8edd diff --git a/manifest.uuid b/manifest.uuid index c22949f1d4..13ef47c004 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1862abb44873f06ec0d772469d8a2d128ae4670b1e98c2d97b0e2da18df9a04 \ No newline at end of file +fe1979552f43e0526f16481457e01981f29707401f77079f9854a8d91b35b5a4 \ No newline at end of file diff --git a/test/crash5.test b/test/crash5.test index 5c8e04c137..fc078b3504 100644 --- a/test/crash5.test +++ b/test/crash5.test @@ -21,7 +21,7 @@ source $testdir/tester.tcl # Only run these tests if memory debugging is turned on. # ifcapable !crashtest||!memorymanage { - puts "Skipping crash5 tests: not compiled with -DSQLITE_MEMDEBUG..." + puts "Skipping crash5 tests: not compiled with -DSQLITE_ENABLE_MEMORY_MANAGEMENT..." finish_test return } @@ -49,8 +49,14 @@ for {set ii 0} {$ii < 10} {incr ii} { [list set iFail $jj] { proc get_pwd {} { if {$::tcl_platform(platform) eq "windows"} { + if {[info exists ::env(ComSpec)]} { + set comSpec $::env(ComSpec) + } else { + # NOTE: Hard-code the typical default value. + set comSpec {C:\Windows\system32\cmd.exe} + } return [string map [list \\ /] \ - [string trim [exec -- $::env(ComSpec) /c echo %CD%]]] + [string trim [exec -- $comSpec /c echo %CD%]]] } else { return [pwd] } diff --git a/test/tester.tcl b/test/tester.tcl index 7ef059571a..5c72dccbb7 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -174,8 +174,14 @@ proc get_pwd {} { # case of the result to what Tcl considers canonical, which would # defeat the purpose of this procedure. # + if {[info exists ::env(ComSpec)]} { + set comSpec $::env(ComSpec) + } else { + # NOTE: Hard-code the typical default value. + set comSpec {C:\Windows\system32\cmd.exe} + } return [string map [list \\ /] \ - [string trim [exec -- $::env(ComSpec) /c echo %CD%]]] + [string trim [exec -- $comSpec /c echo %CD%]]] } else { return [pwd] } From d6665c51e2ed677aab53b2dc6504f1ea2270a951 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 18 Jan 2021 19:28:56 +0000 Subject: [PATCH 22/30] Fix harmless compiler warnings seen with MSVC. FossilOrigin-Name: dc7938d2d715301595dee2fac6880af3716c4b3d1cbe7c3578d7fd30ba146a23 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/insert.c | 2 ++ src/vdbe.c | 2 ++ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 03ea998b51..a604568c21 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\stest\shelper\sprocedure\s'get_pwd'\sto\shandle\sthe\sComSpec\senvironment\svariable\sbeing\sabsent. -D 2021-01-18T19:27:56.252 +C Fix\sharmless\scompiler\swarnings\sseen\swith\sMSVC. +D 2021-01-18T19:28:56.733 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -501,7 +501,7 @@ F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19 F src/hash.h 9d56a9079d523b648774c1784b74b89bd93fac7b365210157482e4319a468f38 F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c c5e0c25cfb9960d9b7d49043de6adc12748853bc6dea76f5adef059e366f2f70 +F src/insert.c 9b970eff058a858fbd9f2db71425ef195942c2610855daa66ae23024432d52f5 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 8c9c8cd2bd8eecdb06d9b6e89de7e9e65bae45cc8fc33609cc74023a5c296067 F src/main.c 1c5de7b3fabcdf05f4fe563aab5d81d175b89c67a8678a12ba86629356afa356 @@ -612,7 +612,7 @@ F src/upsert.c df8f1727d62b5987c4fd302cd4d7c0c84ae57cd65683c5a34a740dfe24039235 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 41c7a72da1df47864faa378a1c720b38adb288c6838cb6be5594511b6287a048 F src/vacuum.c 492422c1463c076473bae1858799c7a0a5fe87a133d1223239447c422cd26286 -F src/vdbe.c 67de20067fa3a2ee8566342c751e941a2fe3fd88940fd9886ca5115f04165cce +F src/vdbe.c fd3ae827dea9da118d22c172ad183888120070d4c709bd0a5a97af7749246783 F src/vdbe.h 83603854bfa5851af601fc0947671eb260f4363e62e960e8a994fb9bbcd2aaa1 F src/vdbeInt.h 3ca5e9fd6e095a8b6cf6bc3587a46fc93499503b2fe48951e1034ba9e2ce2f6e F src/vdbeapi.c c5e7cb2ab89a24d7f723e87b508f21bfb1359a04db5277d8a99fd1e015c12eb9 @@ -1897,7 +1897,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c1862abb44873f06ec0d772469d8a2d128ae4670b1e98c2d97b0e2da18df9a04 -R 3b894a27798e4da33503e4ea0791a039 +P fe1979552f43e0526f16481457e01981f29707401f77079f9854a8d91b35b5a4 +R dc7be2ccd019ce776f5c5401443272a0 U mistachkin -Z 4d9421bedbee9a79d4853a9ab24d8edd +Z dce87543665f3b6eb6e012303a5399fb diff --git a/manifest.uuid b/manifest.uuid index 13ef47c004..f83d261629 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fe1979552f43e0526f16481457e01981f29707401f77079f9854a8d91b35b5a4 \ No newline at end of file +dc7938d2d715301595dee2fac6880af3716c4b3d1cbe7c3578d7fd30ba146a23 \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index c0ab0cf37a..6047969c07 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1301,7 +1301,9 @@ void sqlite3Insert( sqlite3VdbeJumpHere(v, addrInsTop); } +#ifndef SQLITE_OMIT_XFER_OPT insert_end: +#endif /* SQLITE_OMIT_XFER_OPT */ /* Update the sqlite_sequence table by storing the content of the ** maximum rowid counter values recorded while inserting into ** autoincrement tables. diff --git a/src/vdbe.c b/src/vdbe.c index 8e107a8a56..bcfc5df8d9 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4909,8 +4909,10 @@ case OP_NewRowid: { /* out2 */ VdbeCursor *pC; /* Cursor of table to get the new rowid */ int res; /* Result of an sqlite3BtreeLast() */ int cnt; /* Counter to limit the number of searches */ +#ifndef SQLITE_OMIT_AUTOINCREMENT Mem *pMem; /* Register holding largest rowid for AUTOINCREMENT */ VdbeFrame *pFrame; /* Root frame of VDBE */ +#endif v = 0; res = 0; From a2560ce934ddef0ebebfea7702e2ad361b2df654 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Jan 2021 10:59:47 +0000 Subject: [PATCH 23/30] Add tests for sqlite3session_memory_used() interface. FossilOrigin-Name: 5596611f96f3401262b9dadc591bf7e3411d2c4a6f5be5cfe524e203d9820fd1 --- ext/session/sessionmem.test | 57 +++++++++++++++++++++++++++++++++++++ manifest | 13 +++++---- manifest.uuid | 2 +- 3 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 ext/session/sessionmem.test diff --git a/ext/session/sessionmem.test b/ext/session/sessionmem.test new file mode 100644 index 0000000000..75b48623d0 --- /dev/null +++ b/ext/session/sessionmem.test @@ -0,0 +1,57 @@ +# 2020 December 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for the SQLite sessions module +# Specifically, for the sqlite3session_memory_used() API. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionmem + +do_execsql_test 1.0 { + CREATE TABLE t1(i INTEGER PRIMARY KEY, x, y); + CREATE TABLE t2(i INTEGER, x, y, PRIMARY KEY(x, y)); +} + +do_test 1.1 { + sqlite3session S db main + S attach * +} {} + +foreach {tn sql eRes} { + 1 { INSERT INTO t1 VALUES(1, 2, 3) } 1 + 2 { UPDATE t1 SET x=5 } 0 + 3 { UPDATE t1 SET i=5 } 1 + 4 { DELETE FROM t1 } 0 + 5 { INSERT INTO t1 VALUES(1, 2, 3) } 0 + 6 { INSERT INTO t1 VALUES(5, 2, 3) } 0 + 7 { INSERT INTO t2 VALUES('a', 'b', 'c') } 1 + 8 { INSERT INTO t2 VALUES('d', 'e', 'f') } 1 + 9 { UPDATE t2 SET i='e' } 0 +} { + set mem1 [S memory_used] + do_test 1.2.$tn.(mu=$mem1) { + execsql $sql + set mem2 [S memory_used] + expr {$mem2 > $mem1} + } $eRes +} + +do_test 1.3 { + S delete +} {} + +finish_test diff --git a/manifest b/manifest index a604568c21..1f0fed3243 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sseen\swith\sMSVC. -D 2021-01-18T19:28:56.733 +C Add\stests\sfor\ssqlite3session_memory_used()\sinterface. +D 2021-01-20T10:59:47.320 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -450,6 +450,7 @@ F ext/session/sessiondiff.test ad13dd65664bae26744e1f18eb3cbd5588349b7e9118851d8 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 F ext/session/sessionfault2.test dd593f80b6b4786f7adfe83c5939620bc505559770cc181332da26f29cddd7bb F ext/session/sessioninvert.test 04075517a9497a80d39c495ba6b44f3982c7371129b89e2c52219819bc105a25 +F ext/session/sessionmem.test f2a735db84a3e9e19f571033b725b0b2daf847f3f28b1da55a0c1a4e74f1de09 F ext/session/sessionrebase.test ccfa716b23bd1d3b03217ee58cfd90c78d4b99f53e6a9a2f05e82363b9142810 F ext/session/sessionstat1.test 218d351cf9fcd6648f125a26b607b140310160184723c2666091b54450a68fb5 F ext/session/sessionwor.test 67b5ab91d4f93ce65ff1f58240ac5ddf73f8670facc1ffa49cef56293d52818d @@ -1897,7 +1898,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fe1979552f43e0526f16481457e01981f29707401f77079f9854a8d91b35b5a4 -R dc7be2ccd019ce776f5c5401443272a0 -U mistachkin -Z dce87543665f3b6eb6e012303a5399fb +P dc7938d2d715301595dee2fac6880af3716c4b3d1cbe7c3578d7fd30ba146a23 +R 2a7f82e5717f1cb1207576a8070164d7 +U dan +Z 837106ab8ad362dbfd49cfef88809376 diff --git a/manifest.uuid b/manifest.uuid index f83d261629..92afa750c7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc7938d2d715301595dee2fac6880af3716c4b3d1cbe7c3578d7fd30ba146a23 \ No newline at end of file +5596611f96f3401262b9dadc591bf7e3411d2c4a6f5be5cfe524e203d9820fd1 \ No newline at end of file From 7b88f5491487424bd88ad0ebdd22d81ac92690bf Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 20 Jan 2021 23:01:31 +0000 Subject: [PATCH 24/30] Improvements to the auxiliary "main.mk" makefile so that it works better with multi-threaded builds. FossilOrigin-Name: d1873054d8e1006a370ea7891dbb9a62e7d36ce98cb92b58dcb0daf271265de3 --- main.mk | 12 ++++++------ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/main.mk b/main.mk index e47e85fdfb..dfaf7db15e 100644 --- a/main.mk +++ b/main.mk @@ -550,13 +550,13 @@ ST_OPT = -DSQLITE_THREADSAFE=0 # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # -all: sqlite3.h libsqlite3.a sqlite3$(EXE) +all: sqlite3.h sqlite3ext.h libsqlite3.a sqlite3$(EXE) -libsqlite3.a: $(LIBOBJ) +libsqlite3.a: sqlite3.h $(LIBOBJ) $(AR) libsqlite3.a $(LIBOBJ) $(RANLIB) libsqlite3.a -sqlite3$(EXE): shell.c libsqlite3.a sqlite3.h +sqlite3$(EXE): sqlite3.h libsqlite3.a shell.c $(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) $(SHELL_OPT) \ shell.c libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB) @@ -824,13 +824,13 @@ fts3_unicode2.o: $(TOP)/ext/fts3/fts3_unicode2.c $(HDR) $(EXTHDR) fts3_write.o: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c -fts5.o: fts5.c +fts5.o: fts5.c sqlite3ext.h sqlite3.h $(TCCX) -DSQLITE_CORE -c fts5.c -json1.o: $(TOP)/ext/misc/json1.c +json1.o: $(TOP)/ext/misc/json1.c sqlite3ext.h sqlite3.h $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c -stmt.o: $(TOP)/ext/misc/stmt.c +stmt.o: $(TOP)/ext/misc/stmt.c sqlite3ext.h sqlite3.h $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/misc/stmt.c rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) diff --git a/manifest b/manifest index 1f0fed3243..df9622f9fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\ssqlite3session_memory_used()\sinterface. -D 2021-01-20T10:59:47.320 +C Improvements\sto\sthe\sauxiliary\s"main.mk"\smakefile\sso\sthat\sit\sworks\sbetter\swith\nmulti-threaded\sbuilds. +D 2021-01-20T23:01:31.158 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -463,7 +463,7 @@ F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 57451ea5b3d5cd86e9c5324b10c9de184b12e8dcccc31c65d24fbcb55ccd9c53 +F main.mk 443a4ec1ca89ad267cbde45dadc68861154b99c7bd26e7b7dc74303a664002b8 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1898,7 +1898,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc7938d2d715301595dee2fac6880af3716c4b3d1cbe7c3578d7fd30ba146a23 -R 2a7f82e5717f1cb1207576a8070164d7 -U dan -Z 837106ab8ad362dbfd49cfef88809376 +P 5596611f96f3401262b9dadc591bf7e3411d2c4a6f5be5cfe524e203d9820fd1 +R e4325fbe06f6c9247629eb04294fa661 +U drh +Z 1b9b0d992b3d4f78ff264eba5ac968ed diff --git a/manifest.uuid b/manifest.uuid index 92afa750c7..7e5a82c2e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5596611f96f3401262b9dadc591bf7e3411d2c4a6f5be5cfe524e203d9820fd1 \ No newline at end of file +d1873054d8e1006a370ea7891dbb9a62e7d36ce98cb92b58dcb0daf271265de3 \ No newline at end of file From f380c3f13c2b7da8a9f95768c6062d56b0f81d66 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Jan 2021 15:40:52 +0000 Subject: [PATCH 25/30] Fix a problem caused by using an SQL variable in an OVER clause within a trigger program. FossilOrigin-Name: 4f676466e60ee2a420b7b2deace76f3a733ce1af278347428285715d9c67f022 --- manifest | 23 ++--- manifest.uuid | 2 +- src/attach.c | 198 +++++++++++++++++++------------------------- src/sqliteInt.h | 33 ++++---- test/altertab3.test | 2 +- test/triggerE.test | 2 + 6 files changed, 118 insertions(+), 142 deletions(-) diff --git a/manifest b/manifest index df9622f9fc..c88831d299 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\sauxiliary\s"main.mk"\smakefile\sso\sthat\sit\sworks\sbetter\swith\nmulti-threaded\sbuilds. -D 2021-01-20T23:01:31.158 +C Fix\sa\sproblem\scaused\sby\susing\san\sSQL\svariable\sin\san\sOVER\sclause\swithin\sa\strigger\sprogram. +D 2021-01-21T15:40:52.172 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -477,7 +477,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 36cae0d6e3e91a1996e1a472f8c7242c31a4e38ba4295e3056da198c04fd2a87 F src/analyze.c 01c6c6765cb4d40b473b71d85535093730770bb186f2f473abac25f07fcdee5c -F src/attach.c 0f497c15c4cfe3bdcb214f0dbdbbb6c5ed7e8a9308ac445c7959f5e5780437a9 +F src/attach.c 87102aba5ddac8ef3a8a033ab657e4cae9cc8e846efe93b14029cfffaaf8831e F src/auth.c 8d1df0e2ef8bafbedd4f1fe4baff03eb27507da4bf6e449df3613d383c4018b2 F src/backup.c 3014889fa06e20e6adfa0d07b60097eec1f6e5b06671625f476a714d2356513d F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 @@ -546,7 +546,7 @@ F src/shell.c.in 9ebc74e4f05cfbd0f4a36060fdaeff1da4e9af4458358722bc08c5a1ab9a087 F src/sqlite.h.in 0af968a1fa3c717261e1df0ed105fa7bddb4d82de7e0adb3eab49e6aa81b4de7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 61b38c073d5e1e96a3d45271b257aef27d0d13da2bea5347692ae579475cd95e -F src/sqliteInt.h d921214bba203960de10ea7a9183735ec362a5def5910d468282a8f82e8b4c26 +F src/sqliteInt.h 3e5bc0446611e272b93754265e3265f36249d0458da25e32991fce241d69dbcf F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -652,7 +652,7 @@ F test/altermalloc.test 167a47de41b5c638f5f5c6efb59784002b196fff70f98d9b4ed3cd74 F test/altermalloc2.test fa7b1c1139ea39b8dec407cf1feb032ca8e0076bd429574969b619175ad0174b F test/altertab.test 6d7bbac2c4a6ef71b775094a3298fa3a92274d95034ee23157ffba92768e47e6 F test/altertab2.test b0d62f323ca5dab42b0bc028c52e310ebdd13e655e8fac070fe622bad7852c2b -F test/altertab3.test 1db384eb85b4a30b0b332842f5c596b0dc0126f7c61959be3f85ae8b1c271d9a +F test/altertab3.test 2b82fa2236a3a91553d53ae5555d8e723c7eec174c41f1fa62ff497355398479 F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 547bb700f903107b38611b014ca645d6b5bb819f5210d7bf39c40802aafeb7d7 F test/analyze3.test fca2a9de0017becfdcc201647f03b1cfd5ba0e7b5b5c852936e4ec62780cde49 @@ -1617,7 +1617,7 @@ F test/triggerA.test 837be862d8721f903dba3f3ceff05b32e0bee5214cf6ea3da5fadf12d36 F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe F test/triggerC.test 29f5a28d0fe39e6e2c01f6e1f53f08c0955170ae10a63ad023e33cb0a1682a51 F test/triggerD.test 8e7f3921a92a5797d472732108109e44575fa650 -F test/triggerE.test ede2e4bce4ba802337bd69d39447fa04a938e06d84a8bfc53c76850fc36ed86d +F test/triggerE.test 612969cb57a4ef792059ad6d01af0117e1ae862c283753ffcc9a6428642b22ee F test/triggerF.test 5d76f0a8c428ff87a4d5ed52da06f6096a2c787a1e21b846111dfac4123de3ad F test/triggerG.test 2b816093c91ba73c733cfa8aedcc210ad819d72a98b1da30768a3c56505233e9 F test/triggerupfrom.test d25961fa70a99b6736193da7b49a36d8c1d28d56188f0be6406d4366315cd6e4 @@ -1898,7 +1898,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5596611f96f3401262b9dadc591bf7e3411d2c4a6f5be5cfe524e203d9820fd1 -R e4325fbe06f6c9247629eb04294fa661 -U drh -Z 1b9b0d992b3d4f78ff264eba5ac968ed +P d1873054d8e1006a370ea7891dbb9a62e7d36ce98cb92b58dcb0daf271265de3 +R 56f323b51c91a7109bc328027a56f7a2 +T *branch * fix-over-trigger +T *sym-fix-over-trigger * +T -sym-trunk * +U dan +Z c7d4efd5b6b0d32bc23a7887ed998edc diff --git a/manifest.uuid b/manifest.uuid index 7e5a82c2e6..faba828e79 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d1873054d8e1006a370ea7891dbb9a62e7d36ce98cb92b58dcb0daf271265de3 \ No newline at end of file +4f676466e60ee2a420b7b2deace76f3a733ce1af278347428285715d9c67f022 \ No newline at end of file diff --git a/src/attach.c b/src/attach.c index 3b5c57f0cc..04274c6a7a 100644 --- a/src/attach.c +++ b/src/attach.c @@ -433,6 +433,63 @@ void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *pKey){ } #endif /* SQLITE_OMIT_ATTACH */ +/* +** Expression callback used by sqlite3FixAAAA() routines. +*/ +static int fixExprCb(Walker *p, Expr *pExpr){ + DbFixer *pFix = p->u.pFix; + if( !pFix->bTemp ) ExprSetProperty(pExpr, EP_FromDDL); + if( pExpr->op==TK_VARIABLE ){ + if( pFix->pParse->db->init.busy ){ + pExpr->op = TK_NULL; + }else{ + sqlite3ErrorMsg(pFix->pParse, "%s cannot use variables", pFix->zType); + return WRC_Abort; + } + } + return WRC_Continue; +} + +/* +** Select callback used by sqlite3FixAAAA() routines. +*/ +static int fixSelectCb(Walker *p, Select *pSelect){ + DbFixer *pFix = p->u.pFix; + int i; + struct SrcList_item *pItem; + sqlite3 *db = pFix->pParse->db; + int iDb = sqlite3FindDbName(db, pFix->zDb); + SrcList *pList = pSelect->pSrc; + + if( NEVER(pList==0) ) return WRC_Continue; + for(i=0, pItem=pList->a; inSrc; i++, pItem++){ + if( pFix->bTemp==0 ){ + if( pItem->zDatabase && iDb!=sqlite3FindDbName(db, pItem->zDatabase) ){ + sqlite3ErrorMsg(pFix->pParse, + "%s %T cannot reference objects in database %s", + pFix->zType, pFix->pName, pItem->zDatabase); + return WRC_Abort; + } + sqlite3DbFree(db, pItem->zDatabase); + pItem->zDatabase = 0; + pItem->pSchema = pFix->pSchema; + pItem->fg.fromDDL = 1; + } +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) + if( sqlite3WalkExpr(&pFix->w, pList->a[i].pOn) ) return WRC_Abort; +#endif + } + if( pSelect->pWith ){ + int i; + for(i=0; ipWith->nCte; i++){ + if( sqlite3WalkSelect(p, pSelect->pWith->a[i].pSelect) ){ + return WRC_Abort; + } + } + } + return WRC_Continue; +} + /* ** Initialize a DbFixer structure. This routine must be called prior ** to passing the structure to one of the sqliteFixAAAA() routines below. @@ -444,9 +501,7 @@ void sqlite3FixInit( const char *zType, /* "view", "trigger", or "index" */ const Token *pName /* Name of the view, trigger, or index */ ){ - sqlite3 *db; - - db = pParse->db; + sqlite3 *db = pParse->db; assert( db->nDb>iDb ); pFix->pParse = pParse; pFix->zDb = db->aDb[iDb].zDbSName; @@ -454,6 +509,13 @@ void sqlite3FixInit( pFix->zType = zType; pFix->pName = pName; pFix->bTemp = (iDb==1); + pFix->w.pParse = pParse; + pFix->w.xExprCallback = fixExprCb; + pFix->w.xSelectCallback = fixSelectCb; + pFix->w.xSelectCallback2 = 0; + pFix->w.walkerDepth = 0; + pFix->w.eCode = 0; + pFix->w.u.pFix = pFix; } /* @@ -474,115 +536,27 @@ int sqlite3FixSrcList( DbFixer *pFix, /* Context of the fixation */ SrcList *pList /* The Source list to check and modify */ ){ - int i; - struct SrcList_item *pItem; - sqlite3 *db = pFix->pParse->db; - int iDb = sqlite3FindDbName(db, pFix->zDb); - - if( NEVER(pList==0) ) return 0; - - for(i=0, pItem=pList->a; inSrc; i++, pItem++){ - if( pFix->bTemp==0 ){ - if( pItem->zDatabase && iDb!=sqlite3FindDbName(db, pItem->zDatabase) ){ - sqlite3ErrorMsg(pFix->pParse, - "%s %T cannot reference objects in database %s", - pFix->zType, pFix->pName, pItem->zDatabase); - return 1; - } - sqlite3DbFree(db, pItem->zDatabase); - pItem->zDatabase = 0; - pItem->pSchema = pFix->pSchema; - pItem->fg.fromDDL = 1; - } -#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) - if( sqlite3FixSelect(pFix, pItem->pSelect) ) return 1; - if( sqlite3FixExpr(pFix, pItem->pOn) ) return 1; -#endif - if( pItem->fg.isTabFunc && sqlite3FixExprList(pFix, pItem->u1.pFuncArg) ){ - return 1; - } + int res = 0; + if( pList ){ + Select s; + memset(&s, 0, sizeof(s)); + s.pSrc = pList; + res = sqlite3WalkSelect(&pFix->w, &s); } - return 0; + return res; } #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) int sqlite3FixSelect( DbFixer *pFix, /* Context of the fixation */ Select *pSelect /* The SELECT statement to be fixed to one database */ ){ - while( pSelect ){ - if( sqlite3FixExprList(pFix, pSelect->pEList) ){ - return 1; - } - if( sqlite3FixSrcList(pFix, pSelect->pSrc) ){ - return 1; - } - if( sqlite3FixExpr(pFix, pSelect->pWhere) ){ - return 1; - } - if( sqlite3FixExprList(pFix, pSelect->pGroupBy) ){ - return 1; - } - if( sqlite3FixExpr(pFix, pSelect->pHaving) ){ - return 1; - } - if( sqlite3FixExprList(pFix, pSelect->pOrderBy) ){ - return 1; - } - if( sqlite3FixExpr(pFix, pSelect->pLimit) ){ - return 1; - } - if( pSelect->pWith ){ - int i; - for(i=0; ipWith->nCte; i++){ - if( sqlite3FixSelect(pFix, pSelect->pWith->a[i].pSelect) ){ - return 1; - } - } - } - pSelect = pSelect->pPrior; - } - return 0; + return sqlite3WalkSelect(&pFix->w, pSelect); } int sqlite3FixExpr( DbFixer *pFix, /* Context of the fixation */ Expr *pExpr /* The expression to be fixed to one database */ ){ - while( pExpr ){ - if( !pFix->bTemp ) ExprSetProperty(pExpr, EP_FromDDL); - if( pExpr->op==TK_VARIABLE ){ - if( pFix->pParse->db->init.busy ){ - pExpr->op = TK_NULL; - }else{ - sqlite3ErrorMsg(pFix->pParse, "%s cannot use variables", pFix->zType); - return 1; - } - } - if( ExprHasProperty(pExpr, EP_TokenOnly|EP_Leaf) ) break; - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - if( sqlite3FixSelect(pFix, pExpr->x.pSelect) ) return 1; - }else{ - if( sqlite3FixExprList(pFix, pExpr->x.pList) ) return 1; - } - if( sqlite3FixExpr(pFix, pExpr->pRight) ){ - return 1; - } - pExpr = pExpr->pLeft; - } - return 0; -} -int sqlite3FixExprList( - DbFixer *pFix, /* Context of the fixation */ - ExprList *pList /* The expression to be fixed to one database */ -){ - int i; - struct ExprList_item *pItem; - if( pList==0 ) return 0; - for(i=0, pItem=pList->a; inExpr; i++, pItem++){ - if( sqlite3FixExpr(pFix, pItem->pExpr) ){ - return 1; - } - } - return 0; + return sqlite3WalkExpr(&pFix->w, pExpr); } #endif @@ -592,25 +566,20 @@ int sqlite3FixTriggerStep( TriggerStep *pStep /* The trigger step be fixed to one database */ ){ while( pStep ){ - if( sqlite3FixSelect(pFix, pStep->pSelect) ){ - return 1; - } - if( sqlite3FixExpr(pFix, pStep->pWhere) ){ - return 1; - } - if( sqlite3FixExprList(pFix, pStep->pExprList) ){ - return 1; - } - if( pStep->pFrom && sqlite3FixSrcList(pFix, pStep->pFrom) ){ + if( sqlite3WalkSelect(&pFix->w, pStep->pSelect) + || sqlite3WalkExpr(&pFix->w, pStep->pWhere) + || sqlite3WalkExprList(&pFix->w, pStep->pExprList) + || sqlite3FixSrcList(pFix, pStep->pFrom) + ){ return 1; } #ifndef SQLITE_OMIT_UPSERT if( pStep->pUpsert ){ Upsert *pUp = pStep->pUpsert; - if( sqlite3FixExprList(pFix, pUp->pUpsertTarget) - || sqlite3FixExpr(pFix, pUp->pUpsertTargetWhere) - || sqlite3FixExprList(pFix, pUp->pUpsertSet) - || sqlite3FixExpr(pFix, pUp->pUpsertWhere) + if( sqlite3WalkExprList(&pFix->w, pUp->pUpsertTarget) + || sqlite3WalkExpr(&pFix->w, pUp->pUpsertTargetWhere) + || sqlite3WalkExprList(&pFix->w, pUp->pUpsertSet) + || sqlite3WalkExpr(&pFix->w, pUp->pUpsertWhere) ){ return 1; } @@ -618,6 +587,7 @@ int sqlite3FixTriggerStep( #endif pStep = pStep->pNext; } + return 0; } #endif diff --git a/src/sqliteInt.h b/src/sqliteInt.h index e1e0707bf3..510aab1ca5 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1137,6 +1137,7 @@ typedef struct Bitvec Bitvec; typedef struct CollSeq CollSeq; typedef struct Column Column; typedef struct Db Db; +typedef struct DbFixer DbFixer; typedef struct Schema Schema; typedef struct Expr Expr; typedef struct ExprList ExprList; @@ -3650,21 +3651,6 @@ struct TriggerStep { TriggerStep *pLast; /* Last element in link-list. Valid for 1st elem only */ }; -/* -** The following structure contains information used by the sqliteFix... -** routines as they walk the parse tree to make database references -** explicit. -*/ -typedef struct DbFixer DbFixer; -struct DbFixer { - Parse *pParse; /* The parsing context. Error messages written here */ - Schema *pSchema; /* Fix items to this schema */ - u8 bTemp; /* True for TEMP schema entries */ - const char *zDb; /* Make sure all objects are contained in this database */ - const char *zType; /* Type of the container - used for error messages */ - const Token *pName; /* Name of the container - used for error messages */ -}; - /* ** An objected used to accumulate the text of a string where we ** do not necessarily know how big the string will be in the end. @@ -3815,9 +3801,25 @@ struct Walker { struct RenameCtx *pRename; /* RENAME COLUMN context */ struct Table *pTab; /* Table of generated column */ struct SrcList_item *pSrcItem; /* A single FROM clause item */ + DbFixer *pFix; } u; }; +/* +** The following structure contains information used by the sqliteFix... +** routines as they walk the parse tree to make database references +** explicit. +*/ +struct DbFixer { + Parse *pParse; /* The parsing context. Error messages written here */ + Walker w; /* Walker object */ + Schema *pSchema; /* Fix items to this schema */ + u8 bTemp; /* True for TEMP schema entries */ + const char *zDb; /* Make sure all objects are contained in this database */ + const char *zType; /* Type of the container - used for error messages */ + const Token *pName; /* Name of the container - used for error messages */ +}; + /* Forward declarations */ int sqlite3WalkExpr(Walker*, Expr*); int sqlite3WalkExprList(Walker*, ExprList*); @@ -4527,7 +4529,6 @@ void sqlite3FixInit(DbFixer*, Parse*, int, const char*, const Token*); int sqlite3FixSrcList(DbFixer*, SrcList*); int sqlite3FixSelect(DbFixer*, Select*); int sqlite3FixExpr(DbFixer*, Expr*); -int sqlite3FixExprList(DbFixer*, ExprList*); int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); int sqlite3RealSameAsInt(double,sqlite3_int64); void sqlite3Int64ToText(i64,char*); diff --git a/test/altertab3.test b/test/altertab3.test index 10d9ef32e3..1823b21edf 100644 --- a/test/altertab3.test +++ b/test/altertab3.test @@ -253,7 +253,7 @@ do_execsql_test 11.1 { do_catchsql_test 11.2 { ALTER TABLE t1 RENAME TO t1x; -} {1 {error in trigger b: no such table: abc}} +} {1 {error in trigger b: no such table: main.abc}} do_execsql_test 11.3 { DROP TRIGGER b; diff --git a/test/triggerE.test b/test/triggerE.test index 9db7035325..de4b068582 100644 --- a/test/triggerE.test +++ b/test/triggerE.test @@ -58,6 +58,8 @@ foreach {tn defn} { 8 { BEFORE UPDATE ON t1 BEGIN UPDATE t2 SET c = ?; END; } 9 { BEFORE UPDATE ON t1 BEGIN UPDATE t2 SET c = 1 WHERE d = ?; END; } 10 { AFTER INSERT ON t1 BEGIN SELECT * FROM pragma_stats(?); END; } + 11 { BEFORE INSERT ON t1 BEGIN + INSERT INTO t1 SELECT max(b) OVER(ORDER BY $1) FROM t1; END } } { catchsql {drop trigger tr1} do_catchsql_test 1.1.$tn "CREATE TRIGGER tr1 $defn" [list 1 $errmsg] From 6cca0aa9df3d4e18fd7eed0b9867120e6cc1e482 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 21 Jan 2021 17:54:41 +0000 Subject: [PATCH 26/30] Always enable the IS NOT NULL optimization, even if STAT4 is not enabled. FossilOrigin-Name: fc98218cf69e63bdb9e5f154521a341508502cd8cfe04cb870cabee2d99e0cb3 --- manifest | 20 +++++++------ manifest.uuid | 2 +- src/whereInt.h | 6 +--- src/whereexpr.c | 78 ++++++++++++++++++++++++------------------------- 4 files changed, 51 insertions(+), 55 deletions(-) diff --git a/manifest b/manifest index c2ea1b0e81..008fa8c252 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\scaused\sby\susing\san\sSQL\svariable\sin\san\sOVER\sclause\swithin\sa\strigger\sprogram. -D 2021-01-21T16:02:14.840 +C Always\senable\sthe\sIS\sNOT\sNULL\soptimization,\seven\sif\sSTAT4\sis\snot\senabled. +D 2021-01-21T17:54:41.795 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -629,9 +629,9 @@ F src/wal.c 69e770e96fd56cc21608992bf2c6f1f3dc5cf2572d0495c6a643b06c3a679f14 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c F src/where.c 0e6abb22a2323fec80b450825593c26a2ad8f4815d1ee3af9969d8f6144bf681 -F src/whereInt.h 9a3f577619f07700d16d89eeb2f3d94d6b7ed7f109c2dacf0ce8844921549506 +F src/whereInt.h ae03b5e3a4cca9bd9cb1b7d3c63faf8f1f177200fc8cecc87d3d0cab6ca338e6 F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee -F src/whereexpr.c a022b4f447c0fb0674e172a9a303537ea5055df60d579f99cec7ead18e8c453f +F src/whereexpr.c 1f11bdd204ad021f5935df6e7726b4f1dbdd3157a4f31d0dafc9c457e5e545a8 F src/window.c edd6f5e25a1e8f2b6f5305b7f5f7da7bb35f07f0d432b255b1d4c2fcab4205aa F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -1898,8 +1898,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d1873054d8e1006a370ea7891dbb9a62e7d36ce98cb92b58dcb0daf271265de3 4f676466e60ee2a420b7b2deace76f3a733ce1af278347428285715d9c67f022 -R 56f323b51c91a7109bc328027a56f7a2 -T +closed 4f676466e60ee2a420b7b2deace76f3a733ce1af278347428285715d9c67f022 -U dan -Z af485d37bfb17e22d383f2711ee5f268 +P 02264ab6a02d6cc95cf865920bcbaf4307d034640e6e4f3371b009ae9818540e +R 8df6cd517dfa239c965fc40092a79782 +T *branch * isnotnull-opt +T *sym-isnotnull-opt * +T -sym-trunk * +U drh +Z d06e074a8f142c72ee8a673bfd500f4d diff --git a/manifest.uuid b/manifest.uuid index 57afc8f5ef..1e251fa4a5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -02264ab6a02d6cc95cf865920bcbaf4307d034640e6e4f3371b009ae9818540e \ No newline at end of file +fc98218cf69e63bdb9e5f154521a341508502cd8cfe04cb870cabee2d99e0cb3 \ No newline at end of file diff --git a/src/whereInt.h b/src/whereInt.h index f8509996fa..89a463dc38 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -270,11 +270,7 @@ struct WhereTerm { #define TERM_ORINFO 0x0010 /* Need to free the WhereTerm.u.pOrInfo object */ #define TERM_ANDINFO 0x0020 /* Need to free the WhereTerm.u.pAndInfo obj */ #define TERM_OR_OK 0x0040 /* Used during OR-clause processing */ -#ifdef SQLITE_ENABLE_STAT4 -# define TERM_VNULL 0x0080 /* Manufactured x>NULL or x<=NULL term */ -#else -# define TERM_VNULL 0x0000 /* Disabled if not using stat4 */ -#endif +#define TERM_VNULL 0x0080 /* Manufactured x>NULL or x<=NULL term */ #define TERM_LIKEOPT 0x0100 /* Virtual terms from the LIKE optimization */ #define TERM_LIKECOND 0x0200 /* Conditionally this LIKE operator term */ #define TERM_LIKE 0x0400 /* The original LIKE operator */ diff --git a/src/whereexpr.c b/src/whereexpr.c index 5cfdad67c1..0dc56ea8a3 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1466,6 +1466,41 @@ static void exprAnalyze( } } + /* The form "x IS NOT NULL" can sometimes be evaluated more efficiently + ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a + ** virtual term of that form. + ** + ** The virtual term must be tagged with TERM_VNULL. + */ + else if( pExpr->op==TK_NOTNULL ){ + if( pExpr->pLeft->op==TK_COLUMN + && pExpr->pLeft->iColumn>=0 + && !ExprHasProperty(pExpr, EP_FromJoin) + ){ + Expr *pNewExpr; + Expr *pLeft = pExpr->pLeft; + int idxNew; + WhereTerm *pNewTerm; + + pNewExpr = sqlite3PExpr(pParse, TK_GT, + sqlite3ExprDup(db, pLeft, 0), + sqlite3ExprAlloc(db, TK_NULL, 0, 0)); + + idxNew = whereClauseInsert(pWC, pNewExpr, + TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL); + if( idxNew ){ + pNewTerm = &pWC->a[idxNew]; + pNewTerm->prereqRight = 0; + pNewTerm->leftCursor = pLeft->iTable; + pNewTerm->u.x.leftColumn = pLeft->iColumn; + pNewTerm->eOperator = WO_GT; + markTermAsChild(pWC, idxNew, idxTerm); + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_COPIED; + pNewTerm->prereqAll = pTerm->prereqAll; + } + } + } #ifndef SQLITE_OMIT_LIKE_OPTIMIZATION /* Add constraints to reduce the search space on a LIKE or GLOB @@ -1637,7 +1672,9 @@ static void exprAnalyze( ** This only works if the RHS is a simple SELECT (not a compound) that does ** not use window functions. */ - if( pWC->op==TK_AND && pExpr->op==TK_IN && pTerm->u.x.iField==0 + if( pWC->op==TK_AND + && pExpr->op==TK_IN + && pTerm->u.x.iField==0 && pExpr->pLeft->op==TK_VECTOR && pExpr->x.pSelect->pPrior==0 #ifndef SQLITE_OMIT_WINDOWFUNC @@ -1654,45 +1691,6 @@ static void exprAnalyze( } } -#ifdef SQLITE_ENABLE_STAT4 - /* When sqlite_stat4 histogram data is available an operator of the - ** form "x IS NOT NULL" can sometimes be evaluated more efficiently - ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a - ** virtual term of that form. - ** - ** Note that the virtual term must be tagged with TERM_VNULL. - */ - if( pExpr->op==TK_NOTNULL - && pExpr->pLeft->op==TK_COLUMN - && pExpr->pLeft->iColumn>=0 - && !ExprHasProperty(pExpr, EP_FromJoin) - && OptimizationEnabled(db, SQLITE_Stat4) - ){ - Expr *pNewExpr; - Expr *pLeft = pExpr->pLeft; - int idxNew; - WhereTerm *pNewTerm; - - pNewExpr = sqlite3PExpr(pParse, TK_GT, - sqlite3ExprDup(db, pLeft, 0), - sqlite3ExprAlloc(db, TK_NULL, 0, 0)); - - idxNew = whereClauseInsert(pWC, pNewExpr, - TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL); - if( idxNew ){ - pNewTerm = &pWC->a[idxNew]; - pNewTerm->prereqRight = 0; - pNewTerm->leftCursor = pLeft->iTable; - pNewTerm->u.x.leftColumn = pLeft->iColumn; - pNewTerm->eOperator = WO_GT; - markTermAsChild(pWC, idxNew, idxTerm); - pTerm = &pWC->a[idxTerm]; - pTerm->wtFlags |= TERM_COPIED; - pNewTerm->prereqAll = pTerm->prereqAll; - } - } -#endif /* SQLITE_ENABLE_STAT4 */ - /* Prevent ON clause terms of a LEFT JOIN from being used to drive ** an index for tables to the left of the join. */ From 71aff85503b826122aa22db1e341b87c29a5b305 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 21 Jan 2021 20:42:36 +0000 Subject: [PATCH 27/30] Performance optimizations in exprAnalyze() FossilOrigin-Name: 6d60cf540b8cc231448175f1e16e1f4f7a0aee26898570a5b8a09c89fae53c02 --- manifest | 15 +++--- manifest.uuid | 2 +- src/whereexpr.c | 124 ++++++++++++++++++++++++------------------------ 3 files changed, 70 insertions(+), 71 deletions(-) diff --git a/manifest b/manifest index 008fa8c252..a19e4809b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Always\senable\sthe\sIS\sNOT\sNULL\soptimization,\seven\sif\sSTAT4\sis\snot\senabled. -D 2021-01-21T17:54:41.795 +C Performance\soptimizations\sin\sexprAnalyze() +D 2021-01-21T20:42:36.445 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -631,7 +631,7 @@ F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c F src/where.c 0e6abb22a2323fec80b450825593c26a2ad8f4815d1ee3af9969d8f6144bf681 F src/whereInt.h ae03b5e3a4cca9bd9cb1b7d3c63faf8f1f177200fc8cecc87d3d0cab6ca338e6 F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee -F src/whereexpr.c 1f11bdd204ad021f5935df6e7726b4f1dbdd3157a4f31d0dafc9c457e5e545a8 +F src/whereexpr.c a182038cf7d10aa9a95a96b8563a34f34990323cf307dee6559e995961966d43 F src/window.c edd6f5e25a1e8f2b6f5305b7f5f7da7bb35f07f0d432b255b1d4c2fcab4205aa F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -1898,10 +1898,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 02264ab6a02d6cc95cf865920bcbaf4307d034640e6e4f3371b009ae9818540e -R 8df6cd517dfa239c965fc40092a79782 -T *branch * isnotnull-opt -T *sym-isnotnull-opt * -T -sym-trunk * +P fc98218cf69e63bdb9e5f154521a341508502cd8cfe04cb870cabee2d99e0cb3 +R f46bf0fe436bc977a841647e6a3e3ebc U drh -Z d06e074a8f142c72ee8a673bfd500f4d +Z 4d2704e467917a55449d9cd09c1bd5e8 diff --git a/manifest.uuid b/manifest.uuid index 1e251fa4a5..3639833e1d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc98218cf69e63bdb9e5f154521a341508502cd8cfe04cb870cabee2d99e0cb3 \ No newline at end of file +6d60cf540b8cc231448175f1e16e1f4f7a0aee26898570a5b8a09c89fae53c02 \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 0dc56ea8a3..f69210975c 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1502,6 +1502,7 @@ static void exprAnalyze( } } + #ifndef SQLITE_OMIT_LIKE_OPTIMIZATION /* Add constraints to reduce the search space on a LIKE or GLOB ** operator. @@ -1516,7 +1517,8 @@ static void exprAnalyze( ** bound is made all lowercase so that the bounds also work when comparing ** BLOBs. */ - if( pWC->op==TK_AND + else if( pExpr->op==TK_FUNCTION + && pWC->op==TK_AND && isLikeOrGlob(pParse, pExpr, &pStr1, &isComplete, &noCase) ){ Expr *pLeft; /* LHS of LIKE/GLOB operator */ @@ -1586,6 +1588,65 @@ static void exprAnalyze( } #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ + /* If there is a vector == or IS term - e.g. "(a, b) == (?, ?)" - create + ** new terms for each component comparison - "a = ?" and "b = ?". The + ** new terms completely replace the original vector comparison, which is + ** no longer used. + ** + ** This is only required if at least one side of the comparison operation + ** is not a sub-select. */ + if( (pExpr->op==TK_EQ || pExpr->op==TK_IS) + && (nLeft = sqlite3ExprVectorSize(pExpr->pLeft))>1 + && sqlite3ExprVectorSize(pExpr->pRight)==nLeft + && ( (pExpr->pLeft->flags & EP_xIsSelect)==0 + || (pExpr->pRight->flags & EP_xIsSelect)==0) + && pWC->op==TK_AND + ){ + int i; + for(i=0; ipLeft, i); + Expr *pRight = sqlite3ExprForVectorField(pParse, pExpr->pRight, i); + + pNew = sqlite3PExpr(pParse, pExpr->op, pLeft, pRight); + transferJoinMarkings(pNew, pExpr); + idxNew = whereClauseInsert(pWC, pNew, TERM_DYNAMIC); + exprAnalyze(pSrc, pWC, idxNew); + } + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_CODED|TERM_VIRTUAL; /* Disable the original */ + pTerm->eOperator = 0; + } + + /* If there is a vector IN term - e.g. "(a, b) IN (SELECT ...)" - create + ** a virtual term for each vector component. The expression object + ** used by each such virtual term is pExpr (the full vector IN(...) + ** expression). The WhereTerm.u.x.iField variable identifies the index within + ** the vector on the LHS that the virtual term represents. + ** + ** This only works if the RHS is a simple SELECT (not a compound) that does + ** not use window functions. + */ + else if( pExpr->op==TK_IN + && pTerm->u.x.iField==0 + && pExpr->pLeft->op==TK_VECTOR + && pExpr->x.pSelect->pPrior==0 +#ifndef SQLITE_OMIT_WINDOWFUNC + && pExpr->x.pSelect->pWin==0 +#endif + && pWC->op==TK_AND + ){ + int i; + for(i=0; ipLeft); i++){ + int idxNew; + idxNew = whereClauseInsert(pWC, pExpr, TERM_VIRTUAL); + pWC->a[idxNew].u.x.iField = i+1; + exprAnalyze(pSrc, pWC, idxNew); + markTermAsChild(pWC, idxNew, idxTerm); + } + } + #ifndef SQLITE_OMIT_VIRTUALTABLE /* Add a WO_AUX auxiliary term to the constraint set if the ** current expression is of the form "column OP expr" where OP @@ -1596,7 +1657,7 @@ static void exprAnalyze( ** virtual tables. The native query optimizer does not attempt ** to do anything with MATCH functions. */ - if( pWC->op==TK_AND ){ + else if( pWC->op==TK_AND ){ Expr *pRight = 0, *pLeft = 0; int res = isAuxiliaryVtabOperator(db, pExpr, &eOp2, &pLeft, &pRight); while( res-- > 0 ){ @@ -1632,65 +1693,6 @@ static void exprAnalyze( } #endif /* SQLITE_OMIT_VIRTUALTABLE */ - /* If there is a vector == or IS term - e.g. "(a, b) == (?, ?)" - create - ** new terms for each component comparison - "a = ?" and "b = ?". The - ** new terms completely replace the original vector comparison, which is - ** no longer used. - ** - ** This is only required if at least one side of the comparison operation - ** is not a sub-select. */ - if( pWC->op==TK_AND - && (pExpr->op==TK_EQ || pExpr->op==TK_IS) - && (nLeft = sqlite3ExprVectorSize(pExpr->pLeft))>1 - && sqlite3ExprVectorSize(pExpr->pRight)==nLeft - && ( (pExpr->pLeft->flags & EP_xIsSelect)==0 - || (pExpr->pRight->flags & EP_xIsSelect)==0) - ){ - int i; - for(i=0; ipLeft, i); - Expr *pRight = sqlite3ExprForVectorField(pParse, pExpr->pRight, i); - - pNew = sqlite3PExpr(pParse, pExpr->op, pLeft, pRight); - transferJoinMarkings(pNew, pExpr); - idxNew = whereClauseInsert(pWC, pNew, TERM_DYNAMIC); - exprAnalyze(pSrc, pWC, idxNew); - } - pTerm = &pWC->a[idxTerm]; - pTerm->wtFlags |= TERM_CODED|TERM_VIRTUAL; /* Disable the original */ - pTerm->eOperator = 0; - } - - /* If there is a vector IN term - e.g. "(a, b) IN (SELECT ...)" - create - ** a virtual term for each vector component. The expression object - ** used by each such virtual term is pExpr (the full vector IN(...) - ** expression). The WhereTerm.u.x.iField variable identifies the index within - ** the vector on the LHS that the virtual term represents. - ** - ** This only works if the RHS is a simple SELECT (not a compound) that does - ** not use window functions. - */ - if( pWC->op==TK_AND - && pExpr->op==TK_IN - && pTerm->u.x.iField==0 - && pExpr->pLeft->op==TK_VECTOR - && pExpr->x.pSelect->pPrior==0 -#ifndef SQLITE_OMIT_WINDOWFUNC - && pExpr->x.pSelect->pWin==0 -#endif - ){ - int i; - for(i=0; ipLeft); i++){ - int idxNew; - idxNew = whereClauseInsert(pWC, pExpr, TERM_VIRTUAL); - pWC->a[idxNew].u.x.iField = i+1; - exprAnalyze(pSrc, pWC, idxNew); - markTermAsChild(pWC, idxNew, idxTerm); - } - } - /* Prevent ON clause terms of a LEFT JOIN from being used to drive ** an index for tables to the left of the join. */ From 906602ac3f04fd329e83ccb4567543276721e843 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 21 Jan 2021 21:36:25 +0000 Subject: [PATCH 28/30] Do not allow VACUUM to resize the page_size to 512 if the reserve_byte value is 31 or greater. [forum:/forumpost/e807885dc5|forum post e807885dc5]. FossilOrigin-Name: d5ea75a09d4bf61262cead2604e35d8331b727c504807592d09f5d9e01ce794a --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/btree.c | 1 + 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c2ea1b0e81..0cc6359996 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\scaused\sby\susing\san\sSQL\svariable\sin\san\sOVER\sclause\swithin\sa\strigger\sprogram. -D 2021-01-21T16:02:14.840 +C Do\snot\sallow\sVACUUM\sto\sresize\sthe\spage_size\sto\s512\sif\sthe\sreserve_byte\svalue\nis\s31\sor\sgreater.\s[forum:/forumpost/e807885dc5|forum\spost\se807885dc5]. +D 2021-01-21T21:36:25.588 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -482,7 +482,7 @@ F src/auth.c 8d1df0e2ef8bafbedd4f1fe4baff03eb27507da4bf6e449df3613d383c4018b2 F src/backup.c 3014889fa06e20e6adfa0d07b60097eec1f6e5b06671625f476a714d2356513d F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c 0f9cb686871ae668817673f0823b55d1bcadbc86ea28bd22c590b064a8322d5a +F src/btree.c 47d9fe97d5c0d74506154e3597f8a23b81a00080751dc4d11fec91ee22796f4c F src/btree.h 285f8377aa1353185a32bf455faafa9ff9a0d40d074d60509534d14990c7829e F src/btreeInt.h 7614cae30f95b6aed0c7cac7718276a55cfe2c77058cbfd8bef5b75329757331 F src/build.c d4c06261b0e532523ede58dc511381a7a9c155132e4b65a6bb2ff76fe657793a @@ -1898,8 +1898,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d1873054d8e1006a370ea7891dbb9a62e7d36ce98cb92b58dcb0daf271265de3 4f676466e60ee2a420b7b2deace76f3a733ce1af278347428285715d9c67f022 -R 56f323b51c91a7109bc328027a56f7a2 -T +closed 4f676466e60ee2a420b7b2deace76f3a733ce1af278347428285715d9c67f022 -U dan -Z af485d37bfb17e22d383f2711ee5f268 +P 02264ab6a02d6cc95cf865920bcbaf4307d034640e6e4f3371b009ae9818540e +R 48ee8ca426c1edea6b9f06bdf29c3bdc +U drh +Z cbe3ac807720d5b866771e6277e15b90 diff --git a/manifest.uuid b/manifest.uuid index 57afc8f5ef..949cc47910 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -02264ab6a02d6cc95cf865920bcbaf4307d034640e6e4f3371b009ae9818540e \ No newline at end of file +d5ea75a09d4bf61262cead2604e35d8331b727c504807592d09f5d9e01ce794a \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index a587332ce2..a3fbfef043 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2901,6 +2901,7 @@ int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ ((pageSize-1)&pageSize)==0 ){ assert( (pageSize & 7)==0 ); assert( !pBt->pCursor ); + if( nReserve>32 && pageSize==512 ) pageSize = 1024; pBt->pageSize = (u32)pageSize; freeTempSpace(pBt); } From e5ceaac446d11372cd4b20cd5d9033a8e868c66a Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 25 Jan 2021 21:24:14 +0000 Subject: [PATCH 29/30] Add an extra log message in the case of an SQLITE_CORRUPT_INDEX error. FossilOrigin-Name: 0571c24177d77ac966bcf42cb8ab00fdf541ce84af1468c8ef0b60c48c45a22f --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 192c496e18..e0a9937e58 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\squery\splanner\sto\senable\sit\sto\suse\san\sindex\sfor\sIS\sNOT\sNULL\nconstraints,\seven\sif\sSTAT4\sis\snot\senabled. -D 2021-01-22T21:23:12.101 +C Add\san\sextra\slog\smessage\sin\sthe\scase\sof\san\sSQLITE_CORRUPT_INDEX\serror. +D 2021-01-25T21:24:14.510 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -613,7 +613,7 @@ F src/upsert.c df8f1727d62b5987c4fd302cd4d7c0c84ae57cd65683c5a34a740dfe24039235 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 41c7a72da1df47864faa378a1c720b38adb288c6838cb6be5594511b6287a048 F src/vacuum.c 492422c1463c076473bae1858799c7a0a5fe87a133d1223239447c422cd26286 -F src/vdbe.c fd3ae827dea9da118d22c172ad183888120070d4c709bd0a5a97af7749246783 +F src/vdbe.c 102d21260bddbb43c845603c3a2d6b4f3762e72f836ccda12991f291485d2539 F src/vdbe.h 83603854bfa5851af601fc0947671eb260f4363e62e960e8a994fb9bbcd2aaa1 F src/vdbeInt.h 3ca5e9fd6e095a8b6cf6bc3587a46fc93499503b2fe48951e1034ba9e2ce2f6e F src/vdbeapi.c c5e7cb2ab89a24d7f723e87b508f21bfb1359a04db5277d8a99fd1e015c12eb9 @@ -1898,8 +1898,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d5ea75a09d4bf61262cead2604e35d8331b727c504807592d09f5d9e01ce794a 0b42f9eb5dd710991af3cf4b16464b22db9539361d618e50eaa4d43c811577b4 -R dd365517921749a64a3732b6a8560730 -T +closed 0b42f9eb5dd710991af3cf4b16464b22db9539361d618e50eaa4d43c811577b4 +P 7b2a7c7314d2239992dc1d707280f3b75ba75bb074ba1e77b55cad01ca7fd51b +R c809d928cb746073aedc3f5cce34c965 U drh -Z 6ed219298d2cd360981b4aacc96226a6 +Z 375659ad9a05c828e459b8457d7233ea diff --git a/manifest.uuid b/manifest.uuid index bdab433263..93a9669da3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7b2a7c7314d2239992dc1d707280f3b75ba75bb074ba1e77b55cad01ca7fd51b \ No newline at end of file +0571c24177d77ac966bcf42cb8ab00fdf541ce84af1468c8ef0b60c48c45a22f \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index bcfc5df8d9..3a00515e57 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5912,7 +5912,7 @@ case OP_IdxDelete: { rc = sqlite3BtreeDelete(pCrsr, BTREE_AUXDELETE); if( rc ) goto abort_due_to_error; }else if( pOp->p5 ){ - rc = SQLITE_CORRUPT_INDEX; + rc = sqlite3ReportError(SQLITE_CORRUPT_INDEX, __LINE__, "index corruption"); goto abort_due_to_error; } assert( pC->deferredMoveto==0 ); From df1b52e72795ad08fc1b80027f3d4fbe6be408c4 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 27 Jan 2021 17:15:06 +0000 Subject: [PATCH 30/30] Ensure a cursor used by the SeekScan operator does not point to a valid row on the first iteration of the loop. Possible fix for [2d6e8400]. FossilOrigin-Name: 390cf60a286b13f454429f4652a133f95a7891a75c1ec6d16cd39990590fd3fb --- manifest | 19 +++++++++++-------- manifest.uuid | 2 +- src/wherecode.c | 6 ++++++ test/in4.test | 28 ++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e0a9937e58..00a745920f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sextra\slog\smessage\sin\sthe\scase\sof\san\sSQLITE_CORRUPT_INDEX\serror. -D 2021-01-25T21:24:14.510 +C Ensure\sa\scursor\sused\sby\sthe\sSeekScan\soperator\sdoes\snot\spoint\sto\sa\svalid\srow\son\sthe\sfirst\siteration\sof\sthe\sloop.\sPossible\sfix\sfor\s[2d6e8400]. +D 2021-01-27T17:15:06.938 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -630,7 +630,7 @@ F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c F src/where.c 0e6abb22a2323fec80b450825593c26a2ad8f4815d1ee3af9969d8f6144bf681 F src/whereInt.h ae03b5e3a4cca9bd9cb1b7d3c63faf8f1f177200fc8cecc87d3d0cab6ca338e6 -F src/wherecode.c a3a1aff30fe99a818d8e7c607980f033f40c68d890e03ed25838b9dbb7908bee +F src/wherecode.c 43a63441f8662ddf86b15975683a502ec33f08167e9636f4d19e38e265e95fd9 F src/whereexpr.c a182038cf7d10aa9a95a96b8563a34f34990323cf307dee6559e995961966d43 F src/window.c edd6f5e25a1e8f2b6f5305b7f5f7da7bb35f07f0d432b255b1d4c2fcab4205aa F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1064,7 +1064,7 @@ F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 F test/in.test 688ed2011d922d83141a45af431601738674a4c0bdde34b6351f688b82a169b3 F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 -F test/in4.test 65460600d48933adba4283c6ebd089aae173d16136ab9d01f74c89089090c5a5 +F test/in4.test 64ac9c767ac5af562f066a40163d4202f8fa3be05d264ec65d6258e74606b30c F test/in5.test b32ce7f4a93f44c5dee94af16886d922cc16ebe33c8e1765c73d4049d0f4b40f F test/in6.test 8562d0945195cab3cc4ab3794e9118e72cb44c43f785c2b04d48a9d06ca6b4ec F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822 @@ -1898,7 +1898,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7b2a7c7314d2239992dc1d707280f3b75ba75bb074ba1e77b55cad01ca7fd51b -R c809d928cb746073aedc3f5cce34c965 -U drh -Z 375659ad9a05c828e459b8457d7233ea +P 0571c24177d77ac966bcf42cb8ab00fdf541ce84af1468c8ef0b60c48c45a22f +R 4574041b82ab28e377188abbe28af4d5 +T *branch * fix-2d6e8400 +T *sym-fix-2d6e8400 * +T -sym-trunk * +U dan +Z e3d30b9ac2ed7bc4de1a15a8bab650b5 diff --git a/manifest.uuid b/manifest.uuid index 93a9669da3..8fc9323ad9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0571c24177d77ac966bcf42cb8ab00fdf541ce84af1468c8ef0b60c48c45a22f \ No newline at end of file +390cf60a286b13f454429f4652a133f95a7891a75c1ec6d16cd39990590fd3fb \ No newline at end of file diff --git a/src/wherecode.c b/src/wherecode.c index 4afe0ac9c9..17dc36152d 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1743,6 +1743,12 @@ Bitmask sqlite3WhereCodeOneLoopStart( SWAP(u8, nBtm, nTop); } + if( iLevel>0 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)!=0 ){ + /* In case OP_SeekScan is used, ensure that the index cursor does not + ** point to a valid row for the first iteration of this loop. */ + sqlite3VdbeAddOp1(v, OP_NullRow, iIdxCur); + } + /* Generate code to evaluate all constraint terms using == or IN ** and store the values of those terms in an array of registers ** starting at regBase. diff --git a/test/in4.test b/test/in4.test index 37eafdebec..bbb32d8a7a 100644 --- a/test/in4.test +++ b/test/in4.test @@ -365,4 +365,32 @@ ifcapable rtree { } {{} 1 {} {} {}} } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 8.0 { + CREATE TABLE t1(x INTEGER PRIMARY KEY, y); + CREATE UNIQUE INDEX t1y ON t1(y); + INSERT INTO t1 VALUES(111, 'AAA'),(222, 'BBB'),(333, 'CCC'); + CREATE TABLE t2(z); + INSERT INTO t2 VALUES('BBB'),('AAA'); + ANALYZE sqlite_schema; + INSERT INTO sqlite_stat1 VALUES('t1', 't1y','100 1'); +} + +db close +sqlite3 db test.db + +do_execsql_test 8.1 { + SELECT t1.x FROM t2 CROSS JOIN t1 WHERE t2.z = t1.y; +} {222 111} + +do_execsql_test 8.2 { + SELECT t1.x FROM t2 CROSS JOIN t1 WHERE t2.z = t1.y AND +t1.x IN (111, 222); +} {222 111} + +do_execsql_test 8.3 { + SELECT t1.x FROM t2 CROSS JOIN t1 WHERE t2.z = t1.y AND t1.x IN (111, 222); +} {222 111} + + finish_test