From e3ec00ccb8ea49dbe8c99fd5e5967dadaddd9852 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 15 Feb 2023 17:53:17 +0000 Subject: [PATCH 1/5] Do not compute result columns of subqueries that are never used. Make those columns NULL instead. This optimization potentially resolves the enhancement request described by [/tktview/baa5bb76c35a124c|ticket baa5bb76c35a124c]. FossilOrigin-Name: 5dec3cc0225296a043d17f73126d477d90a604f82b3180628176d8f950adbce8 --- manifest | 19 +++++++++------ manifest.uuid | 2 +- src/select.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++++ test/with1.test | 2 +- 4 files changed, 78 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 91f4eba3e3..a52c37a9b2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\stestrunner.tcl\sto\srun\szipvfs\stest\sscripts\son\sunix. -D 2023-02-14T18:09:40.825 +C Do\snot\scompute\sresult\scolumns\sof\ssubqueries\sthat\sare\snever\sused.\s\sMake\sthose\ncolumns\sNULL\sinstead.\s\sThis\soptimization\spotentially\sresolves\sthe\senhancement\nrequest\sdescribed\sby\s[/tktview/baa5bb76c35a124c|ticket\sbaa5bb76c35a124c]. +D 2023-02-15T17:53:17.086 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,7 +621,7 @@ F src/printf.c ff4b05e38bf928ff1b80d3dda4f977b10fe39ecbfe69c018224c7e5594fb2455 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c c3ce1b49cca2c66c8c88fe7d9e1f3db23590deb4dd631619ad90e1e5d21bcf1f +F src/select.c 49d9336c4108a765831b2483d17cf52e196caaf341cba95646a220ec70ae080f F src/shell.c.in 5cb750dc00bb8b20b020031fd1bb0cb8759a6e567c27aee6b0af31d187984985 F src/sqlite.h.in c060f9e6f646fd817abf273c068666e079fc6e8467ea1d3e8b20f355f82722e9 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1918,7 +1918,7 @@ F test/windowerr.tcl f5acd6fbc210d7b5546c0e879d157888455cd4a17a1d3f28f07c1c8a387 F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3cc1819b F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c31660a7c F test/windowpushd.test d8895d08870b7226f7693665bd292eb177e62ca06799184957b3ca7dc03067df -F test/with1.test 9ad67fdeb2bbd808a5763c9060e214ea232f9b18d846ea3a59756dc38bdc3880 +F test/with1.test b93833890e5d2a368e78747f124503a0159aa029b98e9ed4795ebf630b2efd3d F test/with2.test a1df41b987198383b9b70bf5e5fda390582e46398653858dbc6ceb24253b28df F test/with3.test e7bf809bf75c1f44f98bca78bc331dbf542002c5227bf53c1261144db4e824c8 F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 @@ -2045,8 +2045,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ecdeef43b27412b0b0b09e09a62ad3a03836a3fc80f2070268090e7ca8f02712 -R 008d808935e4782501905c43f1b48b8b -U dan -Z 0a26d7ad99fa8ab0735a6c8903ff6006 +P e6c8e19ab0d6e7526d4596b75a45bb6becaf3c029690f7e75c016eac803c9990 +R 790a0333fb08765177eb98bd169ddbf7 +T *branch * omit-unused-subquery-columns +T *sym-omit-unused-subquery-columns * +T -sym-trunk * +U drh +Z e4cd68bda867ee6d3fbf83551561c523 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d9053da512..715b2f5ee3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e6c8e19ab0d6e7526d4596b75a45bb6becaf3c029690f7e75c016eac803c9990 \ No newline at end of file +5dec3cc0225296a043d17f73126d477d90a604f82b3180628176d8f950adbce8 \ No newline at end of file diff --git a/src/select.c b/src/select.c index b0e303066a..4958ddefe9 100644 --- a/src/select.c +++ b/src/select.c @@ -5235,6 +5235,57 @@ static int pushDownWhereTerms( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +/* +** Check to see if a subquery contains result-set columns that are +** never used. If it does, change the value of those result-set columns +** to NULL so that they do not cause unnecessary work to compute. +** +** Return the number of column that were changed to NULL. +*/ +static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ + int nCol; + Select *pSub; /* The subquery to be simplified */ + Select *pX; /* For looping over compound elements of pSub */ + Table *pTab; /* The table that describes the subquery */ + int j; /* Column number */ + int nChng = 0; /* Number of columns converted to NULL */ + + assert( pItem!=0 ); + assert( pItem->pTab!=0 ); + pTab = pItem->pTab; + if( pTab->tabFlags & TF_Ephemeral ) return 0; + assert( pItem->pSelect!=0 ); + pSub = pItem->pSelect; + assert( pSub->pEList->nExpr==pTab->nCol ); + for(pX=pSub; pX; pX=pX->pPrior){ + if( pX->pPrior && pX->op!=TK_ALL ){ + /* This optimization does not work for compound subqueries that + ** use UNION, INTERSECT, or EXCEPT. Only UNION ALL is allowed. */ + return 0; + } + if( pX->pWin ){ + /* This optimization does not work for subqueries that use window + ** functions. */ + return 0; + } + } + nCol = pTab->nCol; + for(j=0; jcolUsed)!=0 ) continue; + if( MASKBIT(j)==0 ) break; + for(pX=pSub; pX; pX=pX->pPrior) { + Expr *pY = pX->pEList->a[j].pExpr; + if( pY->op==TK_NULL ) continue; + pY->op = TK_NULL; + pX->selFlags |= SF_PushDown; + nChng++; + } + } + return nChng; +} + + /* ** The pFunc is the only aggregate function in the query. Check to see ** if the query is a candidate for the min/max optimization. @@ -7333,6 +7384,20 @@ int sqlite3Select( TREETRACE(0x4000,pParse,p,("Push-down not possible\n")); } + /* Convert unused result columns of the subquery into simple NULL + ** expressions, to avoid unneeded searching and computation. + */ + if( disableUnusedSubqueryResultColumns(pItem) ){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x4000 ){ + TREETRACE(0x4000,pParse,p, + ("Change unused result columns to NULL for subquery %d:\n", + pSub->selId)); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + } + zSavedAuthContext = pParse->zAuthContext; pParse->zAuthContext = pItem->zName; diff --git a/test/with1.test b/test/with1.test index 91cbc79524..7400a7adf3 100644 --- a/test/with1.test +++ b/test/with1.test @@ -1124,7 +1124,7 @@ do_execsql_test 24.1 { CREATE VIEW v1 AS SELECT max(a), min(b) FROM t1 GROUP BY c; } do_test 24.1 { - set program [db eval {EXPLAIN SELECT 1 FROM v1,v1,v1}] + set program [db eval {EXPLAIN SELECT * FROM v1 AS aa, v1 AS bb, v1 AS cc}] expr [lsearch $program OpenDup]>0 } {1} do_execsql_test 24.2 { From 6945ba787f11eb165ec9f25c359b4c91993643a5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 15 Feb 2023 19:53:08 +0000 Subject: [PATCH 2/5] Fix the subquery result column NULL-ifier so that it correctly handles subquery columns past the 63rd column. FossilOrigin-Name: 77b220a7240425fa83e142ceef78505208e6e38d797070e146b9f7d255f753c7 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/select.c | 4 ++-- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index a52c37a9b2..38e7b2731d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\scompute\sresult\scolumns\sof\ssubqueries\sthat\sare\snever\sused.\s\sMake\sthose\ncolumns\sNULL\sinstead.\s\sThis\soptimization\spotentially\sresolves\sthe\senhancement\nrequest\sdescribed\sby\s[/tktview/baa5bb76c35a124c|ticket\sbaa5bb76c35a124c]. -D 2023-02-15T17:53:17.086 +C Fix\sthe\ssubquery\sresult\scolumn\sNULL-ifier\sso\sthat\sit\scorrectly\shandles\nsubquery\scolumns\spast\sthe\s63rd\scolumn. +D 2023-02-15T19:53:08.659 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,7 +621,7 @@ F src/printf.c ff4b05e38bf928ff1b80d3dda4f977b10fe39ecbfe69c018224c7e5594fb2455 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 49d9336c4108a765831b2483d17cf52e196caaf341cba95646a220ec70ae080f +F src/select.c 98984bd8a2f00ae6e2a1c33991617c0d0db68f204ea124cdd853383204499697 F src/shell.c.in 5cb750dc00bb8b20b020031fd1bb0cb8759a6e567c27aee6b0af31d187984985 F src/sqlite.h.in c060f9e6f646fd817abf273c068666e079fc6e8467ea1d3e8b20f355f82722e9 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2045,11 +2045,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e6c8e19ab0d6e7526d4596b75a45bb6becaf3c029690f7e75c016eac803c9990 -R 790a0333fb08765177eb98bd169ddbf7 -T *branch * omit-unused-subquery-columns -T *sym-omit-unused-subquery-columns * -T -sym-trunk * +P 5dec3cc0225296a043d17f73126d477d90a604f82b3180628176d8f950adbce8 +R d380059ff4fc01bc81597a53152668a1 U drh -Z e4cd68bda867ee6d3fbf83551561c523 +Z d76a4bd1d7943a5eace66bda083496dd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 715b2f5ee3..149e343a10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5dec3cc0225296a043d17f73126d477d90a604f82b3180628176d8f950adbce8 \ No newline at end of file +77b220a7240425fa83e142ceef78505208e6e38d797070e146b9f7d255f753c7 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 4958ddefe9..dc187dfa6a 100644 --- a/src/select.c +++ b/src/select.c @@ -5272,8 +5272,8 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ nCol = pTab->nCol; for(j=0; jcolUsed)!=0 ) continue; - if( MASKBIT(j)==0 ) break; + Bitmask m = jcolUsed)!=0 ) continue; for(pX=pSub; pX; pX=pX->pPrior) { Expr *pY = pX->pEList->a[j].pExpr; if( pY->op==TK_NULL ) continue; From 90b7af77152f2b39ee3de40777487862a83597f3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 16 Feb 2023 15:54:55 +0000 Subject: [PATCH 3/5] Do not perform the omit-unused-subquery-columns optimizations on a subquery that is DISTINCT, as that can lead to incorrect results. FossilOrigin-Name: cc148503db8ef180bce984328da7e84959afadd6a9613c2d03bc1eafeb95dfad --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 7 ++++++- test/selectC.test | 5 +++++ 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 38e7b2731d..4067e40b48 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\ssubquery\sresult\scolumn\sNULL-ifier\sso\sthat\sit\scorrectly\shandles\nsubquery\scolumns\spast\sthe\s63rd\scolumn. -D 2023-02-15T19:53:08.659 +C Do\snot\sperform\sthe\somit-unused-subquery-columns\soptimizations\son\sa\nsubquery\sthat\sis\sDISTINCT,\sas\sthat\scan\slead\sto\sincorrect\sresults. +D 2023-02-16T15:54:55.738 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,7 +621,7 @@ F src/printf.c ff4b05e38bf928ff1b80d3dda4f977b10fe39ecbfe69c018224c7e5594fb2455 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 98984bd8a2f00ae6e2a1c33991617c0d0db68f204ea124cdd853383204499697 +F src/select.c a723a35366d7fc695371dd80b2ea4dbc306d5fb75df81d3e6ce7e200794f7622 F src/shell.c.in 5cb750dc00bb8b20b020031fd1bb0cb8759a6e567c27aee6b0af31d187984985 F src/sqlite.h.in c060f9e6f646fd817abf273c068666e079fc6e8467ea1d3e8b20f355f82722e9 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1457,7 +1457,7 @@ F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d F test/select9.test f7586b207ce2304ab80dc93d3146469a28fd4403621dd3a82d06644563d3c812 F test/selectA.test 6aef8b2136a4ac7a3e2e4161d2b8ca7bc6ebe2779de084f9bb66ca9e2323a937 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 -F test/selectC.test fec14c9015ed4ec941508bbc144f30b42e40ac34a4bb33001450369865dd0b75 +F test/selectC.test 38c530b0cc5728b793c3c11f52b52c70290d39822224acd39011c89c1853bd31 F test/selectD.test 6d1909b49970bf92f45ce657505befcef5fc7cbc13544e18103a316d32189bfb F test/selectE.test a8730ca330fcf40ace158f134f4fe0eb00c7edbf F test/selectF.test 21c94e6438f76537b72532fa9fd4710cdd455fc3 @@ -2045,8 +2045,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5dec3cc0225296a043d17f73126d477d90a604f82b3180628176d8f950adbce8 -R d380059ff4fc01bc81597a53152668a1 +P 77b220a7240425fa83e142ceef78505208e6e38d797070e146b9f7d255f753c7 +R e479bead84e886f8b2c21d0d027a081a U drh -Z d76a4bd1d7943a5eace66bda083496dd +Z 0b93f878d511033e2fa0d6dd3c9e8431 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 149e343a10..3b17642550 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -77b220a7240425fa83e142ceef78505208e6e38d797070e146b9f7d255f753c7 \ No newline at end of file +cc148503db8ef180bce984328da7e84959afadd6a9613c2d03bc1eafeb95dfad \ No newline at end of file diff --git a/src/select.c b/src/select.c index dc187dfa6a..ae320b697b 100644 --- a/src/select.c +++ b/src/select.c @@ -5253,10 +5253,15 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ assert( pItem!=0 ); assert( pItem->pTab!=0 ); pTab = pItem->pTab; - if( pTab->tabFlags & TF_Ephemeral ) return 0; + if( pTab->tabFlags & TF_Ephemeral ){ + return 0; + } assert( pItem->pSelect!=0 ); pSub = pItem->pSelect; assert( pSub->pEList->nExpr==pTab->nCol ); + if( pSub->selFlags & SF_Distinct ){ + return 0; + } for(pX=pSub; pX; pX=pX->pPrior){ if( pX->pPrior && pX->op!=TK_ALL ){ /* This optimization does not work for compound subqueries that diff --git a/test/selectC.test b/test/selectC.test index 63851ca5d7..42fa1d11b8 100644 --- a/test/selectC.test +++ b/test/selectC.test @@ -230,6 +230,11 @@ do_execsql_test selectC-4.2 { select a from (select distinct a, b from t_distinct_bug) } {1 1 1} +do_execsql_test selectC-4.2b { + CREATE VIEW v42b AS SELECT DISTINCT a, b FROM t_distinct_bug; + SELECT a FROM v42b; +} {1 1 1} + do_execsql_test selectC-4.3 { select a, udf() from (select distinct a, b from t_distinct_bug) } {1 1 1 2 1 3} From 7defd20cb0ac85b629ad5b2a7db235464838b193 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 16 Feb 2023 18:04:49 +0000 Subject: [PATCH 4/5] Provide an optimization-disable mask for this optimization. Do not do the optimization if the subquery is an aggregate or is distinct, but allow it to be an ephemeral subquery. Do not omit columns that are used in the ORDER BY of the subquery. FossilOrigin-Name: 6b1a1f374d1372f11f5420d99645b218867100bf070bd3a8885bf5f00c189dff --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 28 ++++++++++++++++++++++------ src/sqliteInt.h | 1 + 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 4067e40b48..75fd2e6d0e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sperform\sthe\somit-unused-subquery-columns\soptimizations\son\sa\nsubquery\sthat\sis\sDISTINCT,\sas\sthat\scan\slead\sto\sincorrect\sresults. -D 2023-02-16T15:54:55.738 +C Provide\san\soptimization-disable\smask\sfor\sthis\soptimization.\s\sDo\snot\sdo\sthe\noptimization\sif\sthe\ssubquery\sis\san\saggregate\sor\sis\sdistinct,\sbut\sallow\sit\sto\nbe\san\sephemeral\ssubquery.\s\sDo\snot\somit\scolumns\sthat\sare\sused\sin\sthe\sORDER\sBY\nof\sthe\ssubquery. +D 2023-02-16T18:04:49.549 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,12 +621,12 @@ F src/printf.c ff4b05e38bf928ff1b80d3dda4f977b10fe39ecbfe69c018224c7e5594fb2455 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c a723a35366d7fc695371dd80b2ea4dbc306d5fb75df81d3e6ce7e200794f7622 +F src/select.c fa66a61608731b4842fb2a2799ddeccbb56b4ec863f8b36d8b08b898ff62067a F src/shell.c.in 5cb750dc00bb8b20b020031fd1bb0cb8759a6e567c27aee6b0af31d187984985 F src/sqlite.h.in c060f9e6f646fd817abf273c068666e079fc6e8467ea1d3e8b20f355f82722e9 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h 4dcc36b21dd0c77efeb0bb38b0d31916bc63fb429c8a89d186c945eba0816447 +F src/sqliteInt.h 25f38ade2d93828231d6cd8614f635c0b084bee004360d970542861ad2e3e3d8 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2045,8 +2045,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 77b220a7240425fa83e142ceef78505208e6e38d797070e146b9f7d255f753c7 -R e479bead84e886f8b2c21d0d027a081a +P cc148503db8ef180bce984328da7e84959afadd6a9613c2d03bc1eafeb95dfad +R 90e53b2f21c8456ed0ad4d26c75d661a U drh -Z 0b93f878d511033e2fa0d6dd3c9e8431 +Z aa83be3d588d7cfee260c5a1a1ab587b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3b17642550..1e9696bac9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cc148503db8ef180bce984328da7e84959afadd6a9613c2d03bc1eafeb95dfad \ No newline at end of file +6b1a1f374d1372f11f5420d99645b218867100bf070bd3a8885bf5f00c189dff \ No newline at end of file diff --git a/src/select.c b/src/select.c index ae320b697b..b8cae155f3 100644 --- a/src/select.c +++ b/src/select.c @@ -5249,17 +5249,20 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ Table *pTab; /* The table that describes the subquery */ int j; /* Column number */ int nChng = 0; /* Number of columns converted to NULL */ + Bitmask colUsed; /* Columns that may not be NULLed out */ assert( pItem!=0 ); - assert( pItem->pTab!=0 ); - pTab = pItem->pTab; - if( pTab->tabFlags & TF_Ephemeral ){ + if( pItem->fg.isCorrelated || pItem->fg.isCte ){ return 0; } + assert( pItem->pTab!=0 ); + pTab = pItem->pTab; assert( pItem->pSelect!=0 ); pSub = pItem->pSelect; assert( pSub->pEList->nExpr==pTab->nCol ); - if( pSub->selFlags & SF_Distinct ){ + if( (pSub->selFlags & (SF_Distinct|SF_Aggregate))!=0 ){ + testcase( pSub->selFlags & SF_Distinct ); + testcase( pSub->selFlags & SF_Aggregate ); return 0; } for(pX=pSub; pX; pX=pX->pPrior){ @@ -5274,11 +5277,22 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ return 0; } } + colUsed = pItem->colUsed; + if( pSub->pOrderBy ){ + ExprList *pList = pSub->pOrderBy; + for(j=0; jnExpr; j++){ + u16 iCol = pList->a[j].u.x.iOrderByCol; + if( iCol>0 ){ + iCol--; + colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); + } + } + } nCol = pTab->nCol; for(j=0; jcolUsed)!=0 ) continue; + if( (m & colUsed)!=0 ) continue; for(pX=pSub; pX; pX=pX->pPrior) { Expr *pY = pX->pEList->a[j].pExpr; if( pY->op==TK_NULL ) continue; @@ -7392,7 +7406,9 @@ int sqlite3Select( /* Convert unused result columns of the subquery into simple NULL ** expressions, to avoid unneeded searching and computation. */ - if( disableUnusedSubqueryResultColumns(pItem) ){ + if( OptimizationEnabled(db, SQLITE_NullUnusedCols) + && disableUnusedSubqueryResultColumns(pItem) + ){ #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x4000 ){ TREETRACE(0x4000,pParse,p, diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9af38e8df4..459af6b4ed 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1839,6 +1839,7 @@ struct sqlite3 { /* TH3 expects this value ^^^^^^^^^^ See flatten04.test */ #define SQLITE_IndexedExpr 0x01000000 /* Pull exprs from index when able */ #define SQLITE_Coroutines 0x02000000 /* Co-routines for subqueries */ +#define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* From e43d101c83bf30b743d74d07eb63a286c90e0724 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 16 Feb 2023 19:41:39 +0000 Subject: [PATCH 5/5] A few simple test cases for the omit-unused-subquery-column optimization. FossilOrigin-Name: cf8f57c53425d89619ece10edc197d7d28946d3b23fcb4f526330196c76d9cc1 --- manifest | 11 ++--- manifest.uuid | 2 +- test/selectH.test | 110 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 6 deletions(-) create mode 100644 test/selectH.test diff --git a/manifest b/manifest index 75fd2e6d0e..8350761a55 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Provide\san\soptimization-disable\smask\sfor\sthis\soptimization.\s\sDo\snot\sdo\sthe\noptimization\sif\sthe\ssubquery\sis\san\saggregate\sor\sis\sdistinct,\sbut\sallow\sit\sto\nbe\san\sephemeral\ssubquery.\s\sDo\snot\somit\scolumns\sthat\sare\sused\sin\sthe\sORDER\sBY\nof\sthe\ssubquery. -D 2023-02-16T18:04:49.549 +C A\sfew\ssimple\stest\scases\sfor\sthe\somit-unused-subquery-column\soptimization. +D 2023-02-16T19:41:39.707 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1462,6 +1462,7 @@ F test/selectD.test 6d1909b49970bf92f45ce657505befcef5fc7cbc13544e18103a316d3218 F test/selectE.test a8730ca330fcf40ace158f134f4fe0eb00c7edbf F test/selectF.test 21c94e6438f76537b72532fa9fd4710cdd455fc3 F test/selectG.test 089f7d3d7e6db91566f00b036cb353107a2cca6220eb1cb264085a836dae8840 +F test/selectH.test a1fed72a83a68ed858db6fcf6d8ddccc4a368645c0ef48dc3ac331d41efe34de F test/session.test 78fa2365e93d3663a6e933f86e7afc395adf18be F test/sessionfuzz-data1.db 1f8d5def831f19b1c74571037f0d53a588ea49a6c4ca2a028fc0c27ef896dbcb F test/sessionfuzz.c 5eef09af01eeff6f20250ae4c0112c2e576e4d2f2026cc9a49dc5be6886fa6ee @@ -2045,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cc148503db8ef180bce984328da7e84959afadd6a9613c2d03bc1eafeb95dfad -R 90e53b2f21c8456ed0ad4d26c75d661a +P 6b1a1f374d1372f11f5420d99645b218867100bf070bd3a8885bf5f00c189dff +R 1e3c678f9fbc9f1a9649605265913343 U drh -Z aa83be3d588d7cfee260c5a1a1ab587b +Z 7d95cef9a9f7a288c86a6fe2f6e032bb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1e9696bac9..3216d50cee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6b1a1f374d1372f11f5420d99645b218867100bf070bd3a8885bf5f00c189dff \ No newline at end of file +cf8f57c53425d89619ece10edc197d7d28946d3b23fcb4f526330196c76d9cc1 \ No newline at end of file diff --git a/test/selectH.test b/test/selectH.test new file mode 100644 index 0000000000..f621fc302c --- /dev/null +++ b/test/selectH.test @@ -0,0 +1,110 @@ +# 2023-02-16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the omit-unused-subquery-column optimization. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix selectH + +do_execsql_test 1.1 { + CREATE TABLE t1( + c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, + c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, + c20, c21, c22, c23, c24, c25, c26, c27, c28, c29, + c30, c31, c32, c33, c34, c35, c36, c37, c38, c39, + c40, c41, c42, c43, c44, c45, c46, c47, c48, c49, + c50, c51, c52, c53, c54, c55, c56, c57, c58, c59, + c60, c61, c62, c63, c64, c65 + ); + INSERT INTO t1 VALUES( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65 + ); + CREATE INDEX t1c60 ON t1(c60); +} + +# The SQL counter(N) function adjusts the value of the global +# TCL variable ::selectH_cnt by the value N and returns the new +# value. By putting calls to counter(N) as unused columns in a +# view or subquery, we can check to see if the counter gets incremented, +# and if not that means that the unused column was omitted. +# +unset -nocomplain selectH_cnt +set selectH_cnt 0 +proc selectH_counter {amt} { + global selectH_cnt + incr selectH_cnt $amt + return $selectH_cnt +} +db func counter selectH_counter + +do_execsql_test 1.2 { + SELECT DISTINCT c44 FROM ( + SELECT c0 AS a, *, counter(1) FROM t1 + UNION ALL + SELECT c1 AS a, *, counter(1) FROM t1 + ) WHERE c60=60; +} {44} +do_test 1.3 { + set ::selectH_cnt +} {0} + +do_execsql_test 2.1 { + SELECT a FROM ( + SELECT counter(1) AS cnt, c15 AS a, *, c62 AS b FROM t1 + UNION ALL + SELECT counter(1) AS cnt, c16 AS a, *, c61 AS b FROM t1 + ORDER BY b + ); +} {16 15} +do_test 2.2 { + set ::selectH_cnt +} {0} + +do_execsql_test 3.1 { + CREATE VIEW v1 AS + SELECT c16 AS a, *, counter(1) AS x FROM t1 + UNION ALL + SELECT c17 AS a, *, counter(1) AS x FROM t1 + UNION ALL + SELECT c18 AS a, *, counter(1) AS x FROM t1 + UNION ALL + SELECT c19 AS a, *, counter(1) AS x FROM t1; + SELECT count(*) FROM v1 WHERE c60=60; +} {4} +do_test 3.2 { + set ::selectH_cnt +} {0} +do_execsql_test 3.3 { + SELECT count(a) FROM v1 WHERE c60=60; +} {4} +do_execsql_test 3.4 { + SELECT a FROM v1 WHERE c60=60; +} {16 17 18 19} +do_test 3.5 { + set ::selectH_cnt +} {0} +do_execsql_test 3.6 { + SELECT x FROM v1 WHERE c60=60; +} {1 2 3 4} +do_test 3.7 { + set ::selectH_cnt +} {4} + + +finish_test