diff --git a/manifest b/manifest index cec9479716..65bf1958f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sversion\snumber\sto\s3.42.0\sto\sbegin\sthe\snext\sdevelopment\scycle. -D 2023-02-22T20:50:51.768 +C Avoid\scomputing\sthe\svalues\sfor\sunused\sresult-set\scolumns\sin\ssubqueries.\nPerformance\soptimization\srequest\s[baa5bb76c35a124c]. +D 2023-02-22T21:11:34.555 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,12 +621,12 @@ F src/printf.c ff4b05e38bf928ff1b80d3dda4f977b10fe39ecbfe69c018224c7e5594fb2455 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c f80d98f13fa64201ababf5e80d682c430559cd7ee00ab14ee99a1aeb27ac097d +F src/select.c 751d2f021cb439abdf3e8e7b74c5d0532ca9793868e4fb4c5a73412aea274edc F src/shell.c.in 6f36f5ca05f1bebf74935a7fcf2dce983016e807a09cbd752a673583ad7da087 F src/sqlite.h.in 662a2fa083d093896b92560c871dea6d86792b49dc4bf7b4e8dbeca8e7171488 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h 4dcc36b21dd0c77efeb0bb38b0d31916bc63fb429c8a89d186c945eba0816447 +F src/sqliteInt.h 25f38ade2d93828231d6cd8614f635c0b084bee004360d970542861ad2e3e3d8 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -1457,11 +1457,12 @@ F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d F test/select9.test f7586b207ce2304ab80dc93d3146469a28fd4403621dd3a82d06644563d3c812 F test/selectA.test 6aef8b2136a4ac7a3e2e4161d2b8ca7bc6ebe2779de084f9bb66ca9e2323a937 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 -F test/selectC.test fec14c9015ed4ec941508bbc144f30b42e40ac34a4bb33001450369865dd0b75 +F test/selectC.test 38c530b0cc5728b793c3c11f52b52c70290d39822224acd39011c89c1853bd31 F test/selectD.test 6d1909b49970bf92f45ce657505befcef5fc7cbc13544e18103a316d32189bfb F test/selectE.test a8730ca330fcf40ace158f134f4fe0eb00c7edbf F test/selectF.test 21c94e6438f76537b72532fa9fd4710cdd455fc3 F test/selectG.test 089f7d3d7e6db91566f00b036cb353107a2cca6220eb1cb264085a836dae8840 +F test/selectH.test a1fed72a83a68ed858db6fcf6d8ddccc4a368645c0ef48dc3ac331d41efe34de F test/session.test 78fa2365e93d3663a6e933f86e7afc395adf18be F test/sessionfuzz-data1.db 1f8d5def831f19b1c74571037f0d53a588ea49a6c4ca2a028fc0c27ef896dbcb F test/sessionfuzz.c 5eef09af01eeff6f20250ae4c0112c2e576e4d2f2026cc9a49dc5be6886fa6ee @@ -1918,7 +1919,7 @@ F test/windowerr.tcl f5acd6fbc210d7b5546c0e879d157888455cd4a17a1d3f28f07c1c8a387 F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3cc1819b F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c31660a7c F test/windowpushd.test d8895d08870b7226f7693665bd292eb177e62ca06799184957b3ca7dc03067df -F test/with1.test 9ad67fdeb2bbd808a5763c9060e214ea232f9b18d846ea3a59756dc38bdc3880 +F test/with1.test b93833890e5d2a368e78747f124503a0159aa029b98e9ed4795ebf630b2efd3d F test/with2.test a1df41b987198383b9b70bf5e5fda390582e46398653858dbc6ceb24253b28df F test/with3.test e7bf809bf75c1f44f98bca78bc331dbf542002c5227bf53c1261144db4e824c8 F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 @@ -2045,8 +2046,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P aa6bd6dff751223e302575c60f335707e4bb820bff716cab3706b564e18ed7ed -R 311ae6e51c5af4c947ddc5fe5e93f886 +P 65910216393ed287dbc7e42a2598b6980183c62a64da6572cb691ff4ee23555d cf8f57c53425d89619ece10edc197d7d28946d3b23fcb4f526330196c76d9cc1 +R 34649150ad57f18afb1178077e5cdae5 +T +closed cf8f57c53425d89619ece10edc197d7d28946d3b23fcb4f526330196c76d9cc1 U drh -Z 558b030af06c9bbcf0267311941c1842 +Z c744fac449b59dcb5adcaf0108c5c30e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fe2ab7c19f..4b12d53327 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -65910216393ed287dbc7e42a2598b6980183c62a64da6572cb691ff4ee23555d \ No newline at end of file +7c2d3406000dc8ac5a99cc205b036356b67e4b0b94738592ffc5680749696904 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 23099b1397..3095685a8f 100644 --- a/src/select.c +++ b/src/select.c @@ -5227,6 +5227,76 @@ static int pushDownWhereTerms( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +/* +** Check to see if a subquery contains result-set columns that are +** never used. If it does, change the value of those result-set columns +** to NULL so that they do not cause unnecessary work to compute. +** +** Return the number of column that were changed to NULL. +*/ +static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ + int nCol; + Select *pSub; /* The subquery to be simplified */ + Select *pX; /* For looping over compound elements of pSub */ + Table *pTab; /* The table that describes the subquery */ + int j; /* Column number */ + int nChng = 0; /* Number of columns converted to NULL */ + Bitmask colUsed; /* Columns that may not be NULLed out */ + + assert( pItem!=0 ); + if( pItem->fg.isCorrelated || pItem->fg.isCte ){ + return 0; + } + assert( pItem->pTab!=0 ); + pTab = pItem->pTab; + assert( pItem->pSelect!=0 ); + pSub = pItem->pSelect; + assert( pSub->pEList->nExpr==pTab->nCol ); + if( (pSub->selFlags & (SF_Distinct|SF_Aggregate))!=0 ){ + testcase( pSub->selFlags & SF_Distinct ); + testcase( pSub->selFlags & SF_Aggregate ); + return 0; + } + for(pX=pSub; pX; pX=pX->pPrior){ + if( pX->pPrior && pX->op!=TK_ALL ){ + /* This optimization does not work for compound subqueries that + ** use UNION, INTERSECT, or EXCEPT. Only UNION ALL is allowed. */ + return 0; + } + if( pX->pWin ){ + /* This optimization does not work for subqueries that use window + ** functions. */ + return 0; + } + } + colUsed = pItem->colUsed; + if( pSub->pOrderBy ){ + ExprList *pList = pSub->pOrderBy; + for(j=0; jnExpr; j++){ + u16 iCol = pList->a[j].u.x.iOrderByCol; + if( iCol>0 ){ + iCol--; + colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); + } + } + } + nCol = pTab->nCol; + for(j=0; jpPrior) { + Expr *pY = pX->pEList->a[j].pExpr; + if( pY->op==TK_NULL ) continue; + pY->op = TK_NULL; + pX->selFlags |= SF_PushDown; + nChng++; + } + } + return nChng; +} + + /* ** The pFunc is the only aggregate function in the query. Check to see ** if the query is a candidate for the min/max optimization. @@ -7326,6 +7396,22 @@ int sqlite3Select( TREETRACE(0x4000,pParse,p,("Push-down not possible\n")); } + /* Convert unused result columns of the subquery into simple NULL + ** expressions, to avoid unneeded searching and computation. + */ + if( OptimizationEnabled(db, SQLITE_NullUnusedCols) + && disableUnusedSubqueryResultColumns(pItem) + ){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x4000 ){ + TREETRACE(0x4000,pParse,p, + ("Change unused result columns to NULL for subquery %d:\n", + pSub->selId)); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + } + zSavedAuthContext = pParse->zAuthContext; pParse->zAuthContext = pItem->zName; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9af38e8df4..459af6b4ed 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1839,6 +1839,7 @@ struct sqlite3 { /* TH3 expects this value ^^^^^^^^^^ See flatten04.test */ #define SQLITE_IndexedExpr 0x01000000 /* Pull exprs from index when able */ #define SQLITE_Coroutines 0x02000000 /* Co-routines for subqueries */ +#define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* diff --git a/test/selectC.test b/test/selectC.test index 63851ca5d7..42fa1d11b8 100644 --- a/test/selectC.test +++ b/test/selectC.test @@ -230,6 +230,11 @@ do_execsql_test selectC-4.2 { select a from (select distinct a, b from t_distinct_bug) } {1 1 1} +do_execsql_test selectC-4.2b { + CREATE VIEW v42b AS SELECT DISTINCT a, b FROM t_distinct_bug; + SELECT a FROM v42b; +} {1 1 1} + do_execsql_test selectC-4.3 { select a, udf() from (select distinct a, b from t_distinct_bug) } {1 1 1 2 1 3} diff --git a/test/selectH.test b/test/selectH.test new file mode 100644 index 0000000000..f621fc302c --- /dev/null +++ b/test/selectH.test @@ -0,0 +1,110 @@ +# 2023-02-16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the omit-unused-subquery-column optimization. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix selectH + +do_execsql_test 1.1 { + CREATE TABLE t1( + c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, + c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, + c20, c21, c22, c23, c24, c25, c26, c27, c28, c29, + c30, c31, c32, c33, c34, c35, c36, c37, c38, c39, + c40, c41, c42, c43, c44, c45, c46, c47, c48, c49, + c50, c51, c52, c53, c54, c55, c56, c57, c58, c59, + c60, c61, c62, c63, c64, c65 + ); + INSERT INTO t1 VALUES( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65 + ); + CREATE INDEX t1c60 ON t1(c60); +} + +# The SQL counter(N) function adjusts the value of the global +# TCL variable ::selectH_cnt by the value N and returns the new +# value. By putting calls to counter(N) as unused columns in a +# view or subquery, we can check to see if the counter gets incremented, +# and if not that means that the unused column was omitted. +# +unset -nocomplain selectH_cnt +set selectH_cnt 0 +proc selectH_counter {amt} { + global selectH_cnt + incr selectH_cnt $amt + return $selectH_cnt +} +db func counter selectH_counter + +do_execsql_test 1.2 { + SELECT DISTINCT c44 FROM ( + SELECT c0 AS a, *, counter(1) FROM t1 + UNION ALL + SELECT c1 AS a, *, counter(1) FROM t1 + ) WHERE c60=60; +} {44} +do_test 1.3 { + set ::selectH_cnt +} {0} + +do_execsql_test 2.1 { + SELECT a FROM ( + SELECT counter(1) AS cnt, c15 AS a, *, c62 AS b FROM t1 + UNION ALL + SELECT counter(1) AS cnt, c16 AS a, *, c61 AS b FROM t1 + ORDER BY b + ); +} {16 15} +do_test 2.2 { + set ::selectH_cnt +} {0} + +do_execsql_test 3.1 { + CREATE VIEW v1 AS + SELECT c16 AS a, *, counter(1) AS x FROM t1 + UNION ALL + SELECT c17 AS a, *, counter(1) AS x FROM t1 + UNION ALL + SELECT c18 AS a, *, counter(1) AS x FROM t1 + UNION ALL + SELECT c19 AS a, *, counter(1) AS x FROM t1; + SELECT count(*) FROM v1 WHERE c60=60; +} {4} +do_test 3.2 { + set ::selectH_cnt +} {0} +do_execsql_test 3.3 { + SELECT count(a) FROM v1 WHERE c60=60; +} {4} +do_execsql_test 3.4 { + SELECT a FROM v1 WHERE c60=60; +} {16 17 18 19} +do_test 3.5 { + set ::selectH_cnt +} {0} +do_execsql_test 3.6 { + SELECT x FROM v1 WHERE c60=60; +} {1 2 3 4} +do_test 3.7 { + set ::selectH_cnt +} {4} + + +finish_test diff --git a/test/with1.test b/test/with1.test index 91cbc79524..7400a7adf3 100644 --- a/test/with1.test +++ b/test/with1.test @@ -1124,7 +1124,7 @@ do_execsql_test 24.1 { CREATE VIEW v1 AS SELECT max(a), min(b) FROM t1 GROUP BY c; } do_test 24.1 { - set program [db eval {EXPLAIN SELECT 1 FROM v1,v1,v1}] + set program [db eval {EXPLAIN SELECT * FROM v1 AS aa, v1 AS bb, v1 AS cc}] expr [lsearch $program OpenDup]>0 } {1} do_execsql_test 24.2 {