diff --git a/manifest b/manifest index 8266e85012..fce3efb027 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s--valid-sql\soption\sto\sthe\soptfuzz\stest\sprogram. -D 2018-03-22T11:28:31.684 +C Add\sthe\sleft\sjoin\sstrength\sreduction\soptimization.\s\sEnhance\sthe\spush-down\noptimization\sso\sthat\sit\sworks\swith\smany\sLEFT\sJOINs. +D 2018-03-22T12:00:43.103 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7016fc56c6b9bfe5daac4f34be8be38d8c0b5fab79ccbfb764d3b23bf1c6fff3 @@ -441,7 +441,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b -F src/expr.c 904f85e5f5fa232488cbb662fcf3306ae203728751468670fb9840a9b1886abc +F src/expr.c 21ba8e1f8b9bc035053941d4f8df3ca5376be5e63ff62f945c2bc4f9b8890362 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51 @@ -489,12 +489,12 @@ F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 66c73fcb7719b8ff0e841b58338f13604ff3e2b50a723f9b8f383595735262f6 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 69ed6ca9d1bf4adb41472b0740b7b90868438dedcdd5ab2fd608070eca30d073 +F src/select.c 2c408ab9b4bfc807e6a9d3d45337d20ebe72a8b7ac793854710bf9d572e7ab6d F src/shell.c.in 911b9e3bce40413c78fdba28efa28363e98183819bd4b300780bf57bacfc4b84 F src/sqlite.h.in 19762b57baa1ade67531f254de94374428fb9c82452ef305017847945f9c2911 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d -F src/sqliteInt.h 7e9deb145c110289f50fcf72a8742d030834885676482391fe83c3cee75f97d4 +F src/sqliteInt.h 541896393a2ac07de571bcb0ce05b9422b94a6fb3900b005fa095d70b4d9545a F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -736,7 +736,7 @@ F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/csv01.test 526fc6aefd052badd5a0283f86b4b395c3df76bfe98d96c801f494f5e2c7836c F test/ctime.test 78749e6c9a5f0010d67985be80788f841e3cd2da18114e2ed6010399a7d807f3 F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856 -F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b +F test/cursorhint2.test 0078ae1ded4afcf5eb80d06e3a72b6e1c3f1a646aab26eeb583b0a9ec6f0d56e F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e @@ -771,7 +771,7 @@ F test/e_fts3.test 8cf40550bb088a6aa187c818c00fabe26ef82900a4cd5c66b427ccafe28be F test/e_insert.test f02f7f17852b2163732c6611d193f84fc67bc641fb4882c77a464076e5eba80e F test/e_reindex.test 2bebf7b393e519198b7c654407221cf171a439b8 F test/e_resolve.test a61751c368b109db73df0f20fc75fb47e166b1d8 -F test/e_select.test 16651bb681e83a1a2875ff4a595ed2b4b4dee375 +F test/e_select.test 6fd45fd4a59ec82b6dda7468699dcc0ec1a72538577750b4f90357a62c1d2723 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f F test/e_totalchanges.test b12ee5809d3e63aeb83238dd501a7bca7fd72c10 F test/e_update.test f46c2554d915c9197548681e8d8c33a267e84528 @@ -1004,7 +1004,7 @@ F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test 2d89873d72248b7a0ef7b5457120100cce75a19181bcf4b30e90a3f96a5c5558 F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678 -F test/join2.test f1b5ba94ba70a791df2d99e61adcebf62edacad1eca57881c97eb3e2ca718595 +F test/join2.test 2339e3a8089178607a718c7fa1f6c3826579132396f5304a0f8e9dbefa60c84f F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test c6bd62effc37a152bea735f9ef241b19bb967bd4593dc99b20e2fc55ae707e38 @@ -1715,7 +1715,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 21346bbce9fd161e8a2037834a6e8eb443a901109ddb3a52c1b7a29000ffeac8 -R 338666e76c5adcc84ca67c4306a8f3fb +P a8dfeec73b069f2dd7371c2792b36f152945d32120843db5361ff7e04de177bc b5d3dd8cb0b1e42ed0671a12d22af05194ea9522e4f41fd4bb0deff70b8b0757 +R 829b4bcd2ce5d178bebfa1657a2d3cee +T +closed b5d3dd8cb0b1e42ed0671a12d22af05194ea9522e4f41fd4bb0deff70b8b0757 U drh -Z 865cfc6e4e0c84a2365c19e9bd9ce995 +Z f7b572645a2b0a1f5c79c87cac8e08a9 diff --git a/manifest.uuid b/manifest.uuid index 2d0756f5dd..e31112db16 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8dfeec73b069f2dd7371c2792b36f152945d32120843db5361ff7e04de177bc \ No newline at end of file +dd568c27b1d7656388ea5b4132cc0265aedd7348d265d8e8c7412b00b28a31aa \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index d2fad0ffc3..0bb7a8d8f9 100644 --- a/src/expr.c +++ b/src/expr.c @@ -5001,6 +5001,58 @@ int sqlite3ExprImpliesExpr(Parse *pParse, Expr *pE1, Expr *pE2, int iTab){ return 0; } +/* +** This is the Expr node callback for sqlite3ExprImpliesNotNullRow(). +** If the expression node requires that the table at pWalker->iCur +** have a non-NULL column, then set pWalker->eCode to 1 and abort. +*/ +static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ + if( ExprHasProperty(pExpr, EP_FromJoin) ) return WRC_Prune; + switch( pExpr->op ){ + case TK_ISNULL: + case TK_IS: + case TK_OR: + case TK_FUNCTION: + case TK_AGG_FUNCTION: + return WRC_Prune; + case TK_COLUMN: + case TK_AGG_COLUMN: + if( pWalker->u.iCur==pExpr->iTable ){ + pWalker->eCode = 1; + return WRC_Abort; + } + return WRC_Prune; + default: + return WRC_Continue; + } +} + +/* +** Return true (non-zero) if expression p can only be true if at least +** one column of table iTab is non-null. In other words, return true +** if expression p will always be NULL or false if every column of iTab +** is NULL. +** +** Terms of p that are marked with EP_FromJoin (and hence that come from +** the ON or USING clauses of LEFT JOINS) are excluded from the analysis. +** +** This routine is used to check if a LEFT JOIN can be converted into +** an ordinary JOIN. The p argument is the WHERE clause. If the WHERE +** clause requires that some column of the right table of the LEFT JOIN +** be non-NULL, then the LEFT JOIN can be safely converted into an +** ordinary join. +*/ +int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab){ + Walker w; + w.xExprCallback = impliesNotNullRow; + w.xSelectCallback = 0; + w.xSelectCallback2 = 0; + w.eCode = 0; + w.u.iCur = iTab; + sqlite3WalkExpr(&w, p); + return w.eCode; +} + /* ** An instance of the following structure is used by the tree walker ** to determine if an expression can be evaluated by reference to the diff --git a/src/select.c b/src/select.c index ded01807ed..ca3e8b7f05 100644 --- a/src/select.c +++ b/src/select.c @@ -382,6 +382,29 @@ static void setJoinExpr(Expr *p, int iTable){ } } +/* Undo the work of setJoinExpr(). In the expression tree p, convert every +** term that is marked with EP_FromJoin and iRightJoinTable==iTable into +** an ordinary term that omits the EP_FromJoin mark. +** +** This happens when a LEFT JOIN is simplified into an ordinary JOIN. +*/ +static void unsetJoinExpr(Expr *p, int iTable){ + while( p ){ + if( ExprHasProperty(p, EP_FromJoin) + && (iTable<0 || p->iRightJoinTable==iTable) ){ + ExprClearProperty(p, EP_FromJoin); + } + if( p->op==TK_FUNCTION && p->x.pList ){ + int i; + for(i=0; ix.pList->nExpr; i++){ + unsetJoinExpr(p->x.pList->a[i].pExpr, iTable); + } + } + unsetJoinExpr(p->pLeft, iTable); + p = p->pRight; + } +} + /* ** This routine processes the join information for a SELECT statement. ** ON and USING clauses are converted into extra terms of the WHERE clause. @@ -3834,12 +3857,21 @@ static int flattenSubquery( ** (3) The inner query has a LIMIT clause (since the changes to the WHERE ** close would change the meaning of the LIMIT). ** -** (4) The inner query is the right operand of a LEFT JOIN. (The caller -** enforces this restriction since this routine does not have enough -** information to know.) +** (4) (** This restriction was removed on 2018-03-21. It used to read: +** The inner query is the right operand of a LEFT JOIN. **) ** ** (5) The WHERE clause expression originates in the ON or USING clause -** of a LEFT JOIN. +** of a LEFT JOIN where iCursor is not the right-hand table of that +** left join. An example: +** +** SELECT * +** FROM (SELECT 1 AS a1 UNION ALL SELECT 2) AS aa +** JOIN (SELECT 1 AS b2 UNION ALL SELECT 2) AS bb ON (a1=b2) +** LEFT JOIN (SELECT 8 AS c3 UNION ALL SELECT 9) AS cc ON (b2=2); +** +** The correct answer is three rows: (1,1,NULL),(2,2,8),(2,2,9). +** But if the (b2=2) term were to be pushed down into the bb subquery, +** then the (1,1,NULL) row would be suppressed. ** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. @@ -3875,12 +3907,15 @@ static int pushDownWhereTerms( nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, iCursor); pWhere = pWhere->pLeft; } - if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction (5) */ + if( ExprHasProperty(pWhere,EP_FromJoin) && pWhere->iRightJoinTable!=iCursor ){ + return 0; /* restriction (5) */ + } if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){ nChng++; while( pSubq ){ SubstContext x; pNew = sqlite3ExprDup(pParse->db, pWhere, 0); + unsetJoinExpr(pNew, -1); x.pParse = pParse; x.iTable = iCursor; x.iNewTable = iCursor; @@ -5175,13 +5210,29 @@ int sqlite3Select( generateColumnNames(pParse, p); } - /* Try to flatten subqueries in the FROM clause up into the main query + /* Try to various optimizations (flattening subqueries, and strength + ** reduction of join operators) in the FROM clause up into the main query */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) for(i=0; !p->pPrior && inSrc; i++){ struct SrcList_item *pItem = &pTabList->a[i]; Select *pSub = pItem->pSelect; Table *pTab = pItem->pTab; + + /* Convert LEFT JOIN into JOIN if there are terms of the right table + ** of the LEFT JOIN used in the WHERE clause. + */ + if( (pItem->fg.jointype & JT_LEFT)!=0 + && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor) + && OptimizationEnabled(db, SQLITE_SimplifyJoin) + ){ + SELECTTRACE(0x100,pParse,p, + ("LEFT-JOIN simplifies to JOIN on term %d\n",i)); + pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER); + unsetJoinExpr(p->pWhere, pItem->iCursor); + } + + /* No futher action if this term of the FROM clause is no a subquery */ if( pSub==0 ) continue; /* Catch mismatch in the declared columns of a view and the number of @@ -5322,8 +5373,7 @@ int sqlite3Select( /* Make copies of constant WHERE-clause terms in the outer query down ** inside the subquery. This can help the subquery to run more efficiently. */ - if( (pItem->fg.jointype & JT_OUTER)==0 - && OptimizationEnabled(db, SQLITE_PushDown) + if( OptimizationEnabled(db, SQLITE_PushDown) && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem->iCursor) ){ #if SELECTTRACE_ENABLED diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7c8cf121f4..216a500258 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1533,6 +1533,7 @@ struct sqlite3 { #define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ /* TH3 expects the Stat34 ^^^^^^ value to be 0x0800. Don't change it */ #define SQLITE_PushDown 0x1000 /* The push-down optimization */ +#define SQLITE_SimplifyJoin 0x2000 /* Convert LEFT JOIN to JOIN */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* @@ -3823,6 +3824,7 @@ int sqlite3ExprCompare(Parse*,Expr*, Expr*, int); int sqlite3ExprCompareSkip(Expr*, Expr*, int); int sqlite3ExprListCompare(ExprList*, ExprList*, int); int sqlite3ExprImpliesExpr(Parse*,Expr*, Expr*, int); +int sqlite3ExprImpliesNonNullRow(Expr*,int); void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*); void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*); int sqlite3ExprCoveredByIndex(Expr*, int iCur, Index *pIdx); diff --git a/test/cursorhint2.test b/test/cursorhint2.test index 616235b376..01755685e2 100644 --- a/test/cursorhint2.test +++ b/test/cursorhint2.test @@ -136,45 +136,50 @@ do_extract_hints_test 2.5 { x2 {EQ(c0,r[2])} } -do_extract_hints_test 2.6 { - SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 0 = (b IS NOT NULL) -} { - x2 {EQ(c0,r[2])} -} - -do_extract_hints_test 2.7 { - SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 0 = (b IS NOT +NULL) -} { - x2 {EQ(c0,r[2])} -} - -do_extract_hints_test 2.8 { - SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE b IS NOT +NULL -} { - x2 {EQ(c0,r[2])} -} - -do_extract_hints_test 2.9 { - SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE CASE b WHEN 0 THEN 0 ELSE 1 END; -} { - x2 {EQ(c0,r[2])} -} - -do_extract_hints_test 2.10 { - SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE x2.b = 32+32 -} { - x2 {AND(EQ(c1,ADD(32,32)),EQ(c0,r[2]))} -} - -ifcapable !icu { - # This test only works using the built-in LIKE, not the ICU LIKE extension. - do_extract_hints_test 2.11 { - SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE x2.b LIKE 'abc%' +if {0} { + # These tests no longer work due to the LEFT-JOIN strength reduction + # optimization + do_extract_hints_test 2.6 { + SELECT * FROM x1 CROSS JOIN x2 ON (a=x) WHERE 0 = (b IS NOT NULL) } { - x2 {AND(expr,EQ(c0,r[2]))} + x2 {EQ(c0,r[2])} + } + + do_extract_hints_test 2.7 { + SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 0 = (b IS NOT +NULL) + } { + x2 {EQ(c0,r[2])} + } + + do_extract_hints_test 2.8 { + SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE b IS NOT +NULL + } { + x2 {EQ(c0,r[2])} + } + + do_extract_hints_test 2.9 { + SELECT * FROM x1 LEFT JOIN x2 ON (a=x) + WHERE CASE b WHEN 0 THEN 0 ELSE 1 END; + } { + x2 {EQ(c0,r[2])} + } + + do_extract_hints_test 2.10 { + SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE x2.b = 32+32 + } { + x2 {AND(EQ(c1,ADD(32,32)),EQ(c0,r[2]))} + } + + ifcapable !icu { + # This test only works using the built-in LIKE, not the ICU LIKE extension. + do_extract_hints_test 2.11 { + SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE x2.b LIKE 'abc%' + } { + x2 {AND(expr,EQ(c0,r[2]))} + } } } - + do_extract_hints_test 2.12 { SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE coalesce(x2.b, 1) } { diff --git a/test/e_select.test b/test/e_select.test index 9aa2de6038..e88d63b54f 100644 --- a/test/e_select.test +++ b/test/e_select.test @@ -748,7 +748,7 @@ do_execsql_test e_select-3.2.1a { SELECT k FROM x1 LEFT JOIN x2 USING(k) } {1 2 3 4 5 6} do_execsql_test e_select-3.2.1b { - SELECT k FROM x1 LEFT JOIN x2 USING(k) WHERE x2.k + SELECT k FROM x1 LEFT JOIN x2 USING(k) WHERE x2.k ORDER BY +k } {1 3 5} do_execsql_test e_select-3.2.2 { SELECT k FROM x1 LEFT JOIN x2 USING(k) WHERE x2.k IS NULL diff --git a/test/join2.test b/test/join2.test index 256846ac91..7dfab4d693 100644 --- a/test/join2.test +++ b/test/join2.test @@ -86,7 +86,7 @@ do_execsql_test 2.0 { } do_catchsql_test 2.1 { - SELECT * FROM aa LEFT JOIN cc ON (a=b) JOIN bb ON (b=c); + SELECT * FROM aa LEFT JOIN cc ON (a=b) JOIN bb ON (b=coalesce(c,1)); } {1 {ON clause references tables to its right}} do_catchsql_test 2.2 { SELECT * FROM aa JOIN cc ON (a=b) JOIN bb ON (b=c);