From 743ae4c68b51fe56e6a269a79532224d2e45ce12 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 5 Apr 2024 20:01:28 +0000 Subject: [PATCH 01/19] Experimental enhancement in which expressions of the form "expr IN table" can be pushed down into subexpressions. FossilOrigin-Name: 2cbd7838fd6ffdf210f34671cd2e3e749a076a3a6f155bbe5f910a67db31c5b1 --- manifest | 22 ++++++------ manifest.uuid | 2 +- src/expr.c | 60 ++++++++++++++++++++++++++++---- src/parse.y | 4 +++ src/sqliteInt.h | 3 +- test/pushdown.test | 85 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 157 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 8b598b2633..8fd5496617 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Check-in\s[a9657c87c53c1922]\sis\swrong:\sthe\sIndexedExpr.bMaybeNullRow\sflag\sis\nrequired\sfor\svirtual\scolumns\sif\sthey\sare\spart\sof\san\souter\sjoin.\s\sAdd\sa\ntest\scase\s(derived\sfrom\sdbsqlfuzz\sb9e65e2f110df998f1306571fae7af6c01e4d92b)\nto\sprove\sit. -D 2024-04-05T13:56:05.021 +C Experimental\senhancement\sin\swhich\sexpressions\sof\sthe\sform\s"expr\sIN\stable"\ncan\sbe\spushed\sdown\sinto\ssubexpressions. +D 2024-04-05T20:01:28.787 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -700,7 +700,7 @@ F src/date.c 126ba2ab10aeb2e7ba6e089b5f07b747c0625b8287f78b60da346eda8d23c875 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 -F src/expr.c cd46ce2ebe46a30e63087dbe8aff7d6e34ba8acc8c658e8e18fcaa35f2711391 +F src/expr.c 48e6732170b5f3c07f9ebbdbe12c478e2867ea4be4361946ed26e2c8d607f7e8 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00 F src/func.c 283d4f3b2751a1d9339fd93a8a013d1948fd5f4474a3cab0955eb4fafd445d0f @@ -739,7 +739,7 @@ F src/os_win.c 6ff43bac175bd9ed79e7c0f96840b139f2f51d01689a638fd05128becf94908a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c ff60e98138d2499082ac6230f01ac508aba545315debccfca2fd6042f5f10fcd F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a -F src/parse.y 5bcef16094213efcc365a9d4dc4e3131f09251dc8838dce4a9e5f9764bff5b82 +F src/parse.y 67b9965a501599d5a1a4333125fab0dd230e03e28045700c9b331f7c3269bda5 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 @@ -755,7 +755,7 @@ F src/shell.c.in 0354ca51eee5fbf6af394a7ef9f5ef6823ef45b743db65431f6777e4d5be219 F src/sqlite.h.in 32389e0d584551b300d0157881336162c14315a424cbf385c0d65eb7c2e31f7b F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 170f85c5a9d6189cf9b5dd304f4d40b73a8c9762fb927f1327e8ed2cb99c06b6 +F src/sqliteInt.h a0d0f014baafcb61f5d0a6d206d4b190644d57bb762dff2073903595653d6319 F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -1510,7 +1510,7 @@ F test/printf.test 685fec5a0c5af2490ab0632775a301554361d674211d690f5bee0a97b0533 F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc -F test/pushdown.test 1495a09837a1cedfc0adf07ba42dc6b83be05a2c15de331b67c39a0e22078238 +F test/pushdown.test a8472a771ff80f9652477a688a5beed40b1f6de6546ed3a38150d3b272891173 F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quickcheck.test a4b7e878cd97e46108291c409b0bf8214f29e18fddd68a42bc5c1375ad1fb80a @@ -2184,9 +2184,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c7896e88850669e18e89d44c4169d4f4a5d4b904bea6ccb2ac64f93b6d348a42 -Q -a9657c87c53c19228a42559c82c54b504a5ad729e407e9e2c7dabcc0c949b261 -R 00fdd25fa9a3c1e58bfe6d44fd3eb812 +P 4484ec6d26b31305e31de89bdbae26344d8083a7e7de20861430d31737d9979c +R 445c665a7eec3209ded422866778f161 +T *branch * pushdown-IN-table +T *sym-pushdown-IN-table * +T -sym-trunk * U drh -Z d45c3b35e63ad3a95f4d8ac3c99323f3 +Z 5ecd3bbce21aa8f702309c384d09dab0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0dad7c2062..7d0dacd7c0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4484ec6d26b31305e31de89bdbae26344d8083a7e7de20861430d31737d9979c \ No newline at end of file +2cbd7838fd6ffdf210f34671cd2e3e749a076a3a6f155bbe5f910a67db31c5b1 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index edfea3fc92..f2b252792b 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2500,7 +2500,7 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ return WRC_Continue; } } -static int exprIsConst(Parse *pParse, Expr *p, int initFlag, int iCur){ +static int exprIsConst(Parse *pParse, Expr *p, int initFlag){ Walker w; w.eCode = initFlag; w.pParse = pParse; @@ -2509,7 +2509,6 @@ static int exprIsConst(Parse *pParse, Expr *p, int initFlag, int iCur){ #ifdef SQLITE_DEBUG w.xSelectCallback2 = sqlite3SelectWalkAssert2; #endif - w.u.iCur = iCur; sqlite3WalkExpr(&w, p); return w.eCode; } @@ -2529,7 +2528,7 @@ static int exprIsConst(Parse *pParse, Expr *p, int initFlag, int iCur){ ** function and on its parameters. */ int sqlite3ExprIsConstant(Parse *pParse, Expr *p){ - return exprIsConst(pParse, p, 1, 0); + return exprIsConst(pParse, p, 1); } /* @@ -2546,7 +2545,44 @@ int sqlite3ExprIsConstant(Parse *pParse, Expr *p){ ** the prepared statement starts up. See sqlite3ExprCodeRunJustOnce(). */ static int sqlite3ExprIsConstantNotJoin(Parse *pParse, Expr *p){ - return exprIsConst(pParse, p, 2, 0); + return exprIsConst(pParse, p, 2); +} + +/* +** This routine examines sub-SELECT statements as an expression is being +** walked as part of sqlite3ExprIsTableConstant() (hereafter IsTabConst()). +** Most SELECT statements will cause IsTabConst() to return false. However, +** if: +** +** (1) The SELECT is the right-hand side of an IN operator, and +** (2) Nothing in the SELECT refers to anything other than itself +** +** Then this routine causes the sub-SELECT to be bypassed, so that if +** nothing else is amiss the IsTabConst() routine can return true. +*/ +static int exprSelectWalkTableConstant(Walker *pWalker, Select *pSelect){ + int savedCursor; + assert( pSelect!=0 ); + assert( pWalker->eCode==3 || pWalker->eCode==0 ); + if( (pSelect->selFlags & SF_RhsOfIN)==0 ){ + pWalker->eCode = 0; + return WRC_Abort; + } + assert( pSelect->pSrc!=0 ); + assert( pSelect->pSrc->nSrc==1 ); + assert( pSelect->pWhere==0 ); + assert( pSelect->pGroupBy==0 ); + assert( pSelect->pHaving==0 ); + assert( pSelect->pOrderBy==0 ); + assert( pSelect->pPrior==0 ); + assert( pSelect->pNext==0 ); + assert( pSelect->pLimit==0 ); + assert( pSelect->pWith==0 ); + savedCursor = pWalker->u.iCur; + pWalker->u.iCur = pSelect->pSrc->a[0].iCursor; + sqlite3WalkExprList(pWalker, pSelect->pEList); + pWalker->u.iCur = savedCursor; + return WRC_Prune; } /* @@ -2554,9 +2590,19 @@ static int sqlite3ExprIsConstantNotJoin(Parse *pParse, Expr *p){ ** for any single row of the table with cursor iCur. In other words, the ** expression must not refer to any non-deterministic function nor any ** table other than iCur. +** +** 2024-04-05: Operators of the form "expr IN table" are now allowed, where +** "table" is the name of a table. */ -int sqlite3ExprIsTableConstant(Expr *p, int iCur){ - return exprIsConst(0, p, 3, iCur); +static int sqlite3ExprIsTableConstant(Expr *p, int iCur){ + Walker w; + w.eCode = 3; + w.pParse = 0; + w.xExprCallback = exprNodeIsConstant; + w.xSelectCallback = exprSelectWalkTableConstant; + w.u.iCur = iCur; + sqlite3WalkExpr(&w, p); + return w.eCode; } /* @@ -2713,7 +2759,7 @@ int sqlite3ExprIsConstantOrGroupBy(Parse *pParse, Expr *p, ExprList *pGroupBy){ */ int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ assert( isInit==0 || isInit==1 ); - return exprIsConst(0, p, 4+isInit, 0); + return exprIsConst(0, p, 4+isInit); } #ifdef SQLITE_ENABLE_CURSOR_HINTS diff --git a/src/parse.y b/src/parse.y index d22c8e6fc2..7a3d637663 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1369,6 +1369,10 @@ expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] { if( E ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, E); A = sqlite3PExpr(pParse, TK_IN, A, 0); sqlite3PExprAddSelect(pParse, A, pSelect); + if( pParse->nErr==0 ){ + assert( pSelect!=0 ); + pSelect->selFlags |= SF_RhsOfIN; + } if( N ) A = sqlite3PExpr(pParse, TK_NOT, A, 0); } expr(A) ::= EXISTS LP select(Y) RP. { diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 365ff2c7f5..29acb570de 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3585,6 +3585,7 @@ struct Select { #define SF_CopyCte 0x4000000 /* SELECT statement is a copy of a CTE */ #define SF_OrderByReqd 0x8000000 /* The ORDER BY clause may not be omitted */ #define SF_UpdateFrom 0x10000000 /* Query originates with UPDATE FROM */ +#define SF_RhsOfIN 0x20000000 /* Right-hand-side of an IN operator */ /* True if S exists and has SF_NestedFrom */ #define IsNestedFrom(S) ((S)!=0 && ((S)->selFlags&SF_NestedFrom)!=0) @@ -5081,7 +5082,7 @@ int sqlite3ExprTruthValue(const Expr*); int sqlite3ExprIsConstant(Parse*,Expr*); int sqlite3ExprIsConstantOrFunction(Expr*, u8); int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*); -int sqlite3ExprIsTableConstant(Expr*,int); +// int sqlite3ExprIsTableConstant(Expr*,int); int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int); #ifdef SQLITE_ENABLE_CURSOR_HINTS int sqlite3ExprContainsSubquery(Expr*); diff --git a/test/pushdown.test b/test/pushdown.test index 1fbe6f34cd..036fe2e2d8 100644 --- a/test/pushdown.test +++ b/test/pushdown.test @@ -227,4 +227,89 @@ do_execsql_test 5.0 { WHERE e>0; } {- - 3 4 5} + +# 2024-04-05 +# Allow push-down of operators of the form "expr IN table". +# +reset_db +do_execsql_test 6.0 { + CREATE TABLE t01(w,x,y,z); + CREATE TABLE t02(w,x,y,z); + CREATE VIEW t0(w,x,y,z) AS + SELECT w,x,y,z FROM t01 UNION ALL SELECT w,x,y,z FROM t02; + CREATE INDEX t01x ON t01(w,x,y); + CREATE INDEX t02x ON t02(w,x,y); + CREATE VIEW v1(k) AS VALUES(77),(88),(99); + CREATE TABLE k1(k); + INSERT INTO k1 SELECT * FROM v1; +} +do_eqp_test 6.1 { + WITH k(n) AS (VALUES(77),(88),(99)) + SELECT max(z) FROM t0 WHERE w=123 AND x IN k AND y BETWEEN 44 AND 55; +} { + QUERY PLAN + |--CO-ROUTINE t0 + | `--COMPOUND QUERY + | |--LEFT-MOST SUBQUERY + | | |--SEARCH t01 USING INDEX t01x (w=? AND x=? AND y>? AND y? AND y? AND y? AND y? AND y? AND y Date: Sat, 6 Apr 2024 12:19:50 +0000 Subject: [PATCH 02/19] Generalize pushdown to allow any uncorrelated subquery to be pushed down. FossilOrigin-Name: 87c45fb0d5f5ca5d6d1ad27bef83f294231d17d94299e1997364a7975b423e38 --- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- src/expr.c | 32 +++++--------------------------- src/parse.y | 4 ---- src/resolve.c | 2 ++ src/sqliteInt.h | 3 +-- 6 files changed, 21 insertions(+), 46 deletions(-) diff --git a/manifest b/manifest index 8fd5496617..581e5d22d1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\senhancement\sin\swhich\sexpressions\sof\sthe\sform\s"expr\sIN\stable"\ncan\sbe\spushed\sdown\sinto\ssubexpressions. -D 2024-04-05T20:01:28.787 +C Generalize\spushdown\sto\sallow\sany\suncorrelated\ssubquery\sto\sbe\spushed\sdown. +D 2024-04-06T12:19:50.927 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -700,7 +700,7 @@ F src/date.c 126ba2ab10aeb2e7ba6e089b5f07b747c0625b8287f78b60da346eda8d23c875 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 -F src/expr.c 48e6732170b5f3c07f9ebbdbe12c478e2867ea4be4361946ed26e2c8d607f7e8 +F src/expr.c f2f4768a25d62f645d11233930c0514f5f05fdadd8150e17d64187dd5aa41e4a F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00 F src/func.c 283d4f3b2751a1d9339fd93a8a013d1948fd5f4474a3cab0955eb4fafd445d0f @@ -739,7 +739,7 @@ F src/os_win.c 6ff43bac175bd9ed79e7c0f96840b139f2f51d01689a638fd05128becf94908a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c ff60e98138d2499082ac6230f01ac508aba545315debccfca2fd6042f5f10fcd F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a -F src/parse.y 67b9965a501599d5a1a4333125fab0dd230e03e28045700c9b331f7c3269bda5 +F src/parse.y 5bcef16094213efcc365a9d4dc4e3131f09251dc8838dce4a9e5f9764bff5b82 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 @@ -748,14 +748,14 @@ F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c 87b67bba3662a0523f39ae6b084a3907109702f717c654d6cecb838af5cd57f1 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c eb1860b134fb044fd819c4347105c148d5aac7c6498032be2829e5cc95619b28 +F src/resolve.c f242e509b2ff9e31799d26ddd87f20db1124426203aa16833a694d401b50d650 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 15a221347789e393b39e7d2d2bd102167979c95a1ce0675bb870b86a24ca6cf4 F src/shell.c.in 0354ca51eee5fbf6af394a7ef9f5ef6823ef45b743db65431f6777e4d5be2199 F src/sqlite.h.in 32389e0d584551b300d0157881336162c14315a424cbf385c0d65eb7c2e31f7b F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h a0d0f014baafcb61f5d0a6d206d4b190644d57bb762dff2073903595653d6319 +F src/sqliteInt.h e9ef8bb759f6ed7c41971bd8189dca3d38cc8bc69c463a6f49aaa9401857091a F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2184,11 +2184,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4484ec6d26b31305e31de89bdbae26344d8083a7e7de20861430d31737d9979c -R 445c665a7eec3209ded422866778f161 -T *branch * pushdown-IN-table -T *sym-pushdown-IN-table * -T -sym-trunk * +P 2cbd7838fd6ffdf210f34671cd2e3e749a076a3a6f155bbe5f910a67db31c5b1 +R 3f1ebbef3f30ee834eac030ff4cc2f9a +T *branch * pushdown-subquery +T *sym-pushdown-subquery * +T -sym-pushdown-IN-table * U drh -Z 5ecd3bbce21aa8f702309c384d09dab0 +Z 00cc18dd5b8247ec2ec6349e18d457dd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7d0dacd7c0..b3cce22c82 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2cbd7838fd6ffdf210f34671cd2e3e749a076a3a6f155bbe5f910a67db31c5b1 \ No newline at end of file +87c45fb0d5f5ca5d6d1ad27bef83f294231d17d94299e1997364a7975b423e38 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index f2b252792b..9513e5ffd5 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2550,38 +2550,17 @@ static int sqlite3ExprIsConstantNotJoin(Parse *pParse, Expr *p){ /* ** This routine examines sub-SELECT statements as an expression is being -** walked as part of sqlite3ExprIsTableConstant() (hereafter IsTabConst()). -** Most SELECT statements will cause IsTabConst() to return false. However, -** if: -** -** (1) The SELECT is the right-hand side of an IN operator, and -** (2) Nothing in the SELECT refers to anything other than itself -** -** Then this routine causes the sub-SELECT to be bypassed, so that if -** nothing else is amiss the IsTabConst() routine can return true. +** walked as part of sqlite3ExprIsTableConstant(). Sub-SELECTs are considered +** constant as long as they are uncorrelated - meaning that they do not +** contain any terms from outer contexts. */ static int exprSelectWalkTableConstant(Walker *pWalker, Select *pSelect){ - int savedCursor; assert( pSelect!=0 ); assert( pWalker->eCode==3 || pWalker->eCode==0 ); - if( (pSelect->selFlags & SF_RhsOfIN)==0 ){ + if( (pSelect->selFlags & SF_Correlated)!=0 ){ pWalker->eCode = 0; return WRC_Abort; } - assert( pSelect->pSrc!=0 ); - assert( pSelect->pSrc->nSrc==1 ); - assert( pSelect->pWhere==0 ); - assert( pSelect->pGroupBy==0 ); - assert( pSelect->pHaving==0 ); - assert( pSelect->pOrderBy==0 ); - assert( pSelect->pPrior==0 ); - assert( pSelect->pNext==0 ); - assert( pSelect->pLimit==0 ); - assert( pSelect->pWith==0 ); - savedCursor = pWalker->u.iCur; - pWalker->u.iCur = pSelect->pSrc->a[0].iCursor; - sqlite3WalkExprList(pWalker, pSelect->pEList); - pWalker->u.iCur = savedCursor; return WRC_Prune; } @@ -2591,8 +2570,7 @@ static int exprSelectWalkTableConstant(Walker *pWalker, Select *pSelect){ ** expression must not refer to any non-deterministic function nor any ** table other than iCur. ** -** 2024-04-05: Operators of the form "expr IN table" are now allowed, where -** "table" is the name of a table. +** Enhanced on 2024-04-06: Allow pExpr to contain uncorrelated subqueries. */ static int sqlite3ExprIsTableConstant(Expr *p, int iCur){ Walker w; diff --git a/src/parse.y b/src/parse.y index 7a3d637663..d22c8e6fc2 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1369,10 +1369,6 @@ expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] { if( E ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, E); A = sqlite3PExpr(pParse, TK_IN, A, 0); sqlite3PExprAddSelect(pParse, A, pSelect); - if( pParse->nErr==0 ){ - assert( pSelect!=0 ); - pSelect->selFlags |= SF_RhsOfIN; - } if( N ) A = sqlite3PExpr(pParse, TK_NOT, A, 0); } expr(A) ::= EXISTS LP select(Y) RP. { diff --git a/src/resolve.c b/src/resolve.c index 6e0c9906a6..7378addf5d 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -1349,6 +1349,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pNC->ncFlags & NC_PartIdx ); testcase( pNC->ncFlags & NC_IdxExpr ); testcase( pNC->ncFlags & NC_GenCol ); + assert( pExpr->x.pSelect ); if( pNC->ncFlags & NC_SelfRef ){ notValidImpl(pParse, pNC, "subqueries", pExpr, pExpr); }else{ @@ -1357,6 +1358,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ ExprSetProperty(pExpr, EP_VarSelect); + pExpr->x.pSelect->selFlags |= SF_Correlated; } pNC->ncFlags |= NC_Subquery; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 29acb570de..a735c445e7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3585,7 +3585,7 @@ struct Select { #define SF_CopyCte 0x4000000 /* SELECT statement is a copy of a CTE */ #define SF_OrderByReqd 0x8000000 /* The ORDER BY clause may not be omitted */ #define SF_UpdateFrom 0x10000000 /* Query originates with UPDATE FROM */ -#define SF_RhsOfIN 0x20000000 /* Right-hand-side of an IN operator */ +#define SF_Correlated 0x20000000 /* True if references the outer context */ /* True if S exists and has SF_NestedFrom */ #define IsNestedFrom(S) ((S)!=0 && ((S)->selFlags&SF_NestedFrom)!=0) @@ -5082,7 +5082,6 @@ int sqlite3ExprTruthValue(const Expr*); int sqlite3ExprIsConstant(Parse*,Expr*); int sqlite3ExprIsConstantOrFunction(Expr*, u8); int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*); -// int sqlite3ExprIsTableConstant(Expr*,int); int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int); #ifdef SQLITE_ENABLE_CURSOR_HINTS int sqlite3ExprContainsSubquery(Expr*); From 8181cc6c105f3a0253999563d73aa541a6f9916e Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 6 Apr 2024 17:37:30 +0000 Subject: [PATCH 03/19] Improvements to the accurancy of the fuzzinvariants.c testing module when SQLITE_ALLOW_ROWID_IN_VIEW is defined and the test query involves rowids. FossilOrigin-Name: c6e873d4db3ef36a0d561e64ead6feada5d1654c0757b4b6e55f671c9db66469 --- manifest | 13 ++++++------- manifest.uuid | 2 +- test/fuzzinvariants.c | 35 ++++++++++++++++++++++++++++++++--- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 8b598b2633..3fc2321c7a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Check-in\s[a9657c87c53c1922]\sis\swrong:\sthe\sIndexedExpr.bMaybeNullRow\sflag\sis\nrequired\sfor\svirtual\scolumns\sif\sthey\sare\spart\sof\san\souter\sjoin.\s\sAdd\sa\ntest\scase\s(derived\sfrom\sdbsqlfuzz\sb9e65e2f110df998f1306571fae7af6c01e4d92b)\nto\sprove\sit. -D 2024-04-05T13:56:05.021 +C Improvements\sto\sthe\saccurancy\sof\sthe\sfuzzinvariants.c\stesting\smodule\swhen\nSQLITE_ALLOW_ROWID_IN_VIEW\sis\sdefined\sand\sthe\stest\squery\sinvolves\srowids. +D 2024-04-06T17:37:30.138 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1258,7 +1258,7 @@ F test/fuzzdata8.db 4a53b6d077c6a5c23b609d8d3ac66996fa55ba3f8d02f9b6efdd0214a767 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc -F test/fuzzinvariants.c 4355043e98cd8555c62462fcbba91c17c6492b0b017bbbe68656d5f2208f6444 +F test/fuzzinvariants.c 9fc6810dde21e1aee20ed7a6c8ed944ffee127cf082fc48caba398fa31c762ca F test/gcfault.test 4ea410ac161e685f17b19e1f606f58514a2850e806c65b846d05f60d436c5b0d F test/gencol1.test e169bdfa11c7ed5e9f322a98a7db3afe9e66235750b68c923efee8e1876b46ec F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 @@ -2184,9 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c7896e88850669e18e89d44c4169d4f4a5d4b904bea6ccb2ac64f93b6d348a42 -Q -a9657c87c53c19228a42559c82c54b504a5ad729e407e9e2c7dabcc0c949b261 -R 00fdd25fa9a3c1e58bfe6d44fd3eb812 +P 4484ec6d26b31305e31de89bdbae26344d8083a7e7de20861430d31737d9979c +R e668dfe755aeff2fdd8b334389296750 U drh -Z d45c3b35e63ad3a95f4d8ac3c99323f3 +Z 4d6c822e968c71422e1200a444d830a0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0dad7c2062..57aabf40aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4484ec6d26b31305e31de89bdbae26344d8083a7e7de20861430d31737d9979c \ No newline at end of file +c6e873d4db3ef36a0d561e64ead6feada5d1654c0757b4b6e55f671c9db66469 \ No newline at end of file diff --git a/test/fuzzinvariants.c b/test/fuzzinvariants.c index 00b2c11742..25b35d4f6b 100644 --- a/test/fuzzinvariants.c +++ b/test/fuzzinvariants.c @@ -223,6 +223,18 @@ not_a_fault: return SQLITE_OK; } +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW +/* +** Return TRUE if the i-th column of pStmt might be a ROWID value. +*/ +static int column_might_be_rowid(sqlite3_stmt *pStmt, int i){ + const char *zColName = sqlite3_column_name(pStmt, i); + if( sqlite3_strlike("%rowid%",zColName,0)==0 ) return 1; + if( sqlite3_strlike("%oid%",zColName,0)==0 ) return 1; + return 0; +} +#endif /* SQLITE_ALLOW_ROWID_IN_VIEW */ + /* ** Generate SQL used to test a statement invariant. @@ -297,9 +309,7 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){ continue; } #ifdef SQLITE_ALLOW_ROWID_IN_VIEW - if( sqlite3_strlike("%rowid%",zColName,0)==0 - || sqlite3_strlike("%oid%",zColName,0)==0 - ){ + if( column_might_be_rowid(pBase,i) ){ /* ROWID values are unreliable if SQLITE_ALLOW_ROWID_IN_VIEW is used */ continue; } @@ -332,6 +342,11 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){ /* ** Return true if and only if v1 and is the same as v2. +** +** When compiled with SQLITE_ALLOW_ROWID_IN_VIEW, and if either +** v1 or v2 has a column name that indicates that it is a rowid +** then a NULL value in the rowid column will compare equal to +** an integer value in the other. */ static int sameValue( sqlite3_stmt *pS1, int i1, /* Value to text on the left */ @@ -346,6 +361,20 @@ static int sameValue( || (t1==SQLITE_FLOAT && t2==SQLITE_INTEGER) ){ /* Comparison of numerics is ok */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + }else + if( t1==SQLITE_INTEGER + && t2==SQLITE_NULL + && column_might_be_rowid(pS2,i2) + ){ + return 1; + }else + if( t2==SQLITE_INTEGER + && t1==SQLITE_NULL + && column_might_be_rowid(pS1,i1) + ){ + return 1; +#endif /* SQLITE_ALLOW_ROWID_IN_VIEW */ }else{ return 0; } From 6951c4977976ddca0b9e4ce75e79fc69c44dfd77 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 6 Apr 2024 18:30:09 +0000 Subject: [PATCH 04/19] Do not allow changes to sqlite3ExprIsTableConstant() that support pushdown of subqueries interfere with the hash-join logic. FossilOrigin-Name: 8682931f9c6b40e1b09139c10bbe932c38330b5eb0c5c84f2432ad19a6793e37 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 25 +++++++++++++++++++------ src/select.c | 2 +- src/sqliteInt.h | 2 +- src/where.c | 4 ++-- 6 files changed, 33 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index b13e8b0b82..781df2e045 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\stesting\slogic\sfixes\son\strunk\sinto\sthe\spushdown-subquery\sbranch. -D 2024-04-06T17:42:14.598 +C Do\snot\sallow\schanges\sto\ssqlite3ExprIsTableConstant()\sthat\ssupport\spushdown\sof\nsubqueries\sinterfere\swith\sthe\shash-join\slogic. +D 2024-04-06T18:30:09.352 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -700,7 +700,7 @@ F src/date.c 126ba2ab10aeb2e7ba6e089b5f07b747c0625b8287f78b60da346eda8d23c875 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 -F src/expr.c f2f4768a25d62f645d11233930c0514f5f05fdadd8150e17d64187dd5aa41e4a +F src/expr.c a475049e45329a4586ce8967963784a72023cef0f5821a2265ccacb7f68e0bc6 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00 F src/func.c 283d4f3b2751a1d9339fd93a8a013d1948fd5f4474a3cab0955eb4fafd445d0f @@ -750,12 +750,12 @@ F src/printf.c 87b67bba3662a0523f39ae6b084a3907109702f717c654d6cecb838af5cd57f1 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c f242e509b2ff9e31799d26ddd87f20db1124426203aa16833a694d401b50d650 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 15a221347789e393b39e7d2d2bd102167979c95a1ce0675bb870b86a24ca6cf4 +F src/select.c 6208d6d4acafc944b9916df28e63fc4e47b922b0335d0db057f68e334b87fc8e F src/shell.c.in 0354ca51eee5fbf6af394a7ef9f5ef6823ef45b743db65431f6777e4d5be2199 F src/sqlite.h.in 32389e0d584551b300d0157881336162c14315a424cbf385c0d65eb7c2e31f7b F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h e9ef8bb759f6ed7c41971bd8189dca3d38cc8bc69c463a6f49aaa9401857091a +F src/sqliteInt.h 257a7addbdb67b1f9f514a28904ed3f574de16bd87c21570faf30600c9c39f40 F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -835,7 +835,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c 9982128f010a699560fb5be37633801c7641ac3540f49db43cec878025929a0e +F src/where.c 09253e913dc1a93aa6e0e1a5d25a392b8fa39096cc30d230dd302264b3be0661 F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c 7b64295f1d82ad0928df435925dd7bbd5997b44a026153113eace0d9e71ff435 @@ -2184,8 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 87c45fb0d5f5ca5d6d1ad27bef83f294231d17d94299e1997364a7975b423e38 c6e873d4db3ef36a0d561e64ead6feada5d1654c0757b4b6e55f671c9db66469 -R 5c918916ac2b7aa6a88b463bfdfa1fb1 +P a4e1f03dcad70e828a4b56e7a332946daf84d6eae078e3ec9a248b8157a53963 +R d7462427e22b33861c511b137742aefb U drh -Z 8a9056dc314aaed8a12c375787e3cb55 +Z cfcd3c15df77757710cc4630347f7777 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6465b19d72..545d7417cd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4e1f03dcad70e828a4b56e7a332946daf84d6eae078e3ec9a248b8157a53963 \ No newline at end of file +8682931f9c6b40e1b09139c10bbe932c38330b5eb0c5c84f2432ad19a6793e37 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 9513e5ffd5..0ccb39cc6e 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2570,14 +2570,22 @@ static int exprSelectWalkTableConstant(Walker *pWalker, Select *pSelect){ ** expression must not refer to any non-deterministic function nor any ** table other than iCur. ** -** Enhanced on 2024-04-06: Allow pExpr to contain uncorrelated subqueries. +** Consider uncorrelated subqueries to be constants if the bAllowSubq +** parameter is true. */ -static int sqlite3ExprIsTableConstant(Expr *p, int iCur){ +static int sqlite3ExprIsTableConstant(Expr *p, int iCur, int bAllowSubq){ Walker w; w.eCode = 3; w.pParse = 0; w.xExprCallback = exprNodeIsConstant; - w.xSelectCallback = exprSelectWalkTableConstant; + if( bAllowSubq ){ + w.xSelectCallback = exprSelectWalkTableConstant; + }else{ + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif + } w.u.iCur = iCur; sqlite3WalkExpr(&w, p); return w.eCode; @@ -2598,7 +2606,10 @@ static int sqlite3ExprIsTableConstant(Expr *p, int iCur){ ** ** (1) pExpr cannot refer to any table other than pSrc->iCursor. ** -** (2) pExpr cannot use subqueries or non-deterministic functions. +** (2a) pExpr cannot use subqueries unless the bAllowSubq parameter is +** true and the subquery is non-correlated +** +** (2b) pExpr cannot use non-deterministic functions. ** ** (3) pSrc cannot be part of the left operand for a RIGHT JOIN. ** (Is there some way to relax this constraint?) @@ -2627,7 +2638,8 @@ static int sqlite3ExprIsTableConstant(Expr *p, int iCur){ int sqlite3ExprIsSingleTableConstraint( Expr *pExpr, /* The constraint */ const SrcList *pSrcList, /* Complete FROM clause */ - int iSrc /* Which element of pSrcList to use */ + int iSrc, /* Which element of pSrcList to use */ + int bAllowSubq /* Allow non-correlated subqueries */ ){ const SrcItem *pSrc = &pSrcList->a[iSrc]; if( pSrc->fg.jointype & JT_LTORJ ){ @@ -2652,7 +2664,8 @@ int sqlite3ExprIsSingleTableConstraint( } } } - return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor); /* rules (1), (2) */ + /* Rules (1), (2a), and (2b) handled by the following: */ + return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor, bAllowSubq); } diff --git a/src/select.c b/src/select.c index b200518518..b13aae1f9a 100644 --- a/src/select.c +++ b/src/select.c @@ -5260,7 +5260,7 @@ static int pushDownWhereTerms( } #endif - if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc) ){ + if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc, 1) ){ nChng++; pSubq->selFlags |= SF_PushDown; while( pSubq ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a735c445e7..d80a646093 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -5082,7 +5082,7 @@ int sqlite3ExprTruthValue(const Expr*); int sqlite3ExprIsConstant(Parse*,Expr*); int sqlite3ExprIsConstantOrFunction(Expr*, u8); int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*); -int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int); +int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int,int); #ifdef SQLITE_ENABLE_CURSOR_HINTS int sqlite3ExprContainsSubquery(Expr*); #endif diff --git a/src/where.c b/src/where.c index a1837b5879..65b6ae2c92 100644 --- a/src/where.c +++ b/src/where.c @@ -942,7 +942,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** WHERE clause (or the ON clause of a LEFT join) that constrain which ** rows of the target table (pSrc) that can be used. */ if( (pTerm->wtFlags & TERM_VIRTUAL)==0 - && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom) + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom, 0) ){ pPartial = sqlite3ExprAnd(pParse, pPartial, sqlite3ExprDup(pParse->db, pExpr, 0)); @@ -1211,7 +1211,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( for(pTerm=pWInfo->sWC.a; pTermpExpr; if( (pTerm->wtFlags & TERM_VIRTUAL)==0 - && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc) + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc, 0) ){ sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); } From 05c6d13027db5e2f8dd6311fd6520e1ca0e7bdb9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 7 Apr 2024 10:27:18 +0000 Subject: [PATCH 05/19] Add comments to note the name abiguity between the MySQL push-down optimization and the WHERE-clause push-down optimization. FossilOrigin-Name: 3d5fb1ec7a0440072d6e3b957903c85d0f32b8b07207a1ef22d1a69cf5e664d1 --- manifest | 21 +++++++++++---------- manifest.uuid | 2 +- src/expr.c | 5 +++-- src/select.c | 22 +++++++++++++++++----- src/sqliteInt.h | 4 ++-- src/wherecode.c | 6 ++++++ test/pushdown.test | 28 ++++++++++++++++++++++++---- 7 files changed, 64 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 3fc2321c7a..64b52c9b24 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\saccurancy\sof\sthe\sfuzzinvariants.c\stesting\smodule\swhen\nSQLITE_ALLOW_ROWID_IN_VIEW\sis\sdefined\sand\sthe\stest\squery\sinvolves\srowids. -D 2024-04-06T17:37:30.138 +C Add\scomments\sto\snote\sthe\sname\sabiguity\sbetween\sthe\sMySQL\spush-down\noptimization\sand\sthe\sWHERE-clause\spush-down\soptimization. +D 2024-04-07T10:27:18.165 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -700,7 +700,7 @@ F src/date.c 126ba2ab10aeb2e7ba6e089b5f07b747c0625b8287f78b60da346eda8d23c875 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 -F src/expr.c cd46ce2ebe46a30e63087dbe8aff7d6e34ba8acc8c658e8e18fcaa35f2711391 +F src/expr.c 98a1cabc44fbf64e4c87a639f4399d1afde640875c16dbc2e62a37c102cca9e8 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00 F src/func.c 283d4f3b2751a1d9339fd93a8a013d1948fd5f4474a3cab0955eb4fafd445d0f @@ -750,12 +750,12 @@ F src/printf.c 87b67bba3662a0523f39ae6b084a3907109702f717c654d6cecb838af5cd57f1 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c eb1860b134fb044fd819c4347105c148d5aac7c6498032be2829e5cc95619b28 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 15a221347789e393b39e7d2d2bd102167979c95a1ce0675bb870b86a24ca6cf4 +F src/select.c 3c8e1b921edfb71904b01c1ef9fb378224cdba71543342b164c7718cc8e82a44 F src/shell.c.in 0354ca51eee5fbf6af394a7ef9f5ef6823ef45b743db65431f6777e4d5be2199 F src/sqlite.h.in 32389e0d584551b300d0157881336162c14315a424cbf385c0d65eb7c2e31f7b F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 170f85c5a9d6189cf9b5dd304f4d40b73a8c9762fb927f1327e8ed2cb99c06b6 +F src/sqliteInt.h a275d95f5612f2640380532e484d1f6c46a48d84ae227e6c12e36712c22b7b8a F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -837,7 +837,7 @@ F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 F src/where.c 9982128f010a699560fb5be37633801c7641ac3540f49db43cec878025929a0e F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 -F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 +F src/wherecode.c e033875570f5d65b99fdb5189e597d91c8bc34f4196089fd50e6086d174035e7 F src/whereexpr.c 7b64295f1d82ad0928df435925dd7bbd5997b44a026153113eace0d9e71ff435 F src/window.c 5d95122dd330bfaebd732358c8ef067c5a9394a53ac249470d611d0ce2c52be2 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1510,7 +1510,7 @@ F test/printf.test 685fec5a0c5af2490ab0632775a301554361d674211d690f5bee0a97b0533 F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc -F test/pushdown.test 1495a09837a1cedfc0adf07ba42dc6b83be05a2c15de331b67c39a0e22078238 +F test/pushdown.test 45c6fa615f225041bbebac398d42aaaffdb6e0cdef8e542be9a8337a43d0ad95 F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quickcheck.test a4b7e878cd97e46108291c409b0bf8214f29e18fddd68a42bc5c1375ad1fb80a @@ -2184,8 +2184,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4484ec6d26b31305e31de89bdbae26344d8083a7e7de20861430d31737d9979c -R e668dfe755aeff2fdd8b334389296750 +P c6e873d4db3ef36a0d561e64ead6feada5d1654c0757b4b6e55f671c9db66469 +Q +681dfe2dd60714daf4987d45231d877f389825ab93e9bd0749391230186d6091 +R 023f47c11f43b462b0bef8a0994a0c0a U drh -Z 4d6c822e968c71422e1200a444d830a0 +Z 5c41a9ea676fc6eeb1fc7c2dc29fd70c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 57aabf40aa..1888e324a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c6e873d4db3ef36a0d561e64ead6feada5d1654c0757b4b6e55f671c9db66469 \ No newline at end of file +3d5fb1ec7a0440072d6e3b957903c85d0f32b8b07207a1ef22d1a69cf5e664d1 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index edfea3fc92..9ec77e823b 100644 --- a/src/expr.c +++ b/src/expr.c @@ -5031,8 +5031,9 @@ expr_code_doover: if( !ExprHasProperty(pExpr, EP_Collate) ){ /* A TK_COLLATE Expr node without the EP_Collate tag is a so-called ** "SOFT-COLLATE" that is added to constraints that are pushed down - ** from outer queries into sub-queries by the push-down optimization. - ** Clear subtypes as subtypes may not cross a subquery boundary. + ** from outer queries into sub-queries by the WHERE-clause push-down + ** optimization. Clear subtypes as subtypes may not cross a subquery + ** boundary. */ assert( pExpr->pLeft ); sqlite3ExprCode(pParse, pExpr->pLeft, target); diff --git a/src/select.c b/src/select.c index b200518518..821f9a0c4e 100644 --- a/src/select.c +++ b/src/select.c @@ -5060,6 +5060,18 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** The hope is that the terms added to the inner query will make it more ** efficient. ** +** NAME AMBIGUITY +** +** This optimization is called the "WHERE-clause push-down optimization". +** +** Do not confuse this optimization with another unrelated optimization +** with a similar name: The "MySQL push-down optimization" causes WHERE +** clause terms that can be evaluated using only the index and without +** reference to the table are run first, so that if they are false, +** unnecessary table seeks are avoided. +** +** RULES +** ** Do not attempt this optimization if: ** ** (1) (** This restriction was removed on 2017-09-29. We used to @@ -5125,10 +5137,10 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** (9c) There is a RIGHT JOIN (or FULL JOIN) in between the ON/USING ** clause and the subquery. ** -** Without this restriction, the push-down optimization might move -** the ON/USING filter expression from the left side of a RIGHT JOIN -** over to the right side, which leads to incorrect answers. See -** also restriction (6) in sqlite3ExprIsSingleTableConstraint(). +** Without this restriction, the WHERE-clause push-down optimization +** might move the ON/USING filter expression from the left side of a +** RIGHT JOIN over to the right side, which leads to incorrect answers. +** See also restriction (6) in sqlite3ExprIsSingleTableConstraint(). ** ** (10) The inner query is not the right-hand table of a RIGHT JOIN. ** @@ -7689,7 +7701,7 @@ int sqlite3Select( #endif assert( pItem->pSelect && (pItem->pSelect->selFlags & SF_PushDown)!=0 ); }else{ - TREETRACE(0x4000,pParse,p,("Push-down not possible\n")); + TREETRACE(0x4000,pParse,p,("WHERE-lcause push-down not possible\n")); } /* Convert unused result columns of the subquery into simple NULL diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 365ff2c7f5..d7e339d181 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1906,7 +1906,7 @@ struct sqlite3 { #define SQLITE_CursorHints 0x00000400 /* Add OP_CursorHint opcodes */ #define SQLITE_Stat4 0x00000800 /* Use STAT4 data */ /* TH3 expects this value ^^^^^^^^^^ to be 0x0000800. Don't change it */ -#define SQLITE_PushDown 0x00001000 /* The push-down optimization */ +#define SQLITE_PushDown 0x00001000 /* WHERE-clause push-down opt */ #define SQLITE_SimplifyJoin 0x00002000 /* Convert LEFT JOIN to JOIN */ #define SQLITE_SkipScan 0x00004000 /* Skip-scans */ #define SQLITE_PropagateConst 0x00008000 /* The constant propagation opt */ @@ -3580,7 +3580,7 @@ struct Select { #define SF_View 0x0200000 /* SELECT statement is a view */ #define SF_NoopOrderBy 0x0400000 /* ORDER BY is ignored for this query */ #define SF_UFSrcCheck 0x0800000 /* Check pSrc as required by UPDATE...FROM */ -#define SF_PushDown 0x1000000 /* SELECT has be modified by push-down opt */ +#define SF_PushDown 0x1000000 /* Modified by WHERE-clause push-down opt */ #define SF_MultiPart 0x2000000 /* Has multiple incompatible PARTITIONs */ #define SF_CopyCte 0x4000000 /* SELECT statement is a copy of a CTE */ #define SF_OrderByReqd 0x8000000 /* The ORDER BY clause may not be omitted */ diff --git a/src/wherecode.c b/src/wherecode.c index 47ce36ce38..1f9c84f008 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -2474,6 +2474,12 @@ Bitmask sqlite3WhereCodeOneLoopStart( ** iLoop==3: Code all remaining expressions. ** ** An effort is made to skip unnecessary iterations of the loop. + ** + ** This optimization of causing simple query restrictions to occur before + ** more complex one is call the "push-down" optimization in MySQL. Here + ** in SQLite, the name is "MySQL push-down", since there is also another + ** totally unrelated optimization called "WHERE-clause push-down". + ** Sometimes the qualifier is omitted, resulting in an ambiguity, so beware. */ iLoop = (pIdx ? 1 : 2); do{ diff --git a/test/pushdown.test b/test/pushdown.test index 1fbe6f34cd..ac46eee8a8 100644 --- a/test/pushdown.test +++ b/test/pushdown.test @@ -1,4 +1,4 @@ -# 2017 April 29 +# 2017-04-29 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: @@ -8,6 +8,26 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# +# Test cases for the push-down optimizations. +# +# +# There are two different meanings for "push-down optimization". +# +# (1) "MySQL push-down" means that WHERE clause terms that can be +# evaluated using only the index and without reference to the +# table are run first, so that if they are false, unnecessary table +# seeks are avoided. See https://sqlite.org/src/info/d7bb79ed3a40419d +# from 2017-04-29. +# +# (2) "WHERE-clause pushdown" means to push WHERE clause terms in +# outer queries down into subqueries. See +# https://sqlite.org/src/info/6df18e949d367629 from 2015-06-02. +# +# This module started out as tests for MySQL push-down only. But because +# of naming ambiguity, it has picked up test cases for WHERE-clause push-down +# over the years. +# set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -87,8 +107,8 @@ do_test 2.2 { } {three} # 2022-11-25 dbsqlfuzz crash-3a548de406a50e896c1bf7142692d35d339d697f -# Disable the push-down optimization for compound subqueries if any -# arm of the compound has an incompatible affinity. +# Disable the WHERE-clause push-down optimization for compound subqueries +# if any arm of the compound has an incompatible affinity. # reset_db do_execsql_test 3.1 { @@ -185,7 +205,7 @@ do_eqp_test 3.8 { # SELECT (SELECT count(*) FROM t1)+(SELECT count(*) FROM t2) # 2023-05-09 https://sqlite.org/forum/forumpost/a7d4be7fb6 -# Restriction (9) on the push-down optimization. +# Restriction (9) on the WHERE-clause push-down optimization. # reset_db db null - From 1152def76ecc8bfb95a51562b6bad949471f6863 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 7 Apr 2024 18:23:30 +0000 Subject: [PATCH 06/19] If SQLITE_ALLOW_ROWID_IN_VIEW is set to 2, then all rowids for views return a value of NULL. FossilOrigin-Name: 0a53dde21403aa6de11c5085c16def3f95046c5629daf2675b075e4d6683ef94 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/resolve.c | 5 +++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 64b52c9b24..b806287450 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\scomments\sto\snote\sthe\sname\sabiguity\sbetween\sthe\sMySQL\spush-down\noptimization\sand\sthe\sWHERE-clause\spush-down\soptimization. -D 2024-04-07T10:27:18.165 +C If\sSQLITE_ALLOW_ROWID_IN_VIEW\sis\sset\sto\s2,\sthen\sall\srowids\sfor\sviews\sreturn\na\svalue\sof\sNULL. +D 2024-04-07T18:23:30.116 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -748,7 +748,7 @@ F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c 87b67bba3662a0523f39ae6b084a3907109702f717c654d6cecb838af5cd57f1 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c eb1860b134fb044fd819c4347105c148d5aac7c6498032be2829e5cc95619b28 +F src/resolve.c 42d5d75a6ccfaf5b17d81fca64203ebce87f59702089142feed90df28dec1203 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 3c8e1b921edfb71904b01c1ef9fb378224cdba71543342b164c7718cc8e82a44 F src/shell.c.in 0354ca51eee5fbf6af394a7ef9f5ef6823ef45b743db65431f6777e4d5be2199 @@ -2184,9 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c6e873d4db3ef36a0d561e64ead6feada5d1654c0757b4b6e55f671c9db66469 -Q +681dfe2dd60714daf4987d45231d877f389825ab93e9bd0749391230186d6091 -R 023f47c11f43b462b0bef8a0994a0c0a +P 3d5fb1ec7a0440072d6e3b957903c85d0f32b8b07207a1ef22d1a69cf5e664d1 +R 0db0e0612520e2d37c806507a6db9dae U drh -Z 5c41a9ea676fc6eeb1fc7c2dc29fd70c +Z 2bafcb904067739b9429bfd86a29a830 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1888e324a6..2f2bda0775 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3d5fb1ec7a0440072d6e3b957903c85d0f32b8b07207a1ef22d1a69cf5e664d1 \ No newline at end of file +0a53dde21403aa6de11c5085c16def3f95046c5629daf2675b075e4d6683ef94 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 6e0c9906a6..86532db154 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -632,6 +632,11 @@ static int lookupName( && ALWAYS(VisibleRowid(pMatch->pTab) || pMatch->fg.isNestedFrom) ){ cnt = cntTab; +#if SQLITE_ALLOW_ROWID_IN_VIEW+0==2 + if( pMatch->pTab!=0 && IsView(pMatch->pTab) ){ + eNewExprOp = TK_NULL; + } +#endif if( pMatch->fg.isNestedFrom==0 ) pExpr->iColumn = -1; pExpr->affExpr = SQLITE_AFF_INTEGER; } From fa4c4247a573b0f8f5875fba23b1c5a19562c44e Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 7 Apr 2024 18:36:32 +0000 Subject: [PATCH 07/19] Omit all rowid-in-view restrictions from the fuzzinvariant.c test module as they are no longer necessary, as of the previous check-in. FossilOrigin-Name: 6431538f0bb3bb8606786f3c3e5c055c4bc387098dd3bdc8a94f6fda61c47f52 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/fuzzinvariants.c | 38 -------------------------------------- 3 files changed, 7 insertions(+), 45 deletions(-) diff --git a/manifest b/manifest index b806287450..c4e5160b20 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sSQLITE_ALLOW_ROWID_IN_VIEW\sis\sset\sto\s2,\sthen\sall\srowids\sfor\sviews\sreturn\na\svalue\sof\sNULL. -D 2024-04-07T18:23:30.116 +C Omit\sall\srowid-in-view\srestrictions\sfrom\sthe\sfuzzinvariant.c\stest\smodule\nas\sthey\sare\sno\slonger\snecessary,\sas\sof\sthe\sprevious\scheck-in. +D 2024-04-07T18:36:32.081 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1258,7 +1258,7 @@ F test/fuzzdata8.db 4a53b6d077c6a5c23b609d8d3ac66996fa55ba3f8d02f9b6efdd0214a767 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc -F test/fuzzinvariants.c 9fc6810dde21e1aee20ed7a6c8ed944ffee127cf082fc48caba398fa31c762ca +F test/fuzzinvariants.c d89b81c3bdad7b2837f8cce645d5b563ffa965f6d819ce33f0f6d82a71c6ca9a F test/gcfault.test 4ea410ac161e685f17b19e1f606f58514a2850e806c65b846d05f60d436c5b0d F test/gencol1.test e169bdfa11c7ed5e9f322a98a7db3afe9e66235750b68c923efee8e1876b46ec F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 @@ -2184,8 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3d5fb1ec7a0440072d6e3b957903c85d0f32b8b07207a1ef22d1a69cf5e664d1 -R 0db0e0612520e2d37c806507a6db9dae +P 0a53dde21403aa6de11c5085c16def3f95046c5629daf2675b075e4d6683ef94 +R 950039d72c57e0f5f8da3387d03515aa U drh -Z 2bafcb904067739b9429bfd86a29a830 +Z decff099e336dbab63d6324042d80ad4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2f2bda0775..e1d7770bd6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a53dde21403aa6de11c5085c16def3f95046c5629daf2675b075e4d6683ef94 \ No newline at end of file +6431538f0bb3bb8606786f3c3e5c055c4bc387098dd3bdc8a94f6fda61c47f52 \ No newline at end of file diff --git a/test/fuzzinvariants.c b/test/fuzzinvariants.c index 25b35d4f6b..66475270b0 100644 --- a/test/fuzzinvariants.c +++ b/test/fuzzinvariants.c @@ -223,19 +223,6 @@ not_a_fault: return SQLITE_OK; } -#ifdef SQLITE_ALLOW_ROWID_IN_VIEW -/* -** Return TRUE if the i-th column of pStmt might be a ROWID value. -*/ -static int column_might_be_rowid(sqlite3_stmt *pStmt, int i){ - const char *zColName = sqlite3_column_name(pStmt, i); - if( sqlite3_strlike("%rowid%",zColName,0)==0 ) return 1; - if( sqlite3_strlike("%oid%",zColName,0)==0 ) return 1; - return 0; -} -#endif /* SQLITE_ALLOW_ROWID_IN_VIEW */ - - /* ** Generate SQL used to test a statement invariant. ** @@ -308,12 +295,6 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){ ** WHERE clause. */ continue; } -#ifdef SQLITE_ALLOW_ROWID_IN_VIEW - if( column_might_be_rowid(pBase,i) ){ - /* ROWID values are unreliable if SQLITE_ALLOW_ROWID_IN_VIEW is used */ - continue; - } -#endif for(j=0; j Date: Mon, 8 Apr 2024 06:37:19 +0000 Subject: [PATCH 08/19] Enhancements to the fuzzer invariant checker to turn optimizations on and off. FossilOrigin-Name: 67594481379824823105939fea2ce1fe280667f6db91735ac78b4b6164a78dab --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fuzzcheck.c | 16 +++++++++++----- test/fuzzinvariants.c | 30 ++++++++++++++++++++++++------ 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 9489e7d05e..f59889aa63 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\stesting\senhancements\sinto\sthe\spushdown-subquery\sbranch. -D 2024-04-07T18:55:57.788 +C Enhancements\sto\sthe\sfuzzer\sinvariant\schecker\sto\sturn\soptimizations\son\sand\noff. +D 2024-04-08T06:37:19.669 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1246,7 +1246,7 @@ F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c e6a40f53ac5624aa5b7c4f31c385f09ba088d524cecc4512fd3057caeed8f530 +F test/fuzzcheck.c dc159967609d00b0cfe619e735cbbf8482570aca85711397034b0662b6c18fc7 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1258,7 +1258,7 @@ F test/fuzzdata8.db 4a53b6d077c6a5c23b609d8d3ac66996fa55ba3f8d02f9b6efdd0214a767 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc -F test/fuzzinvariants.c d89b81c3bdad7b2837f8cce645d5b563ffa965f6d819ce33f0f6d82a71c6ca9a +F test/fuzzinvariants.c 0729b9d8ed77ad0f8c5c7601168a707d5803087d2da030ede9057c51c809cc6c F test/gcfault.test 4ea410ac161e685f17b19e1f606f58514a2850e806c65b846d05f60d436c5b0d F test/gencol1.test e169bdfa11c7ed5e9f322a98a7db3afe9e66235750b68c923efee8e1876b46ec F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 @@ -2184,8 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 27865e316f8dfbf4c20290cf1be3024d7518fec46655e34f3fc435e15346c63e 6431538f0bb3bb8606786f3c3e5c055c4bc387098dd3bdc8a94f6fda61c47f52 -R a6f74894c0cda8635456b45fc1556948 +P 287ff24b26a512ff7648679767e68244f6eef95df6a49c46ed1f2594030ed523 +R f668a18b739f9caf1d0ed3baa3bf6c8f U drh -Z d0677700c03f1b93b120f5cc4adf200f +Z 70cb6868aa0f914c12631462fc26d140 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c911a505d2..30cfd87b8f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -287ff24b26a512ff7648679767e68244f6eef95df6a49c46ed1f2594030ed523 \ No newline at end of file +67594481379824823105939fea2ce1fe280667f6db91735ac78b4b6164a78dab \ No newline at end of file diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index e4ad1c1137..6cae348bd6 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -979,7 +979,8 @@ extern int fuzz_invariant( int iRow, /* The row number for pStmt */ int nRow, /* Total number of output rows */ int *pbCorrupt, /* IN/OUT: Flag indicating a corrupt database file */ - int eVerbosity /* How much debugging output */ + int eVerbosity, /* How much debugging output */ + unsigned int dbOpt /* Default optimization flags */ ); /* Implementation of sqlite_dbdata and sqlite_dbptr */ @@ -1031,7 +1032,12 @@ static int recoverDatabase(sqlite3 *db){ /* ** Run the SQL text */ -static int runDbSql(sqlite3 *db, const char *zSql, unsigned int *pBtsFlags){ +static int runDbSql( + sqlite3 *db, /* Run SQL on this database connection */ + const char *zSql, /* The SQL to be run */ + unsigned int *pBtsFlags, + unsigned int dbOpt /* Default optimization flags */ +){ int rc; sqlite3_stmt *pStmt; int bCorrupt = 0; @@ -1107,7 +1113,7 @@ static int runDbSql(sqlite3 *db, const char *zSql, unsigned int *pBtsFlags){ iRow++; for(iCnt=0; iCnt<99999; iCnt++){ rc = fuzz_invariant(db, pStmt, iCnt, iRow, nRow, - &bCorrupt, eVerbosity); + &bCorrupt, eVerbosity, dbOpt); if( rc==SQLITE_DONE ) break; if( rc!=SQLITE_ERROR ) g.nInvariant++; if( eVerbosity>0 ){ @@ -1330,7 +1336,7 @@ int runCombinedDbSqlInput( char cSaved = zSql[i+1]; zSql[i+1] = 0; if( sqlite3_complete(zSql+j) ){ - rc = runDbSql(cx.db, zSql+j, &btsFlags); + rc = runDbSql(cx.db, zSql+j, &btsFlags, dbOpt); j = i+1; } zSql[i+1] = cSaved; @@ -1340,7 +1346,7 @@ int runCombinedDbSqlInput( } } if( j100 ) return SQLITE_DONE; zTest = fuzz_invariant_sql(pStmt, iCnt); if( zTest==0 ) return SQLITE_DONE; + if( noOpt ){ + sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, db, ~dbOpt); + } rc = sqlite3_prepare_v2(db, zTest, -1, &pTestStmt, 0); + if( noOpt ){ + sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, db, dbOpt); + } if( rc ){ if( eVerbosity ){ printf("invariant compile failed: %s\n%s\n", @@ -212,7 +226,7 @@ int fuzz_invariant( } sqlite3_finalize(pCk); if( rc==SQLITE_DONE ){ - reportInvariantFailed(pStmt, pTestStmt, iRow); + reportInvariantFailed(pStmt, pTestStmt, iRow, dbOpt, noOpt); return SQLITE_INTERNAL; }else if( eVerbosity>0 ){ printf("invariant-error ignored due to the use of virtual tables\n"); @@ -488,13 +502,17 @@ static void printRow(sqlite3_stmt *pStmt, int iRow){ static void reportInvariantFailed( sqlite3_stmt *pOrig, /* The original query */ sqlite3_stmt *pTest, /* The alternative test query with a missing row */ - int iRow /* Row number in pOrig */ + int iRow, /* Row number in pOrig */ + unsigned int dbOpt, /* Optimization flags on pOrig */ + int noOpt /* True if opt flags inverted for pTest */ ){ int iTestRow = 0; printf("Invariant check failed on row %d.\n", iRow); - printf("Original query --------------------------------------------------\n"); + printf("Original query (opt-flags: 0x%08x) --------------------------\n", + dbOpt); printf("%s\n", sqlite3_expanded_sql(pOrig)); - printf("Alternative query -----------------------------------------------\n"); + printf("Alternative query (opt-flags: 0x%08x) -----------------------\n", + noOpt ? ~dbOpt : dbOpt); printf("%s\n", sqlite3_expanded_sql(pTest)); printf("Result row that is missing from the alternative -----------------\n"); printRow(pOrig, iRow); From 0389d5f8d0838c4b7eb415d6ce515a07aeb958d1 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 8 Apr 2024 11:45:15 +0000 Subject: [PATCH 09/19] Fix a harmless compiler warning in the ICU extension due to [acddbc489d5231da]. FossilOrigin-Name: 3d8750e88558f8ea6a1523a5add4ccca3262e31f348b603e56ab193d81908995 --- ext/icu/icu.c | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/icu/icu.c b/ext/icu/icu.c index f9d9eef905..69867bfa83 100644 --- a/ext/icu/icu.c +++ b/ext/icu/icu.c @@ -499,7 +499,7 @@ static void icuLoadCollation( { "QUARTERNARY", UCOL_QUATERNARY }, { "IDENTICAL", UCOL_IDENTICAL }, }; - int i; + unsigned int i; for(i=0; i Date: Mon, 8 Apr 2024 17:55:15 +0000 Subject: [PATCH 10/19] Update tests in returning1.test to account for [c7896e88]. FossilOrigin-Name: c02956cd5cbf87027ba201b5d273e0efd429836ee6cabdc2f6d5b8096c876b7b --- manifest | 15 +++++++-------- manifest.uuid | 2 +- test/returning1.test | 10 +++++----- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 739651c910..9dfb5b4e7d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sWHERE-clause\spush-down\soptimization\sso\sthat\sit\sis\sable\sto\npush\sdown\sWHERE\sclause\sterms\sthat\scontain\suncorrelated\ssubqueries. -D 2024-04-08T11:50:07.908 +C Update\stests\sin\sreturning1.test\sto\saccount\sfor\s[c7896e88]. +D 2024-04-08T17:55:15.936 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1529,7 +1529,7 @@ F test/regexp2.test 55ed41da802b0e284ac7e2fe944be3948f93ff25abbca0361a609acfed13 F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2118d F test/resetdb.test 54c06f18bc832ac6d6319e5ab23d5c8dd49fdbeec7c696d791682a8006bd5fc3 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb -F test/returning1.test 3ead782eddf51f573cdd43bcbb10d1b485ac095a19a76d16c43fd159ea9b7466 +F test/returning1.test b27646c5e25431dd47cf72f525d1b2d5ad4ae14e21d9cf3d65769f05e48f64f3 F test/returningfault.test ae4c4b5e8745813287a359d9ccdb9d5c883c2e68afb18fb0767937d5de5692a4 F test/rollback.test 06680159bc6746d0f26276e339e3ae2f951c64812468308838e0a3362d911eaa F test/rollback2.test 3f3a4e20401825017df7e7671e9f31b6de5fae5620c2b9b49917f52f8c160a8f @@ -2184,9 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3d8750e88558f8ea6a1523a5add4ccca3262e31f348b603e56ab193d81908995 67594481379824823105939fea2ce1fe280667f6db91735ac78b4b6164a78dab -R 1ffe1af0415e1a4fed09276e65c27d2f -T +closed 67594481379824823105939fea2ce1fe280667f6db91735ac78b4b6164a78dab -U drh -Z a907cc2f6ecf59760a4baca00ef3db65 +P 69ec714b2d698acf9e37635256c01b233ce32f22e8323e226441d5ddd948a940 +R 02f4dab5c53b9393812ce64cd20da4c4 +U dan +Z d5b97b3de9d6c5aa7782c9d7a19a211c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 35e356a937..df4e321c0c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -69ec714b2d698acf9e37635256c01b233ce32f22e8323e226441d5ddd948a940 \ No newline at end of file +c02956cd5cbf87027ba201b5d273e0efd429836ee6cabdc2f6d5b8096c876b7b \ No newline at end of file diff --git a/test/returning1.test b/test/returning1.test index 24032496ce..02743b2bdc 100644 --- a/test/returning1.test +++ b/test/returning1.test @@ -227,21 +227,21 @@ ifcapable !allow_rowid_in_view { } else { # Note: The values returned by the RETURNING clauses of the following # two statements are the rowid columns of views. These values are not - # well defined, so the INSERT returns -1, and the UPDATE returns 1, 2 - # and 3. These match the values used for new.rowid expressions, but - # not much else. + # well defined, so the INSERT returns -1, and the UPDATE returns NULL. + # These match the values used for new.rowid expressions, but not much + # else. do_catchsql_test 10.3a { INSERT INTO t1(a, b) VALUES(1234, 5678) RETURNING rowid; } {0 -1} do_catchsql_test 10.3b { UPDATE t1 SET a='z' WHERE b='y' RETURNING rowid; - } {0 {1 2 3}} + } {0 {{} {} {}}} do_execsql_test 10.4 { SELECT * FROM log; } { - insert -1 1234 5678 update 1 z y update 2 z y update 3 z y + insert -1 1234 5678 update {} z y update {} z y update {} z y } } From d21d5b21db209597aeaf41d0394cf70ddd62235d Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 9 Apr 2024 13:57:27 +0000 Subject: [PATCH 11/19] The read-only CHECK-constraint optimization of [34ddf02d3d21151b] inhibits the xfer optimization for tables with CHECK constraints. However, the xfer optimization is required for correct operation of VACUUM INTO on tables that contain generated columns. Fix this by ignoring CHECK constraints when qualifying the xfer optimization while running VACUUM. Problem reported by [forum:/forumpost/3ec177d68fe7fa2c|forum post 3ec177d68fe7fa2c]. FossilOrigin-Name: a6e26e778812c8409fca77183e24d3b70189c4d02fce10c7e74cd4ccc8c8ea97 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/insert.c | 5 ++++- test/vacuum-into.test | 35 ++++++++++++++++++++++++++++++++++- 4 files changed, 47 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 9dfb5b4e7d..8197099293 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\stests\sin\sreturning1.test\sto\saccount\sfor\s[c7896e88]. -D 2024-04-08T17:55:15.936 +C The\sread-only\sCHECK-constraint\soptimization\sof\s[34ddf02d3d21151b]\sinhibits\sthe\nxfer\soptimization\sfor\stables\swith\sCHECK\sconstraints.\s\sHowever,\sthe\sxfer\noptimization\sis\srequired\sfor\scorrect\soperation\sof\sVACUUM\sINTO\son\stables\sthat\ncontain\sgenerated\scolumns.\sFix\sthis\sby\signoring\sCHECK\sconstraints\swhen\nqualifying\sthe\sxfer\soptimization\swhile\srunning\sVACUUM.\s\sProblem\sreported\sby\n[forum:/forumpost/3ec177d68fe7fa2c|forum\spost\s3ec177d68fe7fa2c]. +D 2024-04-09T13:57:27.143 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -709,7 +709,7 @@ F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c bc4d172b52a6167cd28797397a464c2cca9607dc047dfa0b5cf61687962e6964 +F src/insert.c 4bd7c7e54a1062dcd0214b7a6296f7194eb10fb14d3ddca1ed20b01c2a86a18c F src/json.c e2e40760d6689134c3e2ece38c6a496b34ff5e2661a8f238444a119af666fdce F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 @@ -1916,7 +1916,7 @@ F test/uri.test c1abaaaa28e9422d61e5f3f9cbc8ef993ec49fe802f581520731708561d49384 F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 F test/utf16align.test 9fde0bb5d3a821594aa68c6829ab9c5453a084384137ebb9f6153e2d678039da -F test/vacuum-into.test 35dc6f79b563f91c61822f61797363e97fed1bf28f1f722688b98d43f1980d76 +F test/vacuum-into.test 77845cee98770c416dae9b0da6bb3229753861f2da65c11b4f9715d081712d8a F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d F test/vacuum2.test 9fd45ce6ce29f5614c249e03938d3567c06a9e772d4f155949f8eafe2d8af520 F test/vacuum3.test d9d9a04ee58c485b94694fd4f68cffaba49c32234fdefe1ac1a622c5e17d4ce3 @@ -2184,8 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 69ec714b2d698acf9e37635256c01b233ce32f22e8323e226441d5ddd948a940 -R 02f4dab5c53b9393812ce64cd20da4c4 -U dan -Z d5b97b3de9d6c5aa7782c9d7a19a211c +P c02956cd5cbf87027ba201b5d273e0efd429836ee6cabdc2f6d5b8096c876b7b +R 14ee2c4095f13f5f59e64d6dfae705ab +U drh +Z 40f988c61dc13a84e5a7ab5ecd628e6c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index df4e321c0c..3f94593291 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c02956cd5cbf87027ba201b5d273e0efd429836ee6cabdc2f6d5b8096c876b7b \ No newline at end of file +a6e26e778812c8409fca77183e24d3b70189c4d02fce10c7e74cd4ccc8c8ea97 \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index c126a83952..072386e656 100644 --- a/src/insert.c +++ b/src/insert.c @@ -3182,7 +3182,10 @@ static int xferOptimization( } } #ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ + if( pDest->pCheck + && (db->mDbFlags & DBFLAG_Vacuum)==0 + && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) + ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif diff --git a/test/vacuum-into.test b/test/vacuum-into.test index 698d65f540..d559b7fb39 100644 --- a/test/vacuum-into.test +++ b/test/vacuum-into.test @@ -26,13 +26,36 @@ ifcapable {!vacuum} { forcedelete out.db do_execsql_test vacuum-into-100 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t1( + a INTEGER PRIMARY KEY, + b ANY, + c INT AS (b+1), --- See "2024-04-09" block + CHECK( typeof(b)!='integer' OR b>a-5 ) --- comment below + ); WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) INSERT INTO t1(a,b) SELECT x, randomblob(600) FROM c; CREATE INDEX t1b ON t1(b); DELETE FROM t1 WHERE a%2; SELECT count(*), sum(a), sum(length(b)) FROM t1; } {50 2550 30000} + +# Update 2024-04-09 for forum post eec177d68fe7fa2c. +# +# VACUUM INTO is sensitive to tables holding both generated columns +# and CHECK constraints. +# +# CHECK constraints are ignored for read-only databases in order to save +# memory (see check-in 34ddf02d3d21151b on 2014-05-21). But the xfer +# optimization normally only works if CHECK constraints match between the +# source and destination tables. So the xfer optimization was not +# working for VACUUM INTO when the source was a read-only database and the +# table held CHECK constraints. But if the table has generated columns, +# then the xfer optimization is required or else VACUUM will raise an +# error. +# +# Fix this by ignoring CHECK constraints when determining whether or not +# the xfer optimization can run while doing VACUUM. + do_execsql_test vacuum-into-110 { VACUUM main INTO 'out.db'; } {} @@ -88,11 +111,21 @@ do_catchsql_test vacuum-into-420 { # The ability to VACUUM INTO a read-only database db close +if {$tcl_platform(platform)=="windows"} { + file attributes test.db -readonly 1 +} else { + file attributes test.db -permissions 292 ;# 292 == 0444 +} sqlite3 db test.db -readonly 1 forcedelete test.db2 do_execsql_test vacuum-into-500 { VACUUM INTO 'test.db2'; } +if {$tcl_platform(platform)=="windows"} { + file attributes test.db -readonly 0 +} else { + file attributes test.db -permissions 420 ;# 420 = 0644 +} sqlite3 db2 test.db2 do_test vacuum-into-510 { db2 eval {SELECT name FROM sqlite_master ORDER BY 1} From 73a50fa42cbc2ddd1bf9e795a1407c3c746896e9 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 9 Apr 2024 15:09:54 +0000 Subject: [PATCH 12/19] Avoid some performance problems in the recover extension when recovering strategically corrupted databases. FossilOrigin-Name: e66a834bbab9d71e02b70588ad4c71eae2e13b50e299b0269d488402bc4a7fc5 --- ext/recover/dbdata.c | 24 +++++++++++++++++++----- ext/recover/recovercorrupt2.test | 28 ++++++++++++++++++++++++++++ ext/recover/sqlite3recover.c | 2 +- manifest | 18 +++++++++--------- manifest.uuid | 2 +- 5 files changed, 58 insertions(+), 16 deletions(-) diff --git a/ext/recover/dbdata.c b/ext/recover/dbdata.c index b6cb26ecd3..ca6371026e 100644 --- a/ext/recover/dbdata.c +++ b/ext/recover/dbdata.c @@ -494,6 +494,15 @@ static void dbdataValue( } } +/* This macro is a copy of the MX_CELL() macro in the SQLite core. Given +** a page-size, it returns the maximum number of cells that may be present +** on the page. */ +#define DBDATA_MX_CELL(pgsz) ((pgsz-8)/6) + +/* Maximum number of fields that may appear in a single record. This is +** the "hard-limit", according to comments in sqliteLimit.h. */ +#define DBDATA_MX_FIELD 32676 + /* ** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry. */ @@ -522,6 +531,9 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ assert( iOff+3+2<=pCsr->nPage ); pCsr->iCell = pTab->bPtr ? -2 : 0; pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); + if( pCsr->nCell>DBDATA_MX_CELL(pCsr->nPage) ){ + pCsr->nCell = DBDATA_MX_CELL(pCsr->nPage); + } } if( pTab->bPtr ){ @@ -566,19 +578,19 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ if( pCsr->iCell>=pCsr->nCell ){ bNextPage = 1; }else{ + int iCellPtr = iOff + 8 + nPointer + pCsr->iCell*2; - iOff += 8 + nPointer + pCsr->iCell*2; - if( iOff>pCsr->nPage ){ + if( iCellPtr>pCsr->nPage ){ bNextPage = 1; }else{ - iOff = get_uint16(&pCsr->aPage[iOff]); + iOff = get_uint16(&pCsr->aPage[iCellPtr]); } /* For an interior node cell, skip past the child-page number */ iOff += nPointer; /* Load the "byte of payload including overflow" field */ - if( bNextPage || iOff>pCsr->nPage ){ + if( bNextPage || iOff>pCsr->nPage || iOff<=iCellPtr ){ bNextPage = 1; }else{ iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload); @@ -661,7 +673,9 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ pCsr->iField++; if( pCsr->iField>0 ){ sqlite3_int64 iType; - if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){ + if( pCsr->pHdrPtr>=&pCsr->pRec[pCsr->nRec] + || pCsr->iField>=DBDATA_MX_FIELD + ){ bNextPage = 1; }else{ int szField = 0; diff --git a/ext/recover/recovercorrupt2.test b/ext/recover/recovercorrupt2.test index 29acc27a35..6c216308f0 100644 --- a/ext/recover/recovercorrupt2.test +++ b/ext/recover/recovercorrupt2.test @@ -524,5 +524,33 @@ do_test 7.1 { list [catch { $R finish } msg] $msg } {1 {file is not a database}} +reset_db +breakpoint +do_test 8.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 8192 pagesize 4096 filename db.sqlite +| page 1 offset 0 +| 0: ac ae b3 76 74 65 20 66 6f 72 6d 61 74 20 33 00 ...vte format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 01 00 00 00 02 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ................ +| 96: 00 2e 76 8a 0d ff ff ff 1e 0f cb 00 0f cb 00 00 ..v............. +| 4032: 00 00 00 00 00 00 00 00 00 00 00 33 01 06 17 19 ...........3.... +| 4048: 19 01 43 74 61 62 6c 65 54 61 62 6c 65 30 54 61 ..CtableTable0Ta +| 4064: 62 6c 65 30 02 43 52 45 41 54 45 20 54 41 42 4c ble0.CREATE TABL +| 4080: 45 20 54 61 62 6c 65 30 20 28 43 6f 6c 30 20 29 E Table0 (Col0 ) +| page 2 offset 4096 +| 0: 0d 00 00 00 00 10 00 00 00 00 00 00 00 00 00 00 ................ +| end db.sqlite +}]} {} + +do_test 8.1 { + set R [sqlite3_recover_init db main test.db2] + catch { $R run } + list [catch { $R finish } msg] $msg +} {0 {}} + finish_test diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index c445c51793..1d858c0ab9 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -1189,7 +1189,7 @@ static int recoverWriteSchema1(sqlite3_recover *p){ if( bTable && !bVirtual ){ if( SQLITE_ROW==sqlite3_step(pTblname) ){ const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0); - recoverAddTable(p, zTbl, iRoot); + if( zTbl ) recoverAddTable(p, zTbl, iRoot); } recoverReset(p, pTblname); } diff --git a/manifest b/manifest index 8197099293..0fa7d973a9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sread-only\sCHECK-constraint\soptimization\sof\s[34ddf02d3d21151b]\sinhibits\sthe\nxfer\soptimization\sfor\stables\swith\sCHECK\sconstraints.\s\sHowever,\sthe\sxfer\noptimization\sis\srequired\sfor\scorrect\soperation\sof\sVACUUM\sINTO\son\stables\sthat\ncontain\sgenerated\scolumns.\sFix\sthis\sby\signoring\sCHECK\sconstraints\swhen\nqualifying\sthe\sxfer\soptimization\swhile\srunning\sVACUUM.\s\sProblem\sreported\sby\n[forum:/forumpost/3ec177d68fe7fa2c|forum\spost\s3ec177d68fe7fa2c]. -D 2024-04-09T13:57:27.143 +C Avoid\ssome\sperformance\sproblems\sin\sthe\srecover\sextension\swhen\srecovering\sstrategically\scorrupted\sdatabases. +D 2024-04-09T15:09:54.458 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -478,13 +478,13 @@ F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69 F ext/rbu/sqlite3rbu.c 4a3376c0fb9a844a799ac529fb81260523f6b13c9f629bc270c632dbae5fc1f8 F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304 F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055 -F ext/recover/dbdata.c b7746c2ec801b453840831311be4b31f8c8f9dd97791060a69bbf12392c78949 +F ext/recover/dbdata.c d2e00d3cac74319c9c6def2e56ab2146b4f4ba5d820ab275e8da24e9766c247e F ext/recover/recover1.test c484d01502239f11b61f23c1cee9f5dd19fa17617f8974e42e74d64639c524cf F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a F ext/recover/recoverbuild.test c74170e0f7b02456af41838afeb5353fdb985a48cc2331d661bbabbca7c6b8e3 F ext/recover/recoverclobber.test 3ba6c0c373c5c63d17e82eced64c05c57ccaf26c1abe1ca7141334022a79f32e F ext/recover/recovercorrupt.test 64c081ad1200ae77b447da99eb724785d6bf71715f394543dc7689642e92bf49 -F ext/recover/recovercorrupt2.test b9b974f006340a1300b5a7e687bd6097c5238498181c6f92d87406a77ece430b +F ext/recover/recovercorrupt2.test 1418f1710debc24ff38276cedfcea234beb37a34205708e7e3e6d76cc4a979db F ext/recover/recoverfault.test 9d9f88eeb222615a25e7514f234c950d46bee20d24cd8db49d8fff8d650dcfe1 F ext/recover/recoverfault2.test 730e7371bcda769554d15460cb23126abba1be8eca9539ccabf63623e7bb7e09 F ext/recover/recoverold.test 68db3d6f85dd2b98e785b6c4da4f5eea4bbe52ccf6674d9a94c7506dc92596aa @@ -492,7 +492,7 @@ F ext/recover/recoverpgsz.test 3658ab8e68475b1bb87d6af88baa04551c84b73280a566a1b F ext/recover/recoverrowid.test f948bf4024a5f41b0e21b8af80c60564c5b5d78c05a8d64fc00787715ff9f45f F ext/recover/recoverslowidx.test 5205a9742dd9490ee99950dabb622307355ef1662dea6a3a21030057bfd81411 F ext/recover/recoversql.test e66d01f95302a223bcd3fd42b5ee58dc2b53d70afa90b0d00e41e4b8eab20486 -F ext/recover/sqlite3recover.c e6eb20c469bcdb96f297f2241860bccabf9f036bfa7f3d47bcc6ca1191b108dc +F ext/recover/sqlite3recover.c 65ef0f56301a16c0536c9839fb7e23540c9c4f75da0afe3b7b4d163c8f624404 F ext/recover/sqlite3recover.h 011c799f02deb70ab685916f6f538e6bb32c4e0025e79bfd0e24ff9c74820959 F ext/recover/test_recover.c fd871a40f2238022bedcbdf3cb493b91225edaa94d6ae8892af97a10e7ccc4ba F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -2184,8 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c02956cd5cbf87027ba201b5d273e0efd429836ee6cabdc2f6d5b8096c876b7b -R 14ee2c4095f13f5f59e64d6dfae705ab -U drh -Z 40f988c61dc13a84e5a7ab5ecd628e6c +P a6e26e778812c8409fca77183e24d3b70189c4d02fce10c7e74cd4ccc8c8ea97 +R b9c21ffc9d4065568ea5ca672bfa94a1 +U dan +Z a2100870dc1c15edbdbfdff1cdb0d2d0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3f94593291..e73f4c80e2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a6e26e778812c8409fca77183e24d3b70189c4d02fce10c7e74cd4ccc8c8ea97 \ No newline at end of file +e66a834bbab9d71e02b70588ad4c71eae2e13b50e299b0269d488402bc4a7fc5 \ No newline at end of file From 495ce22d49a8be9df2753b1ab33b354be2066a20 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 9 Apr 2024 19:23:18 +0000 Subject: [PATCH 13/19] Fix a "shift exponent is too large" usan error that could occur in the fts3 snippet() function when a query phrase contains more than 64 tokens. FossilOrigin-Name: 6d2c7ac4ecfc1ec408cb1820985a70650d7cc5613b4b8fd1829f68ea48fce6a3 --- ext/fts3/fts3_snippet.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fts3snippet2.test | 11 +++++++++++ 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 227c5f00f6..f6caabf4c9 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -446,7 +446,7 @@ static void fts3SnippetDetails( } mCover |= mPhrase; - for(j=0; jnToken; j++){ + for(j=0; jnToken && jnSnippet; j++){ mHighlight |= (mPos>>j); } diff --git a/manifest b/manifest index 0fa7d973a9..764ffbd97a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\ssome\sperformance\sproblems\sin\sthe\srecover\sextension\swhen\srecovering\sstrategically\scorrupted\sdatabases. -D 2024-04-09T15:09:54.458 +C Fix\sa\s"shift\sexponent\sis\stoo\slarge"\susan\serror\sthat\scould\soccur\sin\sthe\sfts3\ssnippet()\sfunction\swhen\sa\squery\sphrase\scontains\smore\sthan\s64\stokens. +D 2024-04-09T19:23:18.637 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -74,7 +74,7 @@ F ext/fts3/fts3_hash.c 8b6e31bfb0844c27dc6092c2620bdb1fca17ed613072db057d96952c6 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c 305ce7fb6036484085b5556a9c8e62acdc7763f0f4cdf5fd538212a9f3720116 F ext/fts3/fts3_porter.c e19807ce0ae31c1c6e9898e89ecc93183d7ec224ea101af039722a4f49e5f2b8 -F ext/fts3/fts3_snippet.c 4d6523e3eddeb7b46e7a82b3476a0a86a0c04821e0e2b8dd40f45ee28057cb13 +F ext/fts3/fts3_snippet.c 610328fe128c047c6b0eba77768982ccf3933daae095d497949a75c9dfd47409 F ext/fts3/fts3_term.c 845f0e2456b1be42f7f1bec1da1dfc05bc347531eff90775ffc6698902c281de F ext/fts3/fts3_test.c d8d7b2734f894e8a489987447658e374cdd3a3bc8575c401decf1911cb7c6454 F ext/fts3/fts3_tokenize_vtab.c 7fd9ef364f257b97218b9c331f2378e307375c592f70fd541f714e747d944962 @@ -1200,7 +1200,7 @@ F test/fts3rank.test cd99bc83a3c923c8d52afd90d86979cf05fc41849f892faeac3988055ef F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0 F test/fts3shared.test 57e26a801f21027b7530da77db54286a6fe4997e F test/fts3snippet.test 0887196d67cffbe365edde535b95ecc642a532ce8551ccd9a73aab5999c3ffae -F test/fts3snippet2.test e79afeb1f673713f96d7fc5655726081975399d11e659d15553207be43301dc4 +F test/fts3snippet2.test 03f6738ab3897bea2ba6be424a0613872e167acbf37a66200d655d737b470f65 F test/fts3sort.test ed34c716a11cc2009a35210e84ad5f9c102362ca F test/fts3tok1.test a663f4cac22a9505400bc22aacb818d7055240409c28729669ea7d4cc2120d15 F test/fts3tok_err.test 52273cd193b9036282f7bacb43da78c6be87418d @@ -2184,8 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a6e26e778812c8409fca77183e24d3b70189c4d02fce10c7e74cd4ccc8c8ea97 -R b9c21ffc9d4065568ea5ca672bfa94a1 +P e66a834bbab9d71e02b70588ad4c71eae2e13b50e299b0269d488402bc4a7fc5 +R 75450b3380949a2b13147c3a28eab747 U dan -Z a2100870dc1c15edbdbfdff1cdb0d2d0 +Z 0a239b81f481c07e165698bd6a5a4da2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e73f4c80e2..71eaeff0e3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e66a834bbab9d71e02b70588ad4c71eae2e13b50e299b0269d488402bc4a7fc5 \ No newline at end of file +6d2c7ac4ecfc1ec408cb1820985a70650d7cc5613b4b8fd1829f68ea48fce6a3 \ No newline at end of file diff --git a/test/fts3snippet2.test b/test/fts3snippet2.test index 607b01e0f8..f55a5f203a 100644 --- a/test/fts3snippet2.test +++ b/test/fts3snippet2.test @@ -55,5 +55,16 @@ do_execsql_test 2.2 { '(def AND (one NEAR abc)) OR one' } {one} +#------------------------------------------------------------------------- + +do_execsql_test 3.0 { +CREATE VIRTUAL TABLE f USING fts3(a,b); +INSERT INTO f VALUES (101,x'056522650565056505650d051e056505650565286505650565056505056505650565056505650565056505650565056505650565056505656505650565056505650d05650505656505650565ef65056505844c746e65650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056f05650565056505650565056505650565050565056505640565056505650565056505651e05650565056505650565056505650505656565056505650565056505651e0565056505650565056505650565052265056505650569056505650565056505650565056505650565056505650500406505650565056505650565056505000101e5c501014b010101c501c5c501010101f5010201010101014101017373737373737373737373737373737373737373737373737373737330737373737373737373737373737365056505650d051e05650565056528056505650d05650505656505650565650565056505650565056505e505650565056505656505650565056505650d05650505656505650565ef65056505844c746e65650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565286505c705650565050565059494949494949494949494949494949494949494949494949494949494949494949494650565056505650565056505650565056505650565056505656505650565056505650d05650505656505650565ef650565058405056505650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056505650565056505650505650565056505650565056505650565056500000000000000000000000000000000000000000000000000000000000000000565056505656505650565056505650d056500000000000000000000000000000000000000000000000000000000000000000100000000000000ed0000000000ffffffffffffffffffffff0007ffffff0001c5c50001c5c50001c5c50001c5c50001c5c50001c5c50001e5c50001c5c50001c5c50001c5c50001c5c50001c5c5000100000014720000000000000016dac5c50001c5c50001c5c50001c5c50001c5c50001c5c50d0505656505650565ef650565058405056505650565056505650565056505650565056505650565058405800465056505650565056505651e650565056505650565056505650d05650505656505650565050565650584058005650565056505650565056505650565056522650565056505650d051e056505650561286505c70565056505056505650565056505650565056505650565056505650565056505656505650565056505650d05650505656505650565ef650565058405056505650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056505650565056505650565226505737373737373737373737373737373737373737373737373737373737373737373737373737373737373737373737373733a73737373737373737373737373737373737373737373737373737373737373737373737373737373737c7373737365056505650d051e05650565056528650565056505650505650565056505650565056505650565056505e505650565056505656505650565056505650d05650505656505650565ef65056505'); +} + +do_execsql_test 3.1 { + SELECT length(snippet(f)) FROM f WHERE b MATCH x'0565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056505650565056505650565056505650565050565056505640565056505650565056505651e05650565056522650565056505650d051e056505650565286505650565056505056505650565056505650565056505650565056505650565056505656505650565056505650d05650505656505650565ef65056505844c746e65650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056f05650565056505650565056505650565050565056505640565056505650565056505651e05650565056505650565056505650505656565056505650565056505651e0565056505650565056505650565052265056505650569056505650565056505650565056505650565056505650500406505650565056505650565056505000101e5c501014b010101c501c5c501010101f50102010101010141010141010001017bf15905000000000017'; +} {192} + set sqlite_fts3_enable_parentheses 0 finish_test From b6d108f0b37a0fcda58647cfb75d18909a496d77 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 11 Apr 2024 19:08:12 +0000 Subject: [PATCH 14/19] Fix handling of the SQLITE_OMIT_WAL check in one place. Resolves [forum:87cc13302de160eb|forum post 87cc13302de160eb]. FossilOrigin-Name: 5dae6e6df4921f42e45c6c8de40853ab63f53a4bd1d9088a8cdac957ce62f196 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 764ffbd97a..eab5ebd169 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\s"shift\sexponent\sis\stoo\slarge"\susan\serror\sthat\scould\soccur\sin\sthe\sfts3\ssnippet()\sfunction\swhen\sa\squery\sphrase\scontains\smore\sthan\s64\stokens. -D 2024-04-09T19:23:18.637 +C Fix\shandling\sof\sthe\sSQLITE_OMIT_WAL\scheck\sin\sone\splace.\sResolves\s[forum:87cc13302de160eb|forum\spost\s87cc13302de160eb]. +D 2024-04-11T19:08:12.217 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -737,7 +737,7 @@ F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d87210 F src/os_unix.c 6227cbc4ac93046f121436886cf3712da6f4e2082af6314f976eeae1d86b794a F src/os_win.c 6ff43bac175bd9ed79e7c0f96840b139f2f51d01689a638fd05128becf94908a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c ff60e98138d2499082ac6230f01ac508aba545315debccfca2fd6042f5f10fcd +F src/pager.c 9beb80f6e330dd63c5d8ba0f7a7f3a55fff22067a68d424949c389bfc6fa0c56 F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a F src/parse.y 5bcef16094213efcc365a9d4dc4e3131f09251dc8838dce4a9e5f9764bff5b82 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 @@ -2184,8 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e66a834bbab9d71e02b70588ad4c71eae2e13b50e299b0269d488402bc4a7fc5 -R 75450b3380949a2b13147c3a28eab747 -U dan -Z 0a239b81f481c07e165698bd6a5a4da2 +P 6d2c7ac4ecfc1ec408cb1820985a70650d7cc5613b4b8fd1829f68ea48fce6a3 +R 0071938b1b4e5105037f1334ca733633 +U stephan +Z d6eaf14882bbc596d43b8bd29ecbee42 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 71eaeff0e3..4d8c2440ec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d2c7ac4ecfc1ec408cb1820985a70650d7cc5613b4b8fd1829f68ea48fce6a3 \ No newline at end of file +5dae6e6df4921f42e45c6c8de40853ab63f53a4bd1d9088a8cdac957ce62f196 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 37588f0b25..268dc7db31 100644 --- a/src/pager.c +++ b/src/pager.c @@ -7087,7 +7087,7 @@ sqlite3_file *sqlite3PagerFile(Pager *pPager){ ** This will be either the rollback journal or the WAL file. */ sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ -#if SQLITE_OMIT_WAL +#ifdef SQLITE_OMIT_WAL return pPager->jfd; #else return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; From c494fdb580bc7c738a3a53be1ad2fd2ca3aa08e5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 12 Apr 2024 02:57:52 +0000 Subject: [PATCH 15/19] Mark the BTree cell overflow cache as invalid whenever the rowid goes invalid. FossilOrigin-Name: 74c9e19c92c887012aebbe96450f6ed7a60ba22d6e3edbaa39a0f989fb7f2901 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/btree.c | 4 ++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index eab5ebd169..2daa24ebaa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\shandling\sof\sthe\sSQLITE_OMIT_WAL\scheck\sin\sone\splace.\sResolves\s[forum:87cc13302de160eb|forum\spost\s87cc13302de160eb]. -D 2024-04-11T19:08:12.217 +C Mark\sthe\sBTree\scell\soverflow\scache\sas\sinvalid\swhenever\sthe\srowid\sgoes\sinvalid. +D 2024-04-12T02:57:52.425 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -689,7 +689,7 @@ F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645 F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 -F src/btree.c 285b493d843e7ba8ef78b6ae7d31238e904901dbc0c484f7904de4cf18fd8802 +F src/btree.c a654c4b4d84821f4fdeb322df185317493be30c1b114cb75d56adb50dd3f661c F src/btree.h 55066f513eb095db935169dab1dc2f7c7a747ef223c533f5d4ad4dfed346cbd0 F src/btreeInt.h 98aadb6dcb77b012cab2574d6a728fad56b337fc946839b9898c4b4c969e30b6 F src/build.c 02f5b25ca854c83b5015cb02b8c9ab236c60b1795528675aee8a5070e58da52a @@ -2184,8 +2184,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6d2c7ac4ecfc1ec408cb1820985a70650d7cc5613b4b8fd1829f68ea48fce6a3 -R 0071938b1b4e5105037f1334ca733633 -U stephan -Z d6eaf14882bbc596d43b8bd29ecbee42 +P 5dae6e6df4921f42e45c6c8de40853ab63f53a4bd1d9088a8cdac957ce62f196 +R 991bf551528ac37b754c129e51db564b +T *branch * btree-ovfl-cache +T *sym-btree-ovfl-cache * +T -sym-trunk * +U drh +Z d43adcfd8cc8586ed5a00034a7473bf9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4d8c2440ec..5ce114455e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5dae6e6df4921f42e45c6c8de40853ab63f53a4bd1d9088a8cdac957ce62f196 \ No newline at end of file +74c9e19c92c887012aebbe96450f6ed7a60ba22d6e3edbaa39a0f989fb7f2901 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index ddfa3c63f3..e957ccb199 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9521,7 +9521,7 @@ int sqlite3BtreeInsert( }else if( loc<0 && pPage->nCell>0 ){ assert( pPage->leaf ); idx = ++pCur->ix; - pCur->curFlags &= ~BTCF_ValidNKey; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); }else{ assert( pPage->leaf ); } @@ -9551,7 +9551,7 @@ int sqlite3BtreeInsert( */ if( pPage->nOverflow ){ assert( rc==SQLITE_OK ); - pCur->curFlags &= ~(BTCF_ValidNKey); + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); rc = balance(pCur); /* Must make sure nOverflow is reset to zero even if the balance() From 366b419d0084fad8b60ec47e89b882f8218b25e6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 12 Apr 2024 11:28:35 +0000 Subject: [PATCH 16/19] Test case for the fix in the previous check-in. FossilOrigin-Name: 0cf4d835dae260b01178e94e77be6b8a130f2031e898ef79ceba8df6c2bba58f --- manifest | 15 ++++++--------- manifest.uuid | 2 +- test/upsert1.test | 24 ++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 2daa24ebaa..3d880dd346 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Mark\sthe\sBTree\scell\soverflow\scache\sas\sinvalid\swhenever\sthe\srowid\sgoes\sinvalid. -D 2024-04-12T02:57:52.425 +C Test\scase\sfor\sthe\sfix\sin\sthe\sprevious\scheck-in. +D 2024-04-12T11:28:35.112 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1906,7 +1906,7 @@ F test/upfrom2.test 66f3ebf721b3cebd922faee5c386bf244f816d416b57c000753ff51af623 F test/upfrom3.test 6130f24ebf97f5ea865e5d2a14a2d543fe5428a62e87cc60f62d875e45c1f5f0 F test/upfrom4.test 78f742a6577c91a7a55c64edb8811004e7c6aa99b8d57b2320f70a918c357807 F test/upfromfault.test 3a10075a0043f0c4fad6614b2c371f88a8ba5a4acab68b907438413865d6a8d6 -F test/upsert1.test a512e2f884d3a36159fce2e45108c236f78ae38e35bda55f4050db580ceb25d3 +F test/upsert1.test beba4316fbd4b7b9d76784313f6129a548cfe7abea04d46db33e2efce1ab0ac2 F test/upsert2.test 720e94d09f7362a282bc69b3c6b83d51daeaaf0440eb4920a08b86518b8c7496 F test/upsert3.test 88d7d590a1948a9cb6eac1b54b0642f67a9f35a1fc0f19b200e97d5d39e3179c F test/upsert4.test 25d2a1da92f149331ae0c51ca6e3eee78189577585eab92de149900d62994fa5 @@ -2184,11 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5dae6e6df4921f42e45c6c8de40853ab63f53a4bd1d9088a8cdac957ce62f196 -R 991bf551528ac37b754c129e51db564b -T *branch * btree-ovfl-cache -T *sym-btree-ovfl-cache * -T -sym-trunk * +P 74c9e19c92c887012aebbe96450f6ed7a60ba22d6e3edbaa39a0f989fb7f2901 +R e5ebeec2e7c87917a5e915be853b1124 U drh -Z d43adcfd8cc8586ed5a00034a7473bf9 +Z 367db0044284a550a3cf0e6c6563bd86 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5ce114455e..3a012e5380 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -74c9e19c92c887012aebbe96450f6ed7a60ba22d6e3edbaa39a0f989fb7f2901 \ No newline at end of file +0cf4d835dae260b01178e94e77be6b8a130f2031e898ef79ceba8df6c2bba58f \ No newline at end of file diff --git a/test/upsert1.test b/test/upsert1.test index 7818311330..8af273a89a 100644 --- a/test/upsert1.test +++ b/test/upsert1.test @@ -268,4 +268,28 @@ do_catchsql_test upsert1-1210 { INSERT INTO t1(a,b) VALUES(1,2) ON CONFLICT(b+?1) DO NOTHING; } {1 {ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint}} +# 2024-04-11 https://sqlite.org/forum/forumpost/284955a3cd454a15 +# Incorrect value passed into a trigger that fires as the result of +# an upsert. +# +reset_db +do_execsql_test upsert1-1300 { + CREATE TABLE t1(x INT, y TEXT); + INSERT INTO t1 VALUES + (11, printf('%.9000c','a')), + (11, printf('%.9000c','a')), + (33, printf('%.9000c','b')), + (33, printf('%.9000c','b')); + CREATE TABLE t2(x INT UNIQUE, y TEXT); + CREATE TRIGGER r1 BEFORE UPDATE ON t2 BEGIN + SELECT raise(ABORT,'Incorrect old.y value passed to trigger!') + WHERE old.y != new.y; + /* ^^^ This trigger will fire and cause the ABORT if the problem has + ** not been fixed, or if there is a regression. */ + END; + INSERT INTO t2(x, y) SELECT x, y FROM t1 + WHERE true + ON CONFLICT (x) DO UPDATE SET y = excluded.y; +} {} + finish_test From ae7ede4a4f70648170b05d176b877f97734d1383 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 12 Apr 2024 12:32:09 +0000 Subject: [PATCH 17/19] New assert() statements to verify the correctness of the BTCF_AtLast flag on btree cursors. FossilOrigin-Name: 4efecd6167de71500c90b63155eba1b8567c90e9d1e282fbea54130f9ee21813 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 31 +++++++++++++++++++------------ 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 3d880dd346..070fa270cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\scase\sfor\sthe\sfix\sin\sthe\sprevious\scheck-in. -D 2024-04-12T11:28:35.112 +C New\sassert()\sstatements\sto\sverify\sthe\scorrectness\sof\sthe\sBTCF_AtLast\sflag\non\sbtree\scursors. +D 2024-04-12T12:32:09.806 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -689,7 +689,7 @@ F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645 F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 -F src/btree.c a654c4b4d84821f4fdeb322df185317493be30c1b114cb75d56adb50dd3f661c +F src/btree.c b489280734e3c085774ecc86a07f5ad5b805d86e2cb28650c3b3e141673993d6 F src/btree.h 55066f513eb095db935169dab1dc2f7c7a747ef223c533f5d4ad4dfed346cbd0 F src/btreeInt.h 98aadb6dcb77b012cab2574d6a728fad56b337fc946839b9898c4b4c969e30b6 F src/build.c 02f5b25ca854c83b5015cb02b8c9ab236c60b1795528675aee8a5070e58da52a @@ -2184,8 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 74c9e19c92c887012aebbe96450f6ed7a60ba22d6e3edbaa39a0f989fb7f2901 -R e5ebeec2e7c87917a5e915be853b1124 +P 0cf4d835dae260b01178e94e77be6b8a130f2031e898ef79ceba8df6c2bba58f +R 3352218b957f71135e62b28b93e477b6 U drh -Z 367db0044284a550a3cf0e6c6563bd86 +Z 55f6b761ddb19ac9081b9562cfbe54ea # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3a012e5380..01cbdd4cd6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0cf4d835dae260b01178e94e77be6b8a130f2031e898ef79ceba8df6c2bba58f \ No newline at end of file +4efecd6167de71500c90b63155eba1b8567c90e9d1e282fbea54130f9ee21813 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e957ccb199..91c9e4fb57 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5624,6 +5624,23 @@ int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ return rc; } +#ifdef SQLITE_DEBUG +/* The cursors is CURSOR_VALID and has BTCF_AtLast set. Verify that +** this flags are true for a consistent database. +** +** This routine is is called from within assert() statements only. +** It is an internal verification routine and does not appear in production +** builds. +*/ +static int cursorIsAtLastEntry(BtCursor *pCur){ + int ii; + for(ii=0; iiiPage; ii++){ + if( pCur->aiIdx[ii]!=pCur->apPage[ii]->nCell ) return 0; + } + return pCur->ix==pCur->pPage->nCell-1 && pCur->pPage->leaf!=0; +} +#endif + /* Move the cursor to the last entry in the table. Return SQLITE_OK ** on success. Set *pRes to 0 if the cursor actually points to something ** or set *pRes to 1 if the table is empty. @@ -5652,18 +5669,7 @@ int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ /* If the cursor already points to the last entry, this is a no-op. */ if( CURSOR_VALID==pCur->eState && (pCur->curFlags & BTCF_AtLast)!=0 ){ -#ifdef SQLITE_DEBUG - /* This block serves to assert() that the cursor really does point - ** to the last entry in the b-tree. */ - int ii; - for(ii=0; iiiPage; ii++){ - assert( pCur->aiIdx[ii]==pCur->apPage[ii]->nCell ); - } - assert( pCur->ix==pCur->pPage->nCell-1 || CORRUPT_DB ); - testcase( pCur->ix!=pCur->pPage->nCell-1 ); - /* ^-- dbsqlfuzz b92b72e4de80b5140c30ab71372ca719b8feb618 */ - assert( pCur->pPage->leaf ); -#endif + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); *pRes = 0; return SQLITE_OK; } @@ -5716,6 +5722,7 @@ int sqlite3BtreeTableMoveto( } if( pCur->info.nKeycurFlags & BTCF_AtLast)!=0 ){ + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); *pRes = -1; return SQLITE_OK; } From 1d40237bc8b1c6ec71dfe763329e1aceacf022fa Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 12 Apr 2024 13:05:36 +0000 Subject: [PATCH 18/19] Add assert() statements to help verify the overflow page cache. FossilOrigin-Name: 0ebc65481f4a3e7974558adea51c620a025bc0e76c0a139e549b56c5abe0cabb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 070fa270cd..17af620303 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\sassert()\sstatements\sto\sverify\sthe\scorrectness\sof\sthe\sBTCF_AtLast\sflag\non\sbtree\scursors. -D 2024-04-12T12:32:09.806 +C Add\sassert()\sstatements\sto\shelp\sverify\sthe\soverflow\spage\scache. +D 2024-04-12T13:05:36.039 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -689,7 +689,7 @@ F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645 F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 -F src/btree.c b489280734e3c085774ecc86a07f5ad5b805d86e2cb28650c3b3e141673993d6 +F src/btree.c 8c2d32661d9b3333b5e2192a9baf0f34aaf50224c7e0130bdb96a1e5dee9009c F src/btree.h 55066f513eb095db935169dab1dc2f7c7a747ef223c533f5d4ad4dfed346cbd0 F src/btreeInt.h 98aadb6dcb77b012cab2574d6a728fad56b337fc946839b9898c4b4c969e30b6 F src/build.c 02f5b25ca854c83b5015cb02b8c9ab236c60b1795528675aee8a5070e58da52a @@ -2184,8 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0cf4d835dae260b01178e94e77be6b8a130f2031e898ef79ceba8df6c2bba58f -R 3352218b957f71135e62b28b93e477b6 +P 4efecd6167de71500c90b63155eba1b8567c90e9d1e282fbea54130f9ee21813 +R 733b2640d8dc44517728193a890baeff U drh -Z 55f6b761ddb19ac9081b9562cfbe54ea +Z 6194d89088ab528754e159fb92573073 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 01cbdd4cd6..621be8e4f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4efecd6167de71500c90b63155eba1b8567c90e9d1e282fbea54130f9ee21813 \ No newline at end of file +0ebc65481f4a3e7974558adea51c620a025bc0e76c0a139e549b56c5abe0cabb \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 91c9e4fb57..756ec942c3 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5143,6 +5143,10 @@ static int accessPayload( memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); pCur->curFlags |= BTCF_ValidOvfl; }else{ + /* Sanity check the validity of the overflow page cache */ + assert( pCur->aOverflow[0]==nextPage || pCur->aOverflow[0]==0 ); + assert( pCur->aOverflow[0]!=0 || pCur->aOverflow[offset/ovflSize]==0 ); + /* If the overflow page-list cache has been allocated and the ** entry for the first required overflow page is valid, skip ** directly to it. From 0221cacea70d50f8cf5033aecfa104e4582a1532 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 12 Apr 2024 18:46:34 +0000 Subject: [PATCH 19/19] If a build fails in testrunner.tcl, do not attempt to run the jobs that depend on that build. Instead, report those jobs as having been skipped. FossilOrigin-Name: b40580be719a129ecd1aa3c69d1086c967d063920fdd48617c864e73c059abc1 --- manifest | 13 ++++++------- manifest.uuid | 2 +- test/testrunner.tcl | 17 +++++++++++++++-- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index ceff35c009..c7c6ea43af 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sand\snew\stests\slogic\sto\sensure\sthat\sthe\sbtree\soverflow\spage\scache\sis\nonly\sused\swhen\sit\sis\sconsistent.\s\sThis\sresolves\sthe\smalfunction\sobserved\nin\s[forum:/forumpost/284955a3cd454a15|forum\spost\s284955a3cd454a15]. -D 2024-04-12T15:02:16.718 +C If\sa\sbuild\sfails\sin\stestrunner.tcl,\sdo\snot\sattempt\sto\srun\sthe\sjobs\sthat\ndepend\son\sthat\sbuild.\s\sInstead,\sreport\sthose\sjobs\sas\shaving\sbeen\sskipped. +D 2024-04-12T18:46:34.309 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1690,7 +1690,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl fe617b88c7eb08bdf983d2aaa31c20fbf439eee7b8e0d61ca636fcd0c305bbbf -F test/testrunner.tcl b48a8fc17e3b200244f53d60348afa7fe9482400d164e95709890743af9536d0 +F test/testrunner.tcl 1386667c04207d0a540ce1a9bc5ee0b734f7a3ba856c14a03943fb4f32de55bb F test/testrunner_data.tcl 3d36660cfd55ea5e20e661e8f94c0520feebcb437848f9b98b33c483cc479c0c F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2184,9 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5dae6e6df4921f42e45c6c8de40853ab63f53a4bd1d9088a8cdac957ce62f196 0ebc65481f4a3e7974558adea51c620a025bc0e76c0a139e549b56c5abe0cabb -R 733b2640d8dc44517728193a890baeff -T +closed 0ebc65481f4a3e7974558adea51c620a025bc0e76c0a139e549b56c5abe0cabb +P 5dede50d9e7b6942df9f7b00fbfeaa2103c36c5da01d63d88136fb0ef4b7d26d +R 4f3eb54a5cce9aa23292ec96983abdb3 U drh -Z 49129cd4194bac4dbb6265651119b7c1 +Z 1648e8675f24aa01764212367c0fba7b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8b1f8f1238..b40ee8722e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5dede50d9e7b6942df9f7b00fbfeaa2103c36c5da01d63d88136fb0ef4b7d26d \ No newline at end of file +b40580be719a129ecd1aa3c69d1086c967d063920fdd48617c864e73c059abc1 \ No newline at end of file diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 05f7d4cc98..fd82716769 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -270,7 +270,7 @@ set TRG(schema) { /* Fields updated as jobs run */ starttime INTEGER, endtime INTEGER, - state TEXT CHECK( state IN ('', 'ready', 'running', 'done', 'failed') ), + state TEXT CHECK( state IN ('','ready','running','done','failed','omit') ), output TEXT ); @@ -447,6 +447,10 @@ if {[llength $argv]==1 } job { display_job [array get job] } + set nOmit [db one {SELECT count(*) FROM jobs WHERE state='omit'}] + if {$nOmit} { + puts "$nOmit jobs omitted due to failures" + } } mydb close @@ -975,11 +979,16 @@ proc make_new_testset {} { proc mark_job_as_finished {jobid output state endtm} { r_write_db { + if {$state=="failed"} { + set childstate omit + } else { + set childstate ready + } trdb eval { UPDATE jobs SET output=$output, state=$state, endtime=$endtm WHERE jobid=$jobid; - UPDATE jobs SET state='ready' WHERE depid=$jobid; + UPDATE jobs SET state=$childstate WHERE depid=$jobid; } } } @@ -1202,6 +1211,10 @@ proc run_testset {} { puts "FAILED: $displayname" } } + set nOmit [trdb one {SELECT count(*) FROM jobs WHERE state='omit'}] + if {$nOmit>0} { + puts "$nOmit jobs skipped due to prior failures" + } } puts "\nTest database is $TRG(dbname)"