From f84cbd167689edf19c4905e515325a3a0c7cabe4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 12 Jan 2023 13:25:48 +0000 Subject: [PATCH 1/5] Attempt to provide a mechanism to do early termination of long-running statement preparation by invoking the progress handler at strategic points during sqlite3_parpare(). This experiment shows that sqlite3_prepare() might leave the resulting prepared statement uninitialized following an interrupt. FossilOrigin-Name: 79636f2d80aee70832913a78933da2a7e30cc037810b93903ebbc1925ea93fef --- manifest | 23 +++++++++++++---------- manifest.uuid | 2 +- src/select.c | 11 +++++++---- src/sqliteInt.h | 1 + src/util.c | 17 +++++++++++++++++ src/vdbeaux.c | 1 + src/where.c | 7 ++++++- 7 files changed, 46 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 863f3ea91d..0227e85025 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sduplicate\ssemicolon\sin\sbtreeInt.h.\n[forum:/forumpost/8db1711ca2|Forum\spost\s8db1711ca2] -D 2023-01-11T20:52:15.984 +C Attempt\sto\sprovide\sa\smechanism\sto\sdo\searly\stermination\sof\slong-running\nstatement\spreparation\sby\sinvoking\sthe\sprogress\shandler\sat\sstrategic\spoints\nduring\ssqlite3_parpare().\s\sThis\sexperiment\sshows\sthat\ssqlite3_prepare()\smight\nleave\sthe\sresulting\sprepared\sstatement\suninitialized\sfollowing\san\sinterrupt. +D 2023-01-12T13:25:48.952 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -645,12 +645,12 @@ F src/printf.c ff4b05e38bf928ff1b80d3dda4f977b10fe39ecbfe69c018224c7e5594fb2455 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 146222acbf9ea4e7f99c45a9e3cb4a0ba7e02c4f0df5186094ea1f9ea9907a02 +F src/select.c cb25679cf2a77526dccdcfb2ee87746a7dabacd583eb1c72b764a5d611694f6c F src/shell.c.in f7c75d1a9f900516e40f17f040668d5797592344bd88cff7ee7df586de6893c6 F src/sqlite.h.in 231d505c86ac4688b82259529273fe5fb2cce86cb80ea50628ff776113b5c706 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h 079ccd9c161f4b74967188fd6321810159fdc4c32371b68559719828fac20f43 +F src/sqliteInt.h d817f813a6f48f488fdde907222f0acd3170226d2dd4e528c0d8c7c141b4026f F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -715,13 +715,13 @@ F src/trigger.c 5e68b790f022b8dafbfb0eb244786512a95c9575fc198719d2557d73e5795858 F src/update.c f118e51768d2c1309e3c81e9f91141b22b8a1339cbc5969b1b2d810feaa25b22 F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 -F src/util.c 0e5cf9062a796f0f1b6b3228b121b2344932c05425f7c8b5a7cb245812473bbd +F src/util.c c4941abc630852692507b7bfc936596d85491a1edf8e1fc9e09c3fb4f111cdf3 F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd F src/vdbe.c 238635c1c40d42d9ded72994b81d4127f99d6d09e9279bdd37f6f34f4025adee F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c F src/vdbeInt.h fc15815b7bdafbb27e7f027faba2b0112e87d382c0d72241672528806ebc0db5 F src/vdbeapi.c 4ee67890913c1d2469c68e3ad2e7ddeab57ac5924a64bbfd0906a8ea0d542c7f -F src/vdbeaux.c 9eb7394126b88e55f81321d5e87636e0cd5e5856818d644d8abd6bfbb9c0be9d +F src/vdbeaux.c 118ef7fbc1be7ef359274a731ed4120f9e3ac2f1b677ee62103d150952f4bcd2 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 316d518115f3720b4097f0231e2a3d6eefd06c787eccf44972f8d8f462153421 F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -732,7 +732,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c d0d8e3cb2c11e77ba0f8f9ed8eada9d84dbd377167cdcf387b8eeb824c35a3ad +F src/where.c 8022eaf219860cbf60b62036b3695794d68cf577fec141df35846f0b3c88f9ab F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 76bca3379219880d2527493b71a3be49e696f75396d3481e4de5d4ceec7886b2 F src/whereexpr.c 7c5671a04b00c876bec5e99fd4e6f688065feb4773160fbf76fd7900d2901777 @@ -2068,8 +2068,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ab5bcb91cda45576ae9f3f272ec92eb3be3c26436a440ebb89f51f49c42e0fd2 -R 06e2ca2ba2e5fe806ceb97c200db87f2 +P 7526c46632578a2b602622b9debc406b52af4a42cc880970c4307d13853d59d3 +R 6e5c5a2e308bedb9ef7a7686c27c4f8f +T *branch * progress-during-prepare +T *sym-progress-during-prepare * +T -sym-trunk * U drh -Z 3af3ada0fac1f257d08b7786ac3b51a1 +Z 15a0449b6bca5de3ec6c827c565351a6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e0cc37f71e..641465e452 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7526c46632578a2b602622b9debc406b52af4a42cc880970c4307d13853d59d3 \ No newline at end of file +79636f2d80aee70832913a78933da2a7e30cc037810b93903ebbc1925ea93fef \ No newline at end of file diff --git a/src/select.c b/src/select.c index 2288858ea7..20bab20331 100644 --- a/src/select.c +++ b/src/select.c @@ -2211,7 +2211,7 @@ int sqlite3ColumnsFromExprList( *pnCol = nCol; *paCol = aCol; - for(i=0, pCol=aCol; imallocFailed; i++, pCol++){ + for(i=0, pCol=aCol; inErr; i++, pCol++){ struct ExprList_item *pX = &pEList->a[i]; struct ExprList_item *pCollide; /* Get an appropriate name for the column @@ -2261,7 +2261,10 @@ int sqlite3ColumnsFromExprList( if( zName[j]==':' ) nName = j; } zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt); - if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt); + sqlite3ProgressCheck(pParse); + if( cnt>3 ){ + sqlite3_randomness(sizeof(cnt), &cnt); + } } pCol->zCnName = zName; pCol->hName = sqlite3StrIHash(zName); @@ -2274,14 +2277,14 @@ int sqlite3ColumnsFromExprList( } } sqlite3HashClear(&ht); - if( db->mallocFailed ){ + if( pParse->nErr ){ for(j=0; jrc; } return SQLITE_OK; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 09e8174066..f1dc1d710f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4631,6 +4631,7 @@ char *sqlite3VMPrintf(sqlite3*,const char*, va_list); #endif void sqlite3SetString(char **, sqlite3*, const char*); +void sqlite3ProgressCheck(Parse*); void sqlite3ErrorMsg(Parse*, const char*, ...); int sqlite3ErrorToParser(sqlite3*,int); void sqlite3Dequote(char*); diff --git a/src/util.c b/src/util.c index 23c6b1a66d..a648c3a810 100644 --- a/src/util.c +++ b/src/util.c @@ -175,6 +175,23 @@ void sqlite3ErrorWithMsg(sqlite3 *db, int err_code, const char *zFormat, ...){ } } +/* +** Check for interrupts and invoke progress callback. +*/ +void sqlite3ProgressCheck(Parse *p){ + sqlite3 *db = p->db; + if( AtomicLoad(&db->u1.isInterrupted) ){ + p->nErr++; + p->rc = SQLITE_INTERRUPT; + } +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( db->xProgress && db->xProgress(db->pProgressArg) ){ + p->nErr++; + p->rc = SQLITE_INTERRUPT; + } +#endif +} + /* ** Add an error message to pParse->zErrMsg and increment pParse->nErr. ** diff --git a/src/vdbeaux.c b/src/vdbeaux.c index ca9e5112a4..b9ee3f82ae 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -574,6 +574,7 @@ static SQLITE_NOINLINE void resizeResolveLabel(Parse *p, Vdbe *v, int j){ int i; for(i=p->nLabelAlloc; iaLabel[i] = -1; #endif + if( nNewSize>=100 ) sqlite3ProgressCheck(p); p->nLabelAlloc = nNewSize; p->aLabel[j] = v->nOp; } diff --git a/src/where.c b/src/where.c index 890c3bf52c..80e199ccf8 100644 --- a/src/where.c +++ b/src/where.c @@ -2686,6 +2686,7 @@ static void whereLoopOutputAdjust( LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); + sqlite3ProgressCheck(pWC->pWInfo->pParse); for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){ assert( pTerm!=0 ); if( (pTerm->prereqAll & notAllowed)!=0 ) continue; @@ -2865,7 +2866,10 @@ static int whereLoopAddBtreeIndex( WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ pNew = pBuilder->pNew; - if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; + assert( db->mallocFailed==0 || pParse->nErr>0 ); + if( pParse->nErr ){ + return pParse->rc; + } WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d, rRun=%d\n", pProbe->pTable->zName,pProbe->zName, pNew->u.btree.nEq, pNew->nSkip, pNew->rRun)); @@ -3181,6 +3185,7 @@ static int whereLoopAddBtreeIndex( && (pNew->u.btree.nEqnKeyCol || pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) ){ + if( pNew->u.btree.nEq>3 ) sqlite3ProgressCheck(pParse); whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } pNew->nOut = saved_nOut; From 2fc9dc9330513fa7c5a5c722af8f9e21646f32e6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 12 Jan 2023 19:51:49 +0000 Subject: [PATCH 2/5] Fix sqlite3_prepare() so that it only invokes the progress handler on every N-th call to sqlite3ProgressCheck(), where N is the progress handler step count. Also fix faulty asserts exposed by the ability to interrupt in the middle of sqlite3_prepare(). FossilOrigin-Name: 05461651599bb490ac6cfd893645dabab9cccedc6adcce15aee2487b2ea6027a --- manifest | 25 +++++++++++-------------- manifest.uuid | 2 +- src/select.c | 2 +- src/sqliteInt.h | 3 +++ src/tclsqlite.c | 3 +++ src/util.c | 9 ++++++--- src/where.c | 2 +- test/capi3c.test | 1 + 8 files changed, 27 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 0227e85025..ff1fd93bd4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Attempt\sto\sprovide\sa\smechanism\sto\sdo\searly\stermination\sof\slong-running\nstatement\spreparation\sby\sinvoking\sthe\sprogress\shandler\sat\sstrategic\spoints\nduring\ssqlite3_parpare().\s\sThis\sexperiment\sshows\sthat\ssqlite3_prepare()\smight\nleave\sthe\sresulting\sprepared\sstatement\suninitialized\sfollowing\san\sinterrupt. -D 2023-01-12T13:25:48.952 +C Fix\ssqlite3_prepare()\sso\sthat\sit\sonly\sinvokes\sthe\sprogress\shandler\son\severy\nN-th\scall\sto\ssqlite3ProgressCheck(),\swhere\sN\sis\sthe\sprogress\shandler\sstep\ncount.\s\sAlso\sfix\sfaulty\sasserts\sexposed\sby\sthe\sability\sto\sinterrupt\sin\sthe\nmiddle\sof\ssqlite3_prepare(). +D 2023-01-12T19:51:49.383 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -645,16 +645,16 @@ F src/printf.c ff4b05e38bf928ff1b80d3dda4f977b10fe39ecbfe69c018224c7e5594fb2455 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c cb25679cf2a77526dccdcfb2ee87746a7dabacd583eb1c72b764a5d611694f6c +F src/select.c d389ccdb96855dbfaadc22d936889e1f0652ffca17e31a6b6522b45d99daa8ce F src/shell.c.in f7c75d1a9f900516e40f17f040668d5797592344bd88cff7ee7df586de6893c6 F src/sqlite.h.in 231d505c86ac4688b82259529273fe5fb2cce86cb80ea50628ff776113b5c706 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h d817f813a6f48f488fdde907222f0acd3170226d2dd4e528c0d8c7c141b4026f +F src/sqliteInt.h 43eeee1ea80543a0e40444bf53643ca259a2b1158ccfe859a6a6435b7358ecdd F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 -F src/tclsqlite.c 4e64ba300a5a26e0f1170e09032429faeb65e45e8f3d1a7833e8edb69fc2979e +F src/tclsqlite.c 8522a04fb9c84faa1d80354430ae0ee9349727a3a4b32e3cfe39b9be8324cabd F src/test1.c 39c13c25f5c4e75a07cb6bd34cfc1b8aa97ecc6f80f08953fa164db7e92ce71e F src/test2.c 827446e259a3b7ab949da1542953edda7b5117982576d3e6f1c24a0dd20a5cef F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644 @@ -715,7 +715,7 @@ F src/trigger.c 5e68b790f022b8dafbfb0eb244786512a95c9575fc198719d2557d73e5795858 F src/update.c f118e51768d2c1309e3c81e9f91141b22b8a1339cbc5969b1b2d810feaa25b22 F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 -F src/util.c c4941abc630852692507b7bfc936596d85491a1edf8e1fc9e09c3fb4f111cdf3 +F src/util.c 3ff7bc2b48dd425b1448304bb86273b05da1621f136d51dbb9789f8803559a1f F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd F src/vdbe.c 238635c1c40d42d9ded72994b81d4127f99d6d09e9279bdd37f6f34f4025adee F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c @@ -732,7 +732,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 8022eaf219860cbf60b62036b3695794d68cf577fec141df35846f0b3c88f9ab +F src/where.c b9abdf0481822af12d428afba4e40d18c4a2c458d4e3b0a1c735a9dfb11a4ec1 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 76bca3379219880d2527493b71a3be49e696f75396d3481e4de5d4ceec7886b2 F src/whereexpr.c 7c5671a04b00c876bec5e99fd4e6f688065feb4773160fbf76fd7900d2901777 @@ -860,7 +860,7 @@ F test/cachespill.test 895997f84a25b323b166aecb69baab2d6380ea98f9e0bcc688c4493c5 F test/capi2.test 4ee545824adc3eb33bf57ef89f77440b28188ec3da72e5425ff0fcdba32e8d5a F test/capi3.test 3910a73c38ac76d69778dd9eb481ab7cd6ed59117fc047b4f6056a5c72529de1 F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 -F test/capi3c.test 54e2dc0c8fd7c34ad1590d1be6864397da2438c95a9f5aee2f8fbc60c112e44b +F test/capi3c.test 31d3a6778f2d06f2d9222bd7660c41a516d1518a059b069e96ebbeadb5a490f7 F test/capi3d.test 8b778794af891b0dca3d900bd345fbc8ebd2aa2aae425a9dccdd10d5233dfbde F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/carray01.test d55d57bf66b1af1c7ac55fae66ff4910884a8f5d21a90a18797ce386212a2634 @@ -2068,11 +2068,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7526c46632578a2b602622b9debc406b52af4a42cc880970c4307d13853d59d3 -R 6e5c5a2e308bedb9ef7a7686c27c4f8f -T *branch * progress-during-prepare -T *sym-progress-during-prepare * -T -sym-trunk * +P 79636f2d80aee70832913a78933da2a7e30cc037810b93903ebbc1925ea93fef +R f60182bf56bc04fd2334c34bed4a50bf U drh -Z 15a0449b6bca5de3ec6c827c565351a6 +Z 0bf05536c4cc204538419442d0293744 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 641465e452..f9bbd13b5f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -79636f2d80aee70832913a78933da2a7e30cc037810b93903ebbc1925ea93fef \ No newline at end of file +05461651599bb490ac6cfd893645dabab9cccedc6adcce15aee2487b2ea6027a \ No newline at end of file diff --git a/src/select.c b/src/select.c index 20bab20331..d27bed026f 100644 --- a/src/select.c +++ b/src/select.c @@ -2316,7 +2316,7 @@ void sqlite3SubqueryColumnTypes( assert( pSelect!=0 ); assert( (pSelect->selFlags & SF_Resolved)!=0 ); - assert( pTab->nCol==pSelect->pEList->nExpr || db->mallocFailed ); + assert( pTab->nCol==pSelect->pEList->nExpr || pParse->nErr>0 ); assert( aff==SQLITE_AFF_NONE || aff==SQLITE_AFF_BLOB ); if( db->mallocFailed ) return; while( pSelect->pPrior ) pSelect = pSelect->pPrior; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f1dc1d710f..3da5f13d5b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3754,6 +3754,9 @@ struct Parse { u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u32 oldmask; /* Mask of old.* columns referenced */ u32 newmask; /* Mask of new.* columns referenced */ +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + u32 nProgressSteps; /* xProgress steps taken during sqlite3_prepare() */ +#endif u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ u8 bReturning; /* Coding a RETURNING trigger */ u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 4e25a65014..c455c29f14 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3054,6 +3054,9 @@ deserialize_error: if( pDb->zProgress ){ Tcl_AppendResult(interp, pDb->zProgress, (char*)0); } +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + sqlite3_progress_handler(pDb->db, 0, 0, 0); +#endif }else if( objc==4 ){ char *zProgress; int len; diff --git a/src/util.c b/src/util.c index a648c3a810..632d317e31 100644 --- a/src/util.c +++ b/src/util.c @@ -185,9 +185,12 @@ void sqlite3ProgressCheck(Parse *p){ p->rc = SQLITE_INTERRUPT; } #ifndef SQLITE_OMIT_PROGRESS_CALLBACK - if( db->xProgress && db->xProgress(db->pProgressArg) ){ - p->nErr++; - p->rc = SQLITE_INTERRUPT; + if( db->xProgress && (++p->nProgressSteps)>=db->nProgressOps ){ + if( db->xProgress(db->pProgressArg) ){ + p->nErr++; + p->rc = SQLITE_INTERRUPT; + } + p->nProgressSteps = 0; } #endif } diff --git a/src/where.c b/src/where.c index 80e199ccf8..5d0321e7aa 100644 --- a/src/where.c +++ b/src/where.c @@ -4346,7 +4346,7 @@ static int whereLoopAddOr( rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable); } assert( rc==SQLITE_OK || rc==SQLITE_DONE || sCur.n==0 - || rc==SQLITE_NOMEM ); + || rc==SQLITE_NOMEM || rc==SQLITE_INTERRUPT ); testcase( rc==SQLITE_NOMEM && sCur.n>0 ); testcase( rc==SQLITE_DONE ); if( sCur.n==0 ){ diff --git a/test/capi3c.test b/test/capi3c.test index 889ecbe859..247dbf35e5 100644 --- a/test/capi3c.test +++ b/test/capi3c.test @@ -1269,6 +1269,7 @@ ifcapable progress { sqlite3_finalize $STMT } {SQLITE_INTERRUPT} do_test capi3c-21.4 { + db progress set STMT [sqlite3_prepare $DB {SELECT * FROM t3} -1 TAIL] db progress 5 "expr 1" sqlite3_step $STMT From 94effc8a05089df9a99975049fc55d2f40011d7a Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 12 Jan 2023 20:36:43 +0000 Subject: [PATCH 3/5] Reduce the frequency of calls to sqlite3ProgressCheck(). FossilOrigin-Name: 795aca4a269ea4ede547844037c218ac714c40e8bee003e4d9b786befc9cf97a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeaux.c | 4 +++- src/where.c | 6 ++++-- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index ff1fd93bd4..c92b66c4fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssqlite3_prepare()\sso\sthat\sit\sonly\sinvokes\sthe\sprogress\shandler\son\severy\nN-th\scall\sto\ssqlite3ProgressCheck(),\swhere\sN\sis\sthe\sprogress\shandler\sstep\ncount.\s\sAlso\sfix\sfaulty\sasserts\sexposed\sby\sthe\sability\sto\sinterrupt\sin\sthe\nmiddle\sof\ssqlite3_prepare(). -D 2023-01-12T19:51:49.383 +C Reduce\sthe\sfrequency\sof\scalls\sto\ssqlite3ProgressCheck(). +D 2023-01-12T20:36:43.946 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -721,7 +721,7 @@ F src/vdbe.c 238635c1c40d42d9ded72994b81d4127f99d6d09e9279bdd37f6f34f4025adee F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c F src/vdbeInt.h fc15815b7bdafbb27e7f027faba2b0112e87d382c0d72241672528806ebc0db5 F src/vdbeapi.c 4ee67890913c1d2469c68e3ad2e7ddeab57ac5924a64bbfd0906a8ea0d542c7f -F src/vdbeaux.c 118ef7fbc1be7ef359274a731ed4120f9e3ac2f1b677ee62103d150952f4bcd2 +F src/vdbeaux.c 3f9e3b6585e7434aa11300169dd66ddf0fc963a0c6f7940bdc058335dadeb353 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 316d518115f3720b4097f0231e2a3d6eefd06c787eccf44972f8d8f462153421 F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -732,7 +732,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c b9abdf0481822af12d428afba4e40d18c4a2c458d4e3b0a1c735a9dfb11a4ec1 +F src/where.c 0e5dd009af01250ff3e1f3f632b7ef471cf77c6edc24ebe882433674325c20c8 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 76bca3379219880d2527493b71a3be49e696f75396d3481e4de5d4ceec7886b2 F src/whereexpr.c 7c5671a04b00c876bec5e99fd4e6f688065feb4773160fbf76fd7900d2901777 @@ -2068,8 +2068,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 79636f2d80aee70832913a78933da2a7e30cc037810b93903ebbc1925ea93fef -R f60182bf56bc04fd2334c34bed4a50bf +P 05461651599bb490ac6cfd893645dabab9cccedc6adcce15aee2487b2ea6027a +R fd47709650fb2ea6d296147bce5093e6 U drh -Z 0bf05536c4cc204538419442d0293744 +Z 71632e3006f164270162b7992c4f1e6a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f9bbd13b5f..b7ac31ef0b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -05461651599bb490ac6cfd893645dabab9cccedc6adcce15aee2487b2ea6027a \ No newline at end of file +795aca4a269ea4ede547844037c218ac714c40e8bee003e4d9b786befc9cf97a \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index b9ee3f82ae..d04d8f1e17 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -574,7 +574,9 @@ static SQLITE_NOINLINE void resizeResolveLabel(Parse *p, Vdbe *v, int j){ int i; for(i=p->nLabelAlloc; iaLabel[i] = -1; #endif - if( nNewSize>=100 ) sqlite3ProgressCheck(p); + if( nNewSize>=100 && (nNewSize/100)>(p->nLabelAlloc/100) ){ + sqlite3ProgressCheck(p); + } p->nLabelAlloc = nNewSize; p->aLabel[j] = v->nOp; } diff --git a/src/where.c b/src/where.c index 5d0321e7aa..1f9f723cd2 100644 --- a/src/where.c +++ b/src/where.c @@ -2686,7 +2686,6 @@ static void whereLoopOutputAdjust( LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); - sqlite3ProgressCheck(pWC->pWInfo->pParse); for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){ assert( pTerm!=0 ); if( (pTerm->prereqAll & notAllowed)!=0 ) continue; @@ -2699,6 +2698,7 @@ static void whereLoopOutputAdjust( if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; } if( j<0 ){ + sqlite3ProgressCheck(pWC->pWInfo->pParse); if( pLoop->maskSelf==pTerm->prereqAll ){ /* If there are extra terms in the WHERE clause not used by an index ** that depend only on the table being scanned, and that will tend to @@ -3185,7 +3185,9 @@ static int whereLoopAddBtreeIndex( && (pNew->u.btree.nEqnKeyCol || pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) ){ - if( pNew->u.btree.nEq>3 ) sqlite3ProgressCheck(pParse); + if( pNew->u.btree.nEq>3 ){ + sqlite3ProgressCheck(pParse); + } whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } pNew->nOut = saved_nOut; From 7ef26cbffa5e0a177d96243e8142994e4d432a78 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 13 Jan 2023 11:31:49 +0000 Subject: [PATCH 4/5] Remove an incorrect legacy assert(). FossilOrigin-Name: 3006b3b32fb1efedde3ade418c19c35b6e9a4af1e6a7cb5d5c72a080ec1f6f3c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 -- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c92b66c4fb..2643ef0fe5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\sfrequency\sof\scalls\sto\ssqlite3ProgressCheck(). -D 2023-01-12T20:36:43.946 +C Remove\san\sincorrect\slegacy\sassert(). +D 2023-01-13T11:31:49.436 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 0e5dd009af01250ff3e1f3f632b7ef471cf77c6edc24ebe882433674325c20c8 +F src/where.c 14ee8da18f9b4518af06931a32386dd0c7ec2eacab520726d6425e252e54b280 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 76bca3379219880d2527493b71a3be49e696f75396d3481e4de5d4ceec7886b2 F src/whereexpr.c 7c5671a04b00c876bec5e99fd4e6f688065feb4773160fbf76fd7900d2901777 @@ -2068,8 +2068,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 05461651599bb490ac6cfd893645dabab9cccedc6adcce15aee2487b2ea6027a -R fd47709650fb2ea6d296147bce5093e6 +P 795aca4a269ea4ede547844037c218ac714c40e8bee003e4d9b786befc9cf97a +R 4ca985ce44569e4afda8cfc28aa1c4f0 U drh -Z 71632e3006f164270162b7992c4f1e6a +Z 15a661decb42652b0c29209f4fd01d6d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b7ac31ef0b..70269045af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -795aca4a269ea4ede547844037c218ac714c40e8bee003e4d9b786befc9cf97a \ No newline at end of file +3006b3b32fb1efedde3ade418c19c35b6e9a4af1e6a7cb5d5c72a080ec1f6f3c \ No newline at end of file diff --git a/src/where.c b/src/where.c index 1f9f723cd2..205f247621 100644 --- a/src/where.c +++ b/src/where.c @@ -4347,8 +4347,6 @@ static int whereLoopAddOr( if( rc==SQLITE_OK ){ rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable); } - assert( rc==SQLITE_OK || rc==SQLITE_DONE || sCur.n==0 - || rc==SQLITE_NOMEM || rc==SQLITE_INTERRUPT ); testcase( rc==SQLITE_NOMEM && sCur.n>0 ); testcase( rc==SQLITE_DONE ); if( sCur.n==0 ){ From b928141edc7144655bbb37f8b2892bf0fecbc8cd Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 13 Jan 2023 15:31:24 +0000 Subject: [PATCH 5/5] Update documentation for sqlite3_progress_handler(). FossilOrigin-Name: 41b4c407b7f6968f4493fb9a0b220c55a6f6a353a8e6cb054efd3eb01bdd11dc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 9 ++++++++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2643ef0fe5..cd5ebeaef3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sincorrect\slegacy\sassert(). -D 2023-01-13T11:31:49.436 +C Update\sdocumentation\sfor\ssqlite3_progress_handler(). +D 2023-01-13T15:31:24.311 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -647,7 +647,7 @@ F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c d389ccdb96855dbfaadc22d936889e1f0652ffca17e31a6b6522b45d99daa8ce F src/shell.c.in f7c75d1a9f900516e40f17f040668d5797592344bd88cff7ee7df586de6893c6 -F src/sqlite.h.in 231d505c86ac4688b82259529273fe5fb2cce86cb80ea50628ff776113b5c706 +F src/sqlite.h.in 9a24b7e0f91412a1afe56f2be3c94857438f9e5e9bd91b4ec8ecc0ced9e9e6ed F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 F src/sqliteInt.h 43eeee1ea80543a0e40444bf53643ca259a2b1158ccfe859a6a6435b7358ecdd @@ -2068,8 +2068,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 795aca4a269ea4ede547844037c218ac714c40e8bee003e4d9b786befc9cf97a -R 4ca985ce44569e4afda8cfc28aa1c4f0 +P 3006b3b32fb1efedde3ade418c19c35b6e9a4af1e6a7cb5d5c72a080ec1f6f3c +R 33ea1cbe269f174f61a6120d13d981a4 U drh -Z 15a661decb42652b0c29209f4fd01d6d +Z 24a4936197f59974fd50260986b7ffd3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 70269045af..4d53a24a4c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3006b3b32fb1efedde3ade418c19c35b6e9a4af1e6a7cb5d5c72a080ec1f6f3c \ No newline at end of file +41b4c407b7f6968f4493fb9a0b220c55a6f6a353a8e6cb054efd3eb01bdd11dc \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 57bbdb7a1d..c6004f92db 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3366,7 +3366,7 @@ int sqlite3_trace_v2( ** ** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback ** function X to be invoked periodically during long running calls to -** [sqlite3_exec()], [sqlite3_step()] and [sqlite3_get_table()] for +** [sqlite3_step()] and [sqlite3_prepare()] and similar for ** database connection D. An example use for this ** interface is to keep a GUI updated during a large query. ** @@ -3391,6 +3391,13 @@ int sqlite3_trace_v2( ** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their ** database connections for the meaning of "modify" in this paragraph. ** +** The progress handler callback would originally only be invoked from the +** bytecode engine. It still might be invoked during [sqlite3_prepare()] +** and similar because those routines might force a reparse of the schema +** which involves running the bytecode engine. However, beginning with +** SQLite version 3.41.0, the progress handler callback might also be +** invoked directly from [sqlite3_prepare()] while analyzing and generating +** code for complex queries. */ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);