From 30cbcc0004fa7c272f35214d8ab022e30d2e7075 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 3 Dec 2022 16:09:32 +0000 Subject: [PATCH 01/11] Cause CLI .sha3sum to warn of text fields that do not survive CAST(CAST(t as BLOB) AS TEXT) due to invalid UTF encoding. FossilOrigin-Name: 123f2a0785790addf9c60a0fd09077dda9cb84d33a2594901a2b22bb555be491 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/shell.c.in | 51 ++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index ee12cfe5ad..34ffa10ecc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\swasm.xWrap.resultAdapter()\sX:free\sentries\sto\sX:dealloc\sfor\sconsistency\swith\swasm.dealloc().\sAdd\san\sundocumented\sfeature\sto\sreplace\swasm.alloc/dealloc/realloc()\swith\sthe\sC-standard\sallocators\s(after\san\sallocator\smisuse\sled\sdown\sa\sseveral-hour\srabbit\shole\strying\sto\sdiscover\sa\smis-free()\sviolation).\sRelated\stest\supdates. -D 2022-12-03T13:10:58.483 +C Cause\sCLI\s.sha3sum\sto\swarn\sof\stext\sfields\sthat\sdo\snot\ssurvive\sCAST(CAST(t\sas\sBLOB)\sAS\sTEXT)\sdue\sto\sinvalid\sUTF\sencoding. +D 2022-12-03T16:09:32.554 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 6983de0e6b8b68c97f82f9fca27ffb8f17161cff4d0d48fdf9eb40b213da0cc6 -F src/shell.c.in 9fda74d40b206a707aaa69fc5dc38e2c6a9137a3f4a1dcd7af581d59d92c063c +F src/shell.c.in aa3e4c78ae25c24562b0c3497a88875baa61425714437143170b2c462842035d F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2065,8 +2065,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ed1ed21221b048ac5a5275cdfc4d9b2a406acdc7d4b648c3b61bcc822d88d955 -R 04802c5e85209b72659bc288df2e94a1 -U stephan -Z 23bb6930a31350fa896c81241f2934b6 +P d9807656f8a7c2a893d3f68ee5592f44826b8e999ae66f7d9000674b5c1b0207 +R 9f83c1fb14dddfc272778480afabb0a3 +T *branch * sha3sum_text_validation +T *sym-sha3sum_text_validation * +T -sym-trunk * +U larrybr +Z 1ba97ab620561a8dccaf913734ec9094 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4e29a56c7f..b35a727cc3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9807656f8a7c2a893d3f68ee5592f44826b8e999ae66f7d9000674b5c1b0207 \ No newline at end of file +123f2a0785790addf9c60a0fd09077dda9cb84d33a2594901a2b22bb555be491 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index a829fa2387..1f0ac24c81 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10077,6 +10077,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_stmt *pStmt; /* For querying tables names */ char *zSql; /* SQL to be run */ char *zSep; /* Separator */ + char *zRevText; /* Query for reversible to-blob-to-text check */ ShellText sSql; /* Complete SQL for the query to run the hash */ ShellText sQuery; /* Set of queries used to read all content */ open_db(p, 0); @@ -10114,17 +10115,35 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( bSchema ){ - zSql = "SELECT lower(name) FROM sqlite_schema" + zSql = "SELECT lower(name) as tname FROM sqlite_schema" " WHERE type='table' AND coalesce(rootpage,0)>1" " UNION ALL SELECT 'sqlite_schema'" " ORDER BY 1 collate nocase"; }else{ - zSql = "SELECT lower(name) FROM sqlite_schema" + zSql = "SELECT lower(name) as tname FROM sqlite_schema" " WHERE type='table' AND coalesce(rootpage,0)>1" " AND name NOT LIKE 'sqlite_%'" " ORDER BY 1 collate nocase"; } sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + zRevText = sqlite3_mprintf( + /* lower-case query is first run, producing upper-case query. */ + "with text_cols as materialized(" + "select tname, cname " + "from (" + " select ss.tname as tname, ti.name as cname, lower(type) as lca" + " from (%s) ss join pragma_table_info(tname) ti" + " where (instr(lca,'int')=0 and " + " instr(lca,'char')+instr(lca,'clob')+instr(lca,'text')>0)" + "))" + "select 'SELECT total(bad_text_count) AS bad_text_count" + " FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query" + " from (select 'SELECT COUNT(*) AS bad_text_count" + " FROM '||tname||' WHERE '" + " ||group_concat('CAST(CAST('||cname||' AS BLOB) AS TEXT)<>'||cname," + " ' OR ') as query, tname from text_cols group by tname)", + zSql); + shell_check_oom(zRevText); initText(&sQuery); initText(&sSql); appendText(&sSql, "WITH [sha3sum$query](a,b) AS(",0); @@ -10180,7 +10199,35 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ shell_exec(p, zSql, 0); } + { + int lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); + if( bDebug ) utf8_printf(p->out, "%s\n", zRevText); + assert(lrc==SQLITE_OK); + lrc = SQLITE_ROW==sqlite3_step(pStmt); + assert(lrc!=0); + if( lrc ){ + const char *zGenQuery = sqlite3_column_text(pStmt,0); + sqlite3_stmt *pCheckStmt; + double countIrreversible; + lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0); + if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery); + assert(lrc==SQLITE_OK); + lrc = SQLITE_ROW==sqlite3_step(pCheckStmt); + assert(lrc!=0); + countIrreversible = sqlite3_column_double(pCheckStmt, 0); + sqlite3_finalize(pStmt); + sqlite3_finalize(pCheckStmt); + if( countIrreversible>0 ){ + int n = (int)(countIrreversible + 0.5); + utf8_printf(stderr, + "Digest includes %d invalidly encoded text field%s.\n", + n, (n>1)? "s": ""); + } + } + + } sqlite3_free(zSql); + sqlite3_free(zRevText); }else #if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) From 29226fc023767ddbb078ae37b64e1a4d8accf111 Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 6 Dec 2022 17:59:05 +0000 Subject: [PATCH 02/11] Clear a few more -Wall warnings and simplify dynaprompt feature keep/omit macros. FossilOrigin-Name: 540e895d877fab1ea138786e56923a202018f68c78199a89adfc296c75735b30 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 33 +++++++++++++++++---------------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 38783ac47e..e0cc60735d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings\sin\sthe\snew\sdynamic\scontinuation\sprompt\slogic\sof\nthe\sCLI. -D 2022-12-06T15:11:13.637 +C Clear\sa\sfew\smore\s-Wall\swarnings\sand\ssimplify\sdynaprompt\sfeature\skeep/omit\smacros. +D 2022-12-06T17:59:05.853 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 56162cde2f71314826f831618eae853205628080afa25cd90faef7a9b761dee4 -F src/shell.c.in f37273af2d5cd2299091b3c1e260254ebd7cd16d7602426983b36009290e8577 +F src/shell.c.in 991eff7f819f96ae76671e7aa44597ac702274fd441d3b0d9297cbd3b9ce1284 F src/sqlite.h.in 1fe1836879ecbb2e28f00f44eb6092db09a2a06bf072af351c6c2466bd515496 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c0cfe0582add87981826d124a0763482f51fae4b105b5a970dd56919f1d04d60 -R 19f4ce887fdd7ce982932018e71e1a9d -U drh -Z 087fdbe4ee41c07ef7676876e727f795 +P 0d80500d358fa1c9b5867c2c8250d278ba813bf2ad81bb0bc3f820a71489b374 +R d2096b6607984433a4c4474baae68bae +U larrybr +Z 397638802057e8cab20b3ed1dff696d1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 16c4d729e4..873db8bc54 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d80500d358fa1c9b5867c2c8250d278ba813bf2ad81bb0bc3f820a71489b374 \ No newline at end of file +540e895d877fab1ea138786e56923a202018f68c78199a89adfc296c75735b30 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 85e708551d..a5c4c48c15 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -482,24 +482,26 @@ static char continuePrompt[PROMPT_LEN_MAX]; */ #ifdef SQLITE_OMIT_DYNAPROMPT # define CONTINUATION_PROMPT continuePrompt -# define CONTINUE_PROMPT_RESET(p) +# define CONTINUE_PROMPT_RESET # define CONTINUE_PROMPT_AWAITS(p,s) # define CONTINUE_PROMPT_AWAITC(p,c) # define CONTINUE_PAREN_INCR(p,n) -# define CONTINUE_PROMPT_STATE 0 -# define SCAN_TRACKER_REFTYPE void* +# define CONTINUE_PROMPT_PSTATE 0 +typedef void *t_NoDynaPrompt; +# define SCAN_TRACKER_REFTYPE t_NoDynaPrompt #else # define CONTINUATION_PROMPT dynamicContinuePrompt() -# define CONTINUE_PROMPT_RESET(p) \ - if(p) (setLexemeOpen(p,0,0), trackParenLevel(p,0)) +# define CONTINUE_PROMPT_RESET \ + do {setLexemeOpen(&dynPrompt,0,0); trackParenLevel(&dynPrompt,0);} while(0) # define CONTINUE_PROMPT_AWAITS(p,s) \ if(p && stdin_is_interactive) setLexemeOpen(p, s, 0) # define CONTINUE_PROMPT_AWAITC(p,c) \ if(p && stdin_is_interactive) setLexemeOpen(p, 0, c) # define CONTINUE_PAREN_INCR(p,n) \ if(p && stdin_is_interactive) (trackParenLevel(p,n)) -# define SCAN_TRACKER_REFTYPE struct DynaPrompt * -# define CONTINUE_PROMPT_STATE (pDynPrompt) +# define CONTINUE_PROMPT_PSTATE (&dynPrompt) +typedef struct DynaPrompt *t_DynaPromptRef; +# define SCAN_TRACKER_REFTYPE t_DynaPromptRef static struct DynaPrompt { char dynamicPrompt[PROMPT_LEN_MAX]; @@ -507,7 +509,6 @@ static struct DynaPrompt { int inParenLevel; char *zScannerAwaits; } dynPrompt = { {0}, {0}, 0, 0 }; -static struct DynaPrompt *pDynPrompt = &dynPrompt; /* Record parenthesis nesting level change, or force level to 0. */ static void trackParenLevel(struct DynaPrompt *p, int ni){ @@ -542,11 +543,11 @@ static char *dynamicContinuePrompt(void){ PROMPT_LEN_MAX-4); }else{ if( dynPrompt.inParenLevel>9 ){ - strncpy(dynPrompt.dynamicPrompt, "(..", 3); + strncpy(dynPrompt.dynamicPrompt, "(..", 4); }else if( dynPrompt.inParenLevel<0 ){ - strncpy(dynPrompt.dynamicPrompt, ")x!", 3); + strncpy(dynPrompt.dynamicPrompt, ")x!", 4); }else{ - strncpy(dynPrompt.dynamicPrompt, "(x.", 3); + strncpy(dynPrompt.dynamicPrompt, "(x.", 4); dynPrompt.dynamicPrompt[2] = (char)('0'+dynPrompt.inParenLevel); } strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, PROMPT_LEN_MAX-4); @@ -11288,7 +11289,7 @@ static int process_input(ShellState *p){ } ++p->inputNesting; p->lineno = 0; - CONTINUE_PROMPT_RESET(CONTINUE_PROMPT_STATE); + CONTINUE_PROMPT_RESET; while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){ fflush(p->out); zLine = one_input_line(p->in, zLine, nSql>0); @@ -11307,7 +11308,7 @@ static int process_input(ShellState *p){ && line_is_complete(zSql, nSql) ){ memcpy(zLine,";",2); } - qss = quickscan(zLine, qss, CONTINUE_PROMPT_STATE); + qss = quickscan(zLine, qss, CONTINUE_PROMPT_PSTATE); if( QSS_PLAINWHITE(qss) && nSql==0 ){ /* Just swallow single-line whitespace */ echo_group_input(p, zLine); @@ -11315,7 +11316,7 @@ static int process_input(ShellState *p){ continue; } if( zLine && (zLine[0]=='.' || zLine[0]=='#') && nSql==0 ){ - CONTINUE_PROMPT_RESET(CONTINUE_PROMPT_STATE); + CONTINUE_PROMPT_RESET; echo_group_input(p, zLine); if( zLine[0]=='.' ){ rc = do_meta_command(zLine, p); @@ -11351,7 +11352,7 @@ static int process_input(ShellState *p){ if( nSql && QSS_SEMITERM(qss) && sqlite3_complete(zSql) ){ echo_group_input(p, zSql); errCnt += runOneSqlLine(p, zSql, p->in, startline); - CONTINUE_PROMPT_RESET(CONTINUE_PROMPT_STATE); + CONTINUE_PROMPT_RESET; nSql = 0; if( p->outCount ){ output_reset(p); @@ -11371,7 +11372,7 @@ static int process_input(ShellState *p){ /* This may be incomplete. Let the SQL parser deal with that. */ echo_group_input(p, zSql); errCnt += runOneSqlLine(p, zSql, p->in, startline); - CONTINUE_PROMPT_RESET(CONTINUE_PROMPT_STATE); + CONTINUE_PROMPT_RESET; } free(zSql); free(zLine); From 2adb309ead2d8a794a65b374c76d7c7246c95a69 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 6 Dec 2022 18:48:06 +0000 Subject: [PATCH 03/11] Have sqlite3_stmt_scanstatus_v2() return an NCYCLE value for all loops, not just virtual tables ones. The value returned is the sum of the NCYCLE counts for the various opcodes that move or read data from the table or index cursor associated with the loop. FossilOrigin-Name: 9499b2f51e8174c6b8a67840c92ba23b7dd1dc8dc2b91fca0c5dc07b71662149 --- manifest | 22 ++++++------- manifest.uuid | 2 +- src/vdbe.c | 76 +++++++++++++++++++++---------------------- src/vdbeapi.c | 6 ++-- src/wherecode.c | 16 +++++++-- test/scanstatus2.test | 31 ++++++++++++++---- tool/mkopcodeh.tcl | 5 +++ 7 files changed, 95 insertions(+), 63 deletions(-) diff --git a/manifest b/manifest index e0cc60735d..c4286f7f7d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clear\sa\sfew\smore\s-Wall\swarnings\sand\ssimplify\sdynaprompt\sfeature\skeep/omit\smacros. -D 2022-12-06T17:59:05.853 +C Have\ssqlite3_stmt_scanstatus_v2()\sreturn\san\sNCYCLE\svalue\sfor\sall\sloops,\snot\sjust\svirtual\stables\sones.\sThe\svalue\sreturned\sis\sthe\ssum\sof\sthe\sNCYCLE\scounts\sfor\sthe\svarious\sopcodes\sthat\smove\sor\sread\sdata\sfrom\sthe\stable\sor\sindex\scursor\sassociated\swith\sthe\sloop. +D 2022-12-06T18:48:06.154 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -717,10 +717,10 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 313f3154e2b85a447326f5dd15de8d31a4df6ab0c3579bd58f426ff634ec9050 F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c e744259050ec9bbcd47acf9a03a66fe3305d7429c823995de11ed524ca06d16f +F src/vdbe.c 8fdef91d3c9bfa363e8eb381d362e2f0cadec456975ec5595c0bfdf65da84e53 F src/vdbe.h 6d921884cf8ec6a53efba99f8b68e32e955367631743e29039840e781aaf547c F src/vdbeInt.h e83be1eea422758d42302941e96e59d93193c373da655a87befdc03a851e0f95 -F src/vdbeapi.c 959cef4c1e5cb6589e67b3295fe9d3e45cbd109ae6618d57b74c900dbd6be0cd +F src/vdbeapi.c 5e265ab5165b0dfeac55e6a2ad3929d64584c4e7ca106b438137d7596da91348 F src/vdbeaux.c 0b81f317c86ed9f4ba822af66a52777fed6e8180edc79d4fc62ffe75c8e52d80 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 6cfed43758d57b6e3b99d9cdedfeccd86e45a07e427b22d8487cbdbebb6c522a @@ -734,7 +734,7 @@ F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c 20f4f51d2d5fb19b984e6ea381b26cf627cc93e64dd9b2ce6a94531aec2f5916 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c -F src/wherecode.c d3146e215f3a716c0e153ded68bfcc747aad550e114a79729bda887cf4ea6f00 +F src/wherecode.c 76bca3379219880d2527493b71a3be49e696f75396d3481e4de5d4ceec7886b2 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae F src/window.c 14836767adb26573b50f528eb37f8b1336f2c430ab38de7cead1e5c546bb4d8c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1458,7 +1458,7 @@ F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test 7dbcfd6adc6a8df6abc59f83d6da5a27e1bce0b2f6fa55147c8176d7c44e0450 -F test/scanstatus2.test cc0be2f645c32ede50affa7d4ecfaffd2b4572a4fd7f723082397c65a910a2b5 +F test/scanstatus2.test 3f45ec1d8b45dcb0df7f98daf3364fc2e03bf89a8c3dd2428251a5e0a34de7af F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce @@ -1996,7 +1996,7 @@ F tool/mkctimec.tcl c185cf1bdcd3d9bd3c06f77a2fd2df8a4a0d07266f992ecda75286965ba3 F tool/mkkeywordhash.c 35bfc41adacc4aa6ef6fca7fd0c63e0ec0534b78daf4d0cfdebe398216bbffc3 F tool/mkmsvcmin.tcl 6ecab9fe22c2c8de4d82d4c46797bda3d2deac8e763885f5a38d0c44a895ab33 F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef -F tool/mkopcodeh.tcl bcb2bd5affb545fd219ef0304c7978e2a356407ab723f45ec8569235892c1c3f +F tool/mkopcodeh.tcl 769d9e6a8b462323150dc13a8539d6064664b72974f7894befe2491cc73e05cd F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa F tool/mkpragmatab.tcl bd07bd59d45d0f3448e123d6937e9811195f9908a51e09d774609883055bfd3d F tool/mkshellc.tcl 02d0de8349ef830c0fb20d29680320bde2466e2ec422e5bd94c4317a7a7e8cc9 @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0d80500d358fa1c9b5867c2c8250d278ba813bf2ad81bb0bc3f820a71489b374 -R d2096b6607984433a4c4474baae68bae -U larrybr -Z 397638802057e8cab20b3ed1dff696d1 +P 540e895d877fab1ea138786e56923a202018f68c78199a89adfc296c75735b30 +R e347a9226ee206cfe1e680e2e5e722d0 +U dan +Z eaf9f5c14b0f034256d24205aa5d85c9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 873db8bc54..79c29e3304 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -540e895d877fab1ea138786e56923a202018f68c78199a89adfc296c75735b30 \ No newline at end of file +9499b2f51e8174c6b8a67840c92ba23b7dd1dc8dc2b91fca0c5dc07b71662149 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 1bf0ceb297..cd27aff44f 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2780,7 +2780,7 @@ case OP_Offset: { /* out3 */ ** typeof() function or the IS NULL or IS NOT NULL operators or the ** equivalent. In this case, all content loading can be omitted. */ -case OP_Column: { +case OP_Column: { /* ncycle */ u32 p2; /* column number to retrieve */ VdbeCursor *pC; /* The VDBE cursor */ BtCursor *pCrsr; /* The B-Tree cursor corresponding to pC */ @@ -4132,7 +4132,7 @@ case OP_SetCookie: { ** ** See also: OP_OpenRead, OP_ReopenIdx */ -case OP_ReopenIdx: { +case OP_ReopenIdx: { /* ncycle */ int nField; KeyInfo *pKeyInfo; u32 p2; @@ -4153,7 +4153,7 @@ case OP_ReopenIdx: { } /* If the cursor is not currently open or is open on a different ** index, then fall through into OP_OpenRead to force a reopen */ -case OP_OpenRead: +case OP_OpenRead: /* ncycle */ case OP_OpenWrite: assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 || pOp->p5==OPFLAG_SEEKEQ ); @@ -4247,7 +4247,7 @@ open_cursor_set_hints: ** ** Duplicate ephemeral cursors are used for self-joins of materialized views. */ -case OP_OpenDup: { +case OP_OpenDup: { /* ncycle */ VdbeCursor *pOrig; /* The original cursor to be duplicated */ VdbeCursor *pCx; /* The new cursor */ @@ -4309,8 +4309,8 @@ case OP_OpenDup: { ** by this opcode will be used for automatically created transient ** indices in joins. */ -case OP_OpenAutoindex: -case OP_OpenEphemeral: { +case OP_OpenAutoindex: /* ncycle */ +case OP_OpenEphemeral: { /* ncycle */ VdbeCursor *pCx; KeyInfo *pKeyInfo; @@ -4468,7 +4468,7 @@ case OP_OpenPseudo: { ** Close a cursor previously opened as P1. If P1 is not ** currently open, this instruction is a no-op. */ -case OP_Close: { +case OP_Close: { /* ncycle */ assert( pOp->p1>=0 && pOp->p1nCursor ); sqlite3VdbeFreeCursor(p, p->apCsr[pOp->p1]); p->apCsr[pOp->p1] = 0; @@ -4585,10 +4585,10 @@ case OP_ColumnsUsed: { ** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ -case OP_SeekLT: /* jump, in3, group */ -case OP_SeekLE: /* jump, in3, group */ -case OP_SeekGE: /* jump, in3, group */ -case OP_SeekGT: { /* jump, in3, group */ +case OP_SeekLT: /* jump, in3, group, ncycle */ +case OP_SeekLE: /* jump, in3, group, ncycle */ +case OP_SeekGE: /* jump, in3, group, ncycle */ +case OP_SeekGT: { /* jump, in3, group, ncycle */ int res; /* Comparison result */ int oc; /* Opcode */ VdbeCursor *pC; /* The cursor to seek */ @@ -4854,7 +4854,7 @@ seek_not_found: ** jump to SeekOP.P2 if This.P5==0 or to This.P2 if This.P5>0. ** */ -case OP_SeekScan: { +case OP_SeekScan: { /* ncycle */ VdbeCursor *pC; int res; int nStep; @@ -4976,7 +4976,7 @@ case OP_SeekScan: { ** ** P1 must be a valid b-tree cursor. */ -case OP_SeekHit: { +case OP_SeekHit: { /* ncycle */ VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -5108,7 +5108,7 @@ case OP_IfNotOpen: { /* jump */ ** ** See also: NotFound, Found, NotExists */ -case OP_IfNoHope: { /* jump, in3 */ +case OP_IfNoHope: { /* jump, in3, ncycle */ VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -5122,9 +5122,9 @@ case OP_IfNoHope: { /* jump, in3 */ /* Fall through into OP_NotFound */ /* no break */ deliberate_fall_through } -case OP_NoConflict: /* jump, in3 */ -case OP_NotFound: /* jump, in3 */ -case OP_Found: { /* jump, in3 */ +case OP_NoConflict: /* jump, in3, ncycle */ +case OP_NotFound: /* jump, in3, ncycle */ +case OP_Found: { /* jump, in3, ncycle */ int alreadyExists; int ii; VdbeCursor *pC; @@ -5254,7 +5254,7 @@ case OP_Found: { /* jump, in3 */ ** ** See also: Found, NotFound, NoConflict, SeekRowid */ -case OP_SeekRowid: { /* jump, in3 */ +case OP_SeekRowid: { /* jump, in3, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -5279,7 +5279,7 @@ case OP_SeekRowid: { /* jump, in3 */ } /* Fall through into OP_NotExists */ /* no break */ deliberate_fall_through -case OP_NotExists: /* jump, in3 */ +case OP_NotExists: /* jump, in3, ncycle */ pIn3 = &aMem[pOp->p3]; assert( (pIn3->flags & MEM_Int)!=0 || pOp->opcode==OP_SeekRowid ); assert( pOp->p1>=0 && pOp->p1nCursor ); @@ -5903,7 +5903,7 @@ case OP_RowData: { ** be a separate OP_VRowid opcode for use with virtual tables, but this ** one opcode now works for both table types. */ -case OP_Rowid: { /* out2 */ +case OP_Rowid: { /* out2, ncycle */ VdbeCursor *pC; i64 v; sqlite3_vtab *pVtab; @@ -6002,8 +6002,8 @@ case OP_NullRow: { ** from the end toward the beginning. In other words, the cursor is ** configured to use Prev, not Next. */ -case OP_SeekEnd: -case OP_Last: { /* jump */ +case OP_SeekEnd: /* ncycle */ +case OP_Last: { /* jump, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -6108,7 +6108,7 @@ case OP_Sort: { /* jump */ ** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. */ -case OP_Rewind: { /* jump */ +case OP_Rewind: { /* jump, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -6202,7 +6202,7 @@ case OP_SorterNext: { /* jump */ rc = sqlite3VdbeSorterNext(db, pC); goto next_tail; -case OP_Prev: /* jump */ +case OP_Prev: /* jump, ncycle */ assert( pOp->p1>=0 && pOp->p1nCursor ); assert( pOp->p5==0 || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP @@ -6217,7 +6217,7 @@ case OP_Prev: /* jump */ rc = sqlite3BtreePrevious(pC->uc.pCursor, pOp->p3); goto next_tail; -case OP_Next: /* jump */ +case OP_Next: /* jump, ncycle */ assert( pOp->p1>=0 && pOp->p1nCursor ); assert( pOp->p5==0 || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP @@ -6409,8 +6409,8 @@ case OP_IdxDelete: { ** ** See also: Rowid, MakeRecord. */ -case OP_DeferredSeek: -case OP_IdxRowid: { /* out2 */ +case OP_DeferredSeek: /* ncycle */ +case OP_IdxRowid: { /* out2, ncycle */ VdbeCursor *pC; /* The P1 index cursor */ VdbeCursor *pTabCur; /* The P2 table cursor (OP_DeferredSeek only) */ i64 rowid; /* Rowid that P1 current points to */ @@ -6472,8 +6472,8 @@ case OP_IdxRowid: { /* out2 */ ** seek operation now, without further delay. If the cursor seek has ** already occurred, this instruction is a no-op. */ -case OP_FinishSeek: { - VdbeCursor *pC; /* The P1 index cursor */ +case OP_FinishSeek: { /* ncycle */ + VdbeCursor *pC; /* The P1 index cursor */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -6528,10 +6528,10 @@ case OP_FinishSeek: { ** If the P1 index entry is less than or equal to the key value then jump ** to P2. Otherwise fall through to the next instruction. */ -case OP_IdxLE: /* jump */ -case OP_IdxGT: /* jump */ -case OP_IdxLT: /* jump */ -case OP_IdxGE: { /* jump */ +case OP_IdxLE: /* jump, ncycle */ +case OP_IdxGT: /* jump, ncycle */ +case OP_IdxLT: /* jump, ncycle */ +case OP_IdxGE: { /* jump, ncycle */ VdbeCursor *pC; int res; UnpackedRecord r; @@ -7952,7 +7952,7 @@ case OP_VDestroy: { ** P1 is a cursor number. This opcode opens a cursor to the virtual ** table and stores that cursor in P1. */ -case OP_VOpen: { +case OP_VOpen: { /* ncycle */ VdbeCursor *pCur; sqlite3_vtab_cursor *pVCur; sqlite3_vtab *pVtab; @@ -7999,7 +7999,7 @@ case OP_VOpen: { ** cursor. Register P3 is used to hold the values returned by ** sqlite3_vtab_in_first() and sqlite3_vtab_in_next(). */ -case OP_VInitIn: { /* out2 */ +case OP_VInitIn: { /* out2, ncycle */ VdbeCursor *pC; /* The cursor containing the RHS values */ ValueList *pRhs; /* New ValueList object to put in reg[P2] */ @@ -8036,7 +8036,7 @@ case OP_VInitIn: { /* out2 */ ** ** A jump is made to P2 if the result set after filtering would be empty. */ -case OP_VFilter: { /* jump */ +case OP_VFilter: { /* jump, ncycle */ int nArg; int iQuery; const sqlite3_module *pModule; @@ -8096,7 +8096,7 @@ case OP_VFilter: { /* jump */ ** bits (OPFLAG_LENGTHARG or OPFLAG_TYPEOFARG) but those bits are ** unused by OP_VColumn. */ -case OP_VColumn: { +case OP_VColumn: { /* ncycle */ sqlite3_vtab *pVtab; const sqlite3_module *pModule; Mem *pDest; @@ -8148,7 +8148,7 @@ case OP_VColumn: { ** jump to instruction P2. Or, if the virtual table has reached ** the end of its result set, then fall through to the next instruction. */ -case OP_VNext: { /* jump */ +case OP_VNext: { /* jump, ncycle */ sqlite3_vtab *pVtab; const sqlite3_module *pModule; int res; diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 0644cffa57..028eb7fe64 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -15,6 +15,7 @@ */ #include "sqliteInt.h" #include "vdbeInt.h" +#include "opcodes.h" #ifndef SQLITE_OMIT_DEPRECATED /* @@ -2225,10 +2226,7 @@ int sqlite3_stmt_scanstatus_v2( for(iOp=0; iOpnOp; iOp++){ Op *pOp = &p->aOp[iOp]; if( pOp->p1!=iEnd ) continue; - if( pOp->opcode!=OP_VFilter && pOp->opcode!=OP_VColumn - && pOp->opcode!=OP_Rowid && pOp->opcode!=OP_VOpen - && pOp->opcode!=OP_VNext - ){ + if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_NCYCLE)==0 ){ continue; } res += p->anCycle[iOp]; diff --git a/src/wherecode.c b/src/wherecode.c index 54c79baf97..4c22e5dd61 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -294,16 +294,26 @@ void sqlite3WhereAddScanStatus( ){ const char *zObj = 0; WhereLoop *pLoop = pLvl->pWLoop; - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ + int wsFlags = pLoop->wsFlags; + int viaCoroutine = 0; + + if( (wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ zObj = pLoop->u.btree.pIndex->zName; }else{ zObj = pSrclist->a[pLvl->iFrom].zName; + viaCoroutine = pSrclist->a[pLvl->iFrom].fg.viaCoroutine; } sqlite3VdbeScanStatus( v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj ); - if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ - sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur); + + if( viaCoroutine==0 ){ + if( (wsFlags & (WHERE_MULTI_OR|WHERE_AUTO_INDEX))==0 ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur); + } + if( wsFlags & WHERE_INDEXED ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); + } } } #endif diff --git a/test/scanstatus2.test b/test/scanstatus2.test index 3c5b4b16a3..9a4758b6d4 100644 --- a/test/scanstatus2.test +++ b/test/scanstatus2.test @@ -88,8 +88,8 @@ proc get_graph {stmt} { proc do_graph_test {tn sql res} { db eval $sql set stmt [db version -last-stmt-ptr] - set graph [string trim [get_graph $stmt]] + set graph [regsub -all {nCycle=[0-9]+} $graph nCycle=nnn] uplevel [list do_test $tn [list set {} $graph] [string trim $res]] } @@ -119,9 +119,9 @@ do_graph_test 1.3 { SELECT (SELECT a FROM t1 WHERE b=x) FROM t2 WHERE y=2 } { QUERY (nCycle=nnn) ---SCAN t2 +--SCAN t2 (nCycle=nnn) --CORRELATED SCALAR SUBQUERY 1 (nCycle=nnn) -----SCAN t1 +----SCAN t1 (nCycle=nnn) } do_graph_test 1.4 { @@ -132,9 +132,9 @@ do_graph_test 1.4 { } { QUERY (nCycle=nnn) --MATERIALIZE v2 (nCycle=nnn) -----SCAN t2 ---SCAN v2 ---SCAN t1 +----SCAN t2 (nCycle=nnn) +--SCAN v2 (nCycle=nnn) +--SCAN t1 (nCycle=nnn) --USE TEMP B-TREE FOR ORDER BY (nCycle=nnn) } @@ -154,6 +154,25 @@ QUERY (nCycle=nnn) --SCAN ft VIRTUAL TABLE INDEX 0:M1 (nCycle=nnn) } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE x1(a, b); + CREATE TABLE x2(c, d); + + WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000) + INSERT INTO x1 SELECT i, i FROM s; + INSERT INTO x2 SELECT a, b FROM x1; +} + +do_graph_test 2.1 { + SELECT * FROM x1, x2 WHERE c=+a; +} { +QUERY (nCycle=nnn) +--SCAN x1 (nCycle=nnn) +--SEARCH x2 USING AUTOMATIC COVERING INDEX (c=?) (nCycle=nnn) +} + finish_test diff --git a/tool/mkopcodeh.tcl b/tool/mkopcodeh.tcl index 8b4e345c67..6fb3b75940 100644 --- a/tool/mkopcodeh.tcl +++ b/tool/mkopcodeh.tcl @@ -86,6 +86,7 @@ while {![eof $in]} { set in3($name) 0 set out2($name) 0 set out3($name) 0 + set ncycle($name) 0 for {set i 3} {$i<[llength $line]-1} {incr i} { switch [string trim [lindex $line $i] ,] { same { @@ -107,6 +108,7 @@ while {![eof $in]} { in3 {set in3($name) 1} out2 {set out2($name) 1} out3 {set out3($name) 1} + ncycle {set ncycle($name) 1} } } if {$group($name)} { @@ -140,6 +142,7 @@ foreach name {OP_Noop OP_Explain OP_Abortable} { set in3($name) 0 set out2($name) 0 set out3($name) 0 + set ncycle($name) 0 set op($name) -1 set order($nOp) $name incr nOp @@ -285,6 +288,7 @@ for {set i 0} {$i<=$max} {incr i} { if {$in3($name)} {incr x 8} if {$out2($name)} {incr x 16} if {$out3($name)} {incr x 32} + if {$ncycle($name)} {incr x 64} } set bv($i) $x } @@ -299,6 +303,7 @@ puts "#define OPFLG_IN2 0x04 /* in2: P2 is an input */" puts "#define OPFLG_IN3 0x08 /* in3: P3 is an input */" puts "#define OPFLG_OUT2 0x10 /* out2: P2 is an output */" puts "#define OPFLG_OUT3 0x20 /* out3: P3 is an output */" +puts "#define OPFLG_NCYCLE 0x40 /* ncycle:Cycles count against P1 */" puts "#define OPFLG_INITIALIZER \173\\" for {set i 0} {$i<=$max} {incr i} { if {$i%8==0} { From 09d953706c29d9189e7c618d6c3fd579d571b335 Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 6 Dec 2022 18:48:37 +0000 Subject: [PATCH 04/11] For CLI .sha3sum, emit warning to stderr for any invalidly encoded text fields. FossilOrigin-Name: 8e833ecc81367658e81acd1d3a0242ab954a62d6f719af56f1d185656d0b73e6 --- manifest | 15 ++++------ manifest.uuid | 2 +- src/shell.c.in | 74 ++++++++++++++++++++++++++------------------------ 3 files changed, 46 insertions(+), 45 deletions(-) diff --git a/manifest b/manifest index 34ffa10ecc..56f9cf4031 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cause\sCLI\s.sha3sum\sto\swarn\sof\stext\sfields\sthat\sdo\snot\ssurvive\sCAST(CAST(t\sas\sBLOB)\sAS\sTEXT)\sdue\sto\sinvalid\sUTF\sencoding. -D 2022-12-03T16:09:32.554 +C For\sCLI\s.sha3sum,\semit\swarning\sto\sstderr\sfor\sany\sinvalidly\sencoded\stext\sfields. +D 2022-12-06T18:48:37.123 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 6983de0e6b8b68c97f82f9fca27ffb8f17161cff4d0d48fdf9eb40b213da0cc6 -F src/shell.c.in aa3e4c78ae25c24562b0c3497a88875baa61425714437143170b2c462842035d +F src/shell.c.in cf32c16e1b4ff8e434b51fe38753bfddccd75d4a3ecb1fb838a6f96a70e2ff3a F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2065,11 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9807656f8a7c2a893d3f68ee5592f44826b8e999ae66f7d9000674b5c1b0207 -R 9f83c1fb14dddfc272778480afabb0a3 -T *branch * sha3sum_text_validation -T *sym-sha3sum_text_validation * -T -sym-trunk * +P 123f2a0785790addf9c60a0fd09077dda9cb84d33a2594901a2b22bb555be491 +R 1a108a993a6e5c4e1463804542f95600 U larrybr -Z 1ba97ab620561a8dccaf913734ec9094 +Z 23502144cb7b8e35b977c4658f2c6efa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b35a727cc3..b0fd6bc454 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -123f2a0785790addf9c60a0fd09077dda9cb84d33a2594901a2b22bb555be491 \ No newline at end of file +8e833ecc81367658e81acd1d3a0242ab954a62d6f719af56f1d185656d0b73e6 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 1f0ac24c81..a74614eaac 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10077,7 +10077,6 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_stmt *pStmt; /* For querying tables names */ char *zSql; /* SQL to be run */ char *zSep; /* Separator */ - char *zRevText; /* Query for reversible to-blob-to-text check */ ShellText sSql; /* Complete SQL for the query to run the hash */ ShellText sQuery; /* Set of queries used to read all content */ open_db(p, 0); @@ -10126,24 +10125,6 @@ static int do_meta_command(char *zLine, ShellState *p){ " ORDER BY 1 collate nocase"; } sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - zRevText = sqlite3_mprintf( - /* lower-case query is first run, producing upper-case query. */ - "with text_cols as materialized(" - "select tname, cname " - "from (" - " select ss.tname as tname, ti.name as cname, lower(type) as lca" - " from (%s) ss join pragma_table_info(tname) ti" - " where (instr(lca,'int')=0 and " - " instr(lca,'char')+instr(lca,'clob')+instr(lca,'text')>0)" - "))" - "select 'SELECT total(bad_text_count) AS bad_text_count" - " FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query" - " from (select 'SELECT COUNT(*) AS bad_text_count" - " FROM '||tname||' WHERE '" - " ||group_concat('CAST(CAST('||cname||' AS BLOB) AS TEXT)<>'||cname," - " ' OR ') as query, tname from text_cols group by tname)", - zSql); - shell_check_oom(zRevText); initText(&sQuery); initText(&sSql); appendText(&sSql, "WITH [sha3sum$query](a,b) AS(",0); @@ -10200,34 +10181,56 @@ static int do_meta_command(char *zLine, ShellState *p){ shell_exec(p, zSql, 0); } { - int lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); + int lrc; + char *zRevText = /* Query for reversible to-blob-to-text check */ + "SELECT lower(name) as tname FROM sqlite_schema\n" + "WHERE type='table' AND coalesce(rootpage,0)>1\n" + "AND name NOT LIKE 'sqlite_%%'%s\n" + "ORDER BY 1 collate nocase"; + zRevText = sqlite3_mprintf(zRevText, zLike? " AND name LIKE $tspec" : ""); + zRevText = sqlite3_mprintf( + /* lower-case query is first run, producing upper-case query. */ + "with tabcols as materialized(\n" + "select tname, cname\n" + "from (" + " select ss.tname as tname, ti.name as cname\n" + " from (%z) ss\n inner join pragma_table_info(tname) ti))\n" + "select 'SELECT total(bad_text_count) AS bad_text_count\n" + "FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query\n" + " from (select 'SELECT COUNT(*) AS bad_text_count\n" + "FROM '||tname||' WHERE '\n" + "||group_concat('CAST(CAST('||cname||' AS BLOB) AS TEXT)<>'||cname\n" + "|| ' AND typeof('||cname||')=''text'' ',\n" + "' OR ') as query, tname from tabcols group by tname)" + , zRevText); + shell_check_oom(zRevText); if( bDebug ) utf8_printf(p->out, "%s\n", zRevText); + lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); assert(lrc==SQLITE_OK); + if( zLike ) sqlite3_bind_text(pStmt,1,zLike,-1,SQLITE_STATIC); lrc = SQLITE_ROW==sqlite3_step(pStmt); - assert(lrc!=0); if( lrc ){ const char *zGenQuery = sqlite3_column_text(pStmt,0); sqlite3_stmt *pCheckStmt; - double countIrreversible; lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0); if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery); - assert(lrc==SQLITE_OK); - lrc = SQLITE_ROW==sqlite3_step(pCheckStmt); - assert(lrc!=0); - countIrreversible = sqlite3_column_double(pCheckStmt, 0); - sqlite3_finalize(pStmt); - sqlite3_finalize(pCheckStmt); - if( countIrreversible>0 ){ - int n = (int)(countIrreversible + 0.5); - utf8_printf(stderr, - "Digest includes %d invalidly encoded text field%s.\n", - n, (n>1)? "s": ""); + if( SQLITE_OK==lrc ){ + if( SQLITE_ROW==sqlite3_step(pCheckStmt) ){ + double countIrreversible = sqlite3_column_double(pCheckStmt, 0); + if( countIrreversible>0 ){ + int n = (int)(countIrreversible + 0.5); + utf8_printf(stderr, + "Digest includes %d invalidly encoded text field%s.\n", + n, (n>1)? "s": ""); + } + } + sqlite3_finalize(pCheckStmt); } + sqlite3_finalize(pStmt); } - + sqlite3_free(zRevText); } sqlite3_free(zSql); - sqlite3_free(zRevText); }else #if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) @@ -11008,6 +11011,7 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss){ goto PlainScan; case '`': case '\'': case '"': if(*zLine==cWait){ + /* Swallow doubled end-delimiter.*/ ++zLine; continue; } From 3c01af6ffc4e7614c6b1a271aa48828e5d4b11dc Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 6 Dec 2022 19:20:49 +0000 Subject: [PATCH 05/11] Avoid several -Wall warnings in textfixture build. FossilOrigin-Name: 03ae8680e430c6d2c39ca26fa2cb6eed46e3c3e52b53f7a8bf548560446c6d3d --- ext/misc/base64.c | 3 ++- ext/misc/basexx.c | 1 - manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/test_demovfs.c | 1 - 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/ext/misc/base64.c b/ext/misc/base64.c index d2185b5405..81767379b1 100755 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -91,7 +91,8 @@ static const ubyte b64DigitValues[128] = { static const char b64Numerals[64+1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -#define BX_DV_PROTO(c) ((((ubyte)(c))<0x80)? b64DigitValues[c] : 0x80) +#define BX_DV_PROTO(c) \ + ((((ubyte)(c))<0x80)? (ubyte)(b64DigitValues[(ubyte)(c)]) : 0x80) #define IS_BX_DIGIT(bdp) (((ubyte)(bdp))<0x80) #define IS_BX_WS(bdp) ((bdp)==WS) #define IS_BX_PAD(bdp) ((bdp)==PC) diff --git a/ext/misc/basexx.c b/ext/misc/basexx.c index 9cb43efeaf..2ca2906fae 100644 --- a/ext/misc/basexx.c +++ b/ext/misc/basexx.c @@ -69,7 +69,6 @@ int sqlite3_basexx_init(sqlite3 *db, char **pzErr, init_api_ptr(pApi); int rc1 = BASE64_INIT(db); int rc2 = BASE85_INIT(db); - int rc = SQLITE_OK; if( rc1==SQLITE_OK && rc2==SQLITE_OK ){ BASE64_EXPOSE(db, pzErr); diff --git a/manifest b/manifest index c4286f7f7d..8f1f6b3afe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\ssqlite3_stmt_scanstatus_v2()\sreturn\san\sNCYCLE\svalue\sfor\sall\sloops,\snot\sjust\svirtual\stables\sones.\sThe\svalue\sreturned\sis\sthe\ssum\sof\sthe\sNCYCLE\scounts\sfor\sthe\svarious\sopcodes\sthat\smove\sor\sread\sdata\sfrom\sthe\stable\sor\sindex\scursor\sassociated\swith\sthe\sloop. -D 2022-12-06T18:48:06.154 +C Avoid\sseveral\s-Wall\swarnings\sin\stextfixture\sbuild. +D 2022-12-06T19:20:49.029 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -289,9 +289,9 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c fb140039d4e15710298e28cafe12d2bdff1b927fc57540cbcd07996f82bf9800 x +F ext/misc/base64.c 1ad313d38dc081abe5a87fa51e44f07ec6cfb618aa9606a77e42bd2fe706a67f x F ext/misc/base85.c ace591246855806a70e63eb6705ce8736f4b782da2137bdbb5d39cc337346e0d -F ext/misc/basexx.c b37ea21f00deea1029d2ddc670f5701d9370ae06f7d173c5bed79d0f26a67ec1 +F ext/misc/basexx.c 58f72b3c159e875bfd2d30a56254e7f2098961c31dc733773a9270de7066aa4c F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b752f46411e4e47e34dce6f0c88bc8e51bb821ba9e49bfcd882506451c928f69 @@ -672,7 +672,7 @@ F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 F src/test_config.c 8264637b06a3c1f0727c88d1ea32dcf7986b9e7e358a970cae87cdac8a5b2708 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f -F src/test_demovfs.c 7cc7623d1025d1e92c51da20fd25060759733b7a356a121545a3b7d2faa8a0f1 +F src/test_demovfs.c 38a459d1c78fd9afa770445b224c485e079018d6ac07332ff9bd07b54d2b8ce9 F src/test_devsym.c aff2255ea290d7718da08af30cdf18e470ff7325a5eff63e0057b1496ed66593 F src/test_fs.c ba1e1dc18fd3159fdba0b9c4256f14032159785320dfbd6776eb9973cb75d480 F src/test_func.c 24df3a346c012b1fc9e1001d346db6054deb426db0a7437e92490630e71c9b0a @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 540e895d877fab1ea138786e56923a202018f68c78199a89adfc296c75735b30 -R e347a9226ee206cfe1e680e2e5e722d0 -U dan -Z eaf9f5c14b0f034256d24205aa5d85c9 +P 9499b2f51e8174c6b8a67840c92ba23b7dd1dc8dc2b91fca0c5dc07b71662149 +R 6d3961aec4c1529b866a56388835711c +U larrybr +Z 815c066cfe76a0b0ad34b27b7be96db7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 79c29e3304..914d5983d2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9499b2f51e8174c6b8a67840c92ba23b7dd1dc8dc2b91fca0c5dc07b71662149 \ No newline at end of file +03ae8680e430c6d2c39ca26fa2cb6eed46e3c3e52b53f7a8bf548560446c6d3d \ No newline at end of file diff --git a/src/test_demovfs.c b/src/test_demovfs.c index 29307675bd..e990e98f29 100644 --- a/src/test_demovfs.c +++ b/src/test_demovfs.c @@ -461,7 +461,6 @@ static int demoDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ if( rc==0 && dirSync ){ int dfd; /* File descriptor open on directory */ - int i; /* Iterator variable */ char *zSlash; char zDir[MAXPATHNAME+1]; /* Name of directory containing file zPath */ From b54f71e205c578303fa50cfb11a985e9b73ca3b0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 7 Dec 2022 00:14:25 +0000 Subject: [PATCH 06/11] Fix a (harmless) off-by-one error in code generation that comes up when doing a DISTINCT query against a virtual table with an OR term in the WHERE clause and where the ORDER BY clause has 64 or more references to the result set. [forum:/forumpost/dfe8084751|Forum post dfe8084751]. FossilOrigin-Name: 04af7ef77043702f93cbff23548610759786893bd3d4d6fc08181e1e249c6663 --- manifest | 17 ++++++++--------- manifest.uuid | 2 +- src/where.c | 2 +- test/where.test | 15 +++++++++++++++ 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index fc2a006d5f..8e9257d9fa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sCLI\s.sha3sum\sto\swarn\sof\stext\sfields\sthat\sare\snot\sto-blob-to-text\sreversible. -D 2022-12-06T19:32:07.195 +C Fix\sa\s(harmless)\soff-by-one\serror\sin\scode\sgeneration\sthat\scomes\sup\swhen\ndoing\sa\sDISTINCT\squery\sagainst\sa\svirtual\stable\swith\san\sOR\sterm\sin\sthe\nWHERE\sclause\sand\swhere\sthe\sORDER\sBY\sclause\shas\s64\sor\smore\sreferences\sto\nthe\sresult\sset.\s\s[forum:/forumpost/dfe8084751|Forum\spost\sdfe8084751]. +D 2022-12-07T00:14:25.828 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 20f4f51d2d5fb19b984e6ea381b26cf627cc93e64dd9b2ce6a94531aec2f5916 +F src/where.c adc6783cd3b136271a70001182e899d3523e199bda41ead5c3e2ab4a84456af1 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 76bca3379219880d2527493b71a3be49e696f75396d3481e4de5d4ceec7886b2 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae @@ -1886,7 +1886,7 @@ F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2a F test/walvfs.test e1a6ad0f3c78e98b55c3d5f0889cf366cc0d0a1cb2bccb44ac9ec67384adc4a1 F test/wapp.tcl b440cd8cf57953d3a49e7ee81e6a18f18efdaf113b69f7d8482b0710a64566ec F test/wapptest.tcl 1bea58a6a8e68a73f542ee4fca28b771b84ed803bd0c9e385087070b3d747b3c x -F test/where.test 98208c95b574269980132c347b4bdb8992c6d5fc30c1954938593336d12e7447 +F test/where.test 3954cf22ba7b17f9606e177001d2963bcd1ecfbc6e1e7caadb14462f7eecd099 F test/where2.test 03c21a11e7b90e2845fc3c8b4002fc44cc2797fa74c86ee47d70bd7ea4f29ed6 F test/where3.test 5b4ffc0ac2ea0fe92f02b1244b7531522fe4d7bccf6fa8741d54e82c10e67753 F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8 @@ -2067,9 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 03ae8680e430c6d2c39ca26fa2cb6eed46e3c3e52b53f7a8bf548560446c6d3d 1d01f8483af7b6a4e93a49fd8bbb6a2b9300e969b574efeb96d56081a33055c4 -R 2aee8123cf630f6dc747669c6dfb6cfe -T +closed 1d01f8483af7b6a4e93a49fd8bbb6a2b9300e969b574efeb96d56081a33055c4 -U larrybr -Z 5b22d70d9e69e4deb5a1464c5dd02f65 +P bbde0f36d03cdbbc749427fe7d2dafd5c5031c9e655ebd772857b521f53eb18f +R 250ed9b5c78a34895ae8f1ae12a9ecc7 +U drh +Z d0ffb89df284ba3dde1a0bb1536bd8ea # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0a576fcc73..d731886fa9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bbde0f36d03cdbbc749427fe7d2dafd5c5031c9e655ebd772857b521f53eb18f \ No newline at end of file +04af7ef77043702f93cbff23548610759786893bd3d4d6fc08181e1e249c6663 \ No newline at end of file diff --git a/src/where.c b/src/where.c index a133799472..1cf9592070 100644 --- a/src/where.c +++ b/src/where.c @@ -67,7 +67,7 @@ int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ ** block sorting is required. */ int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ - return pWInfo->nOBSat; + return pWInfo->nOBSat<0 ? 0 : pWInfo->nOBSat; } /* diff --git a/test/where.test b/test/where.test index 3f4aa8e1be..4f7c2f84b2 100644 --- a/test/where.test +++ b/test/where.test @@ -1618,4 +1618,19 @@ do_execsql_test where-28.1 { 19 5 } +# 2022-12-07 Yong Heng [https://sqlite.org/forum/forumpost/dfe8084751] +# +do_execsql_test where-29.1 { + SELECT DISTINCT 'xyz' FROM pragma_cache_size + WHERE rowid OR abs(0) + ORDER BY + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1; +} {xyz} + finish_test From 7c66faa634be0e89553e0a985270788751f82238 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 7 Dec 2022 16:58:04 +0000 Subject: [PATCH 07/11] Streamline and improve testing of the locking in the memdb VFS. Follow-on to [15f0be8a640e7bfa]. FossilOrigin-Name: d71a08375aeb525c10037c373b8eeb7e29f7dfaf7c4bfc02f4d99616c5940405 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/memdb.c | 28 ++++++++++++++++++++-------- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 8e9257d9fa..d4b5f8cb25 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\s(harmless)\soff-by-one\serror\sin\scode\sgeneration\sthat\scomes\sup\swhen\ndoing\sa\sDISTINCT\squery\sagainst\sa\svirtual\stable\swith\san\sOR\sterm\sin\sthe\nWHERE\sclause\sand\swhere\sthe\sORDER\sBY\sclause\shas\s64\sor\smore\sreferences\sto\nthe\sresult\sset.\s\s[forum:/forumpost/dfe8084751|Forum\spost\sdfe8084751]. -D 2022-12-07T00:14:25.828 +C Streamline\sand\simprove\stesting\sof\sthe\slocking\sin\sthe\smemdb\sVFS.\nFollow-on\sto\s[15f0be8a640e7bfa]. +D 2022-12-07T16:58:04.002 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -615,7 +615,7 @@ F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75 F src/mem3.c 30301196cace2a085cbedee1326a49f4b26deff0af68774ca82c1f7c06fda4f6 F src/mem5.c 5a3dbd8ac8a6501152a4fc1fcae9b0900c2d7eb0589c4ec7456fdde15725a26c -F src/memdb.c fa280078fb48c4bb7ef47e361cd958938a1a3c46a9a45f6622da6efcb57cc055 +F src/memdb.c 3c1f3a3daa670294bad0056e66c17f5fe75cfa1da1850056a5a94347ec32e6bd F src/memjournal.c c283c6c95d940eb9dc70f1863eef3ee40382dbd35e5a1108026e7817c206e8a0 F src/msvc.h 3a15918220367a8876be3fa4f2abe423a861491e84b864fb2b7426bf022a28f8 F src/mutex.c 5e3409715552348732e97b9194abe92fdfcd934cfb681df4ba0ab87ac6c18d25 @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bbde0f36d03cdbbc749427fe7d2dafd5c5031c9e655ebd772857b521f53eb18f -R 250ed9b5c78a34895ae8f1ae12a9ecc7 +P 04af7ef77043702f93cbff23548610759786893bd3d4d6fc08181e1e249c6663 +R 12ec76fa12aeff3e27ce79fb7684d7d6 U drh -Z d0ffb89df284ba3dde1a0bb1536bd8ea +Z af8ae60fb39ff0cbbaef800ceb1294d9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d731886fa9..96dc67715e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04af7ef77043702f93cbff23548610759786893bd3d4d6fc08181e1e249c6663 \ No newline at end of file +d71a08375aeb525c10037c373b8eeb7e29f7dfaf7c4bfc02f4d99616c5940405 \ No newline at end of file diff --git a/src/memdb.c b/src/memdb.c index 7485f51ffb..4ea3cffde8 100644 --- a/src/memdb.c +++ b/src/memdb.c @@ -370,21 +370,33 @@ static int memdbLock(sqlite3_file *pFile, int eLock){ int rc = SQLITE_OK; if( eLock==pThis->eLock ) return SQLITE_OK; memdbEnter(p); + assert( p->nWrLock==0 || p->nWrLock==1 ); /* No more than 1 write lock */ if( eLock>SQLITE_LOCK_SHARED ){ assert( pThis->eLock>=SQLITE_LOCK_SHARED ); if( p->mFlags & SQLITE_DESERIALIZE_READONLY ){ rc = SQLITE_READONLY; }else if( eLock==SQLITE_LOCK_EXCLUSIVE ){ - /* Taking an EXCLUSIVE lock. Fail if we only have SHARED and any - ** other client has any kind of write-lock. Also fail if any other - ** client is holding read-lock. */ - if( pThis->eLock<=SQLITE_LOCK_SHARED && p->nWrLock ){ - rc = SQLITE_BUSY; - }else if( p->nRdLock>1 ){ + /* We never go for an EXCLUSIVE lock unless we already hold SHARED or + ** higher */ + assert( pThis->eLock>=SQLITE_LOCK_SHARED ); + testcase( pThis->eLock==SQLITE_LOCK_SHARED ); + + /* Because we are holding SHARED or more, there must be at least + ** one read lock */ + assert( p->nRdLock>0 ); + + /* The only way that there can be an existing write lock is if we + ** currently hold it. Otherwise, we would have never been able to + ** promote from NONE to SHARED. */ + assert( p->nWrLock==0 || pThis->eLock>SQLITE_LOCK_SHARED ); + + if( p->nRdLock>1 ){ + /* Cannot take EXCLUSIVE if somebody else is holding SHARED */ rc = SQLITE_BUSY; + }else{ + p->nWrLock = 1; } - p->nWrLock = 1; - }else if( pThis->eLock<=SQLITE_LOCK_SHARED ){ + }else if( ALWAYS(pThis->eLock<=SQLITE_LOCK_SHARED) ){ /* Upgrading to RESERVED or PENDING from SHARED. Fail if any other ** client has a write-lock of any kind. */ if( p->nWrLock ){ From f2cc3387f78aac0a0cd2c0bca251fa581a9fd7f7 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 7 Dec 2022 17:29:17 +0000 Subject: [PATCH 08/11] Have sqlite3_stmt_scanstatus() report cycle, loop and row counts separately for creating an automatic index and using that automatic index. FossilOrigin-Name: 3bc9df82ea5b0fb085c56a326a65e19c9baf98d48d8fa6344c0d7004747594ba --- manifest | 16 ++++++------ manifest.uuid | 2 +- src/where.c | 51 ++++++++++++++++++++++++++++++++++++++ test/scanstatus2.test | 57 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d4b5f8cb25..2d0b3556cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Streamline\sand\simprove\stesting\sof\sthe\slocking\sin\sthe\smemdb\sVFS.\nFollow-on\sto\s[15f0be8a640e7bfa]. -D 2022-12-07T16:58:04.002 +C Have\ssqlite3_stmt_scanstatus()\sreport\scycle,\sloop\sand\srow\scounts\sseparately\sfor\screating\san\sautomatic\sindex\sand\susing\sthat\sautomatic\sindex. +D 2022-12-07T17:29:17.346 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c adc6783cd3b136271a70001182e899d3523e199bda41ead5c3e2ab4a84456af1 +F src/where.c 3ee4050e453dfd03da4a8e4b5561d20b20b2302ec9d02437fb62c6f29bb78401 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 76bca3379219880d2527493b71a3be49e696f75396d3481e4de5d4ceec7886b2 F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae @@ -1458,7 +1458,7 @@ F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test 7dbcfd6adc6a8df6abc59f83d6da5a27e1bce0b2f6fa55147c8176d7c44e0450 -F test/scanstatus2.test 3f45ec1d8b45dcb0df7f98daf3364fc2e03bf89a8c3dd2428251a5e0a34de7af +F test/scanstatus2.test 07e2d1b86a07cd8f6cbef9546c9d6b27af0de5e613b6f38f98666f9c729bd4c7 F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 04af7ef77043702f93cbff23548610759786893bd3d4d6fc08181e1e249c6663 -R 12ec76fa12aeff3e27ce79fb7684d7d6 -U drh -Z af8ae60fb39ff0cbbaef800ceb1294d9 +P d71a08375aeb525c10037c373b8eeb7e29f7dfaf7c4bfc02f4d99616c5940405 +R aa03a3260eaa51e2eda4cccd90080651 +U dan +Z 28dcdc9b3132851feb893087f0e4ae04 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 96dc67715e..a5f6f520a5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d71a08375aeb525c10037c373b8eeb7e29f7dfaf7c4bfc02f4d99616c5940405 \ No newline at end of file +3bc9df82ea5b0fb085c56a326a65e19c9baf98d48d8fa6344c0d7004747594ba \ No newline at end of file diff --git a/src/where.c b/src/where.c index 1cf9592070..25ad18b282 100644 --- a/src/where.c +++ b/src/where.c @@ -812,6 +812,51 @@ static int termCanDriveIndex( #ifndef SQLITE_OMIT_AUTOMATIC_INDEX + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Argument pIdx represents an automatic index that the current statement +** will create and populate. Add an OP_Explain with text of the form: +** +** CREATE AUTOMATIC INDEX ON () [WHERE ] +** +** This is only required if sqlite3_stmt_scanstatus() is enabled, to +** associate an SQLITE_SCANSTAT_NCYCLE and SQLITE_SCANSTAT_NLOOP +** values with. In order to avoid breaking legacy code and test cases, +** the OP_Explain is not added if this is an EXPLAIN QUERY PLAN command. +*/ +static void explainAutomaticIndex( + Parse *pParse, + Index *pIdx, /* Automatic index to explain */ + int bPartial, /* True if pIdx is a partial index */ + int *pAddrExplain /* OUT: Address of OP_Explain */ +){ + if( pParse->explain!=2 ){ + Table *pTab = pIdx->pTable; + const char *zSep = ""; + char *zText = 0; + int ii = 0; + zText = sqlite3_mprintf("CREATE AUTOMATIC INDEX ON %s(", pTab->zName); + assert( pIdx->nColumn>1 ); + assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID ); + for(ii=0; ii<(pIdx->nColumn-1); ii++){ + const char *zName = 0; + int iCol = pIdx->aiColumn[ii]; + + zName = pTab->aCol[iCol].zCnName; + zText = sqlite3_mprintf("%z%s%s", zText, zSep, zName); + zSep = ", "; + } + *pAddrExplain = sqlite3VdbeExplain( + pParse, 0, "%s)%s", zText, (bPartial ? " WHERE " : "") + ); + sqlite3_free(zText); + } +} +#else +# define explainAutomaticIndex(a,b,c,d) +#endif + /* ** Generate code to construct the Index object for an automatic index ** and to set up the WhereLevel object pLevel so that the code generator @@ -847,6 +892,9 @@ static SQLITE_NOINLINE void constructAutomaticIndex( SrcItem *pTabItem; /* FROM clause term being indexed */ int addrCounter = 0; /* Address where integer counter is initialized */ int regBase; /* Array of registers where record is assembled */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExp = 0; /* Address of OP_Explain */ +#endif /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ @@ -970,6 +1018,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( pIdx->azColl[n] = sqlite3StrBINARY; /* Create the automatic index */ + explainAutomaticIndex(pParse, pIdx, pPartial!=0, &addrExp); assert( pLevel->iIdxCur>=0 ); pLevel->iIdxCur = pParse->nTab++; sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); @@ -1005,6 +1054,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, regBase, pLoop->u.btree.nEq); } + sqlite3VdbeScanStatusCounters(v, addrExp, addrExp, sqlite3VdbeCurrentAddr(v)); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); @@ -1025,6 +1075,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( /* Jump here when skipping the initialization */ sqlite3VdbeJumpHere(v, addrInit); + sqlite3VdbeScanStatusRange(v, addrExp, addrExp, -1); end_auto_index_create: sqlite3ExprDelete(pParse->db, pPartial); diff --git a/test/scanstatus2.test b/test/scanstatus2.test index 9a4758b6d4..1b5c18cb8b 100644 --- a/test/scanstatus2.test +++ b/test/scanstatus2.test @@ -170,9 +170,66 @@ do_graph_test 2.1 { } { QUERY (nCycle=nnn) --SCAN x1 (nCycle=nnn) +--CREATE AUTOMATIC INDEX ON x2(c, d) (nCycle=nnn) --SEARCH x2 USING AUTOMATIC COVERING INDEX (c=?) (nCycle=nnn) } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE TABLE rt1 (id INTEGER PRIMARY KEY, x1, x2); + CREATE TABLE rt2 (id, x1, x2); +} + +do_graph_test 4.1 { + SELECT * FROM rt1, rt2 WHERE rt1.id%2 AND rt2.x1=rt1.x1; +} { +QUERY (nCycle=nnn) +--SCAN rt1 (nCycle=nnn) +--CREATE AUTOMATIC INDEX ON rt2(x1, id, x2) (nCycle=nnn) +--SEARCH rt2 USING AUTOMATIC COVERING INDEX (x1=?) (nCycle=nnn) +} + +do_graph_test 4.2 { + SELECT rt2.id FROM rt1, rt2 WHERE rt1.id%2 AND rt2.x1=rt1.x1; +} { +QUERY (nCycle=nnn) +--SCAN rt1 (nCycle=nnn) +--CREATE AUTOMATIC INDEX ON rt2(x1, id) (nCycle=nnn) +--SEARCH rt2 USING AUTOMATIC COVERING INDEX (x1=?) (nCycle=nnn) +} + +do_graph_test 4.3 { + SELECT rt2.id FROM rt1, rt2 WHERE rt1.id%2 AND (rt2.x1+1)=(rt1.x1+1); +} { +QUERY (nCycle=nnn) +--SCAN rt1 (nCycle=nnn) +--SCAN rt2 (nCycle=nnn) +} + +do_graph_test 4.4 { + SELECT rt2.id FROM rt1, rt2 WHERE rt1.id%2 AND rt2.x1=(rt1.x1+1) AND rt2.id>5; +} { +QUERY (nCycle=nnn) +--SCAN rt1 (nCycle=nnn) +--CREATE AUTOMATIC INDEX ON rt2(x1, id) WHERE (nCycle=nnn) +--SEARCH rt2 USING AUTOMATIC PARTIAL COVERING INDEX (x1=?) (nCycle=nnn) +} + +do_graph_test 4.5 { + SELECT v1.cnt FROM rt1, ( + SELECT count(*) AS cnt, rt2.x1 AS x1 FROM rt2 GROUP BY x1 + ) AS v1 WHERE rt1.x1=v1.x1 +} { +QUERY (nCycle=nnn) +--MATERIALIZE v1 (nCycle=nnn) +----SCAN rt2 (nCycle=nnn) +----USE TEMP B-TREE FOR GROUP BY +--SCAN rt1 (nCycle=nnn) +--CREATE AUTOMATIC INDEX ON v1(x1, cnt) (nCycle=nnn) +--SEARCH v1 USING AUTOMATIC COVERING INDEX (x1=?) (nCycle=nnn) +} + finish_test From 7474de702d76cfa49e8dca5660806068fb73437d Mon Sep 17 00:00:00 2001 From: larrybr Date: Wed, 7 Dec 2022 19:29:13 +0000 Subject: [PATCH 09/11] Omit CLI use of pragma_table_xinfo when it is not defined in the build. FossilOrigin-Name: 976c23520fa64b39aa54047f2fd12445e65940206768ce51c0d690d0d0d570ab --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2d0b3556cf..601f881b94 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\ssqlite3_stmt_scanstatus()\sreport\scycle,\sloop\sand\srow\scounts\sseparately\sfor\screating\san\sautomatic\sindex\sand\susing\sthat\sautomatic\sindex. -D 2022-12-07T17:29:17.346 +C Omit\sCLI\suse\sof\spragma_table_xinfo\swhen\sit\sis\snot\sdefined\sin\sthe\sbuild. +D 2022-12-07T19:29:13.407 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 56162cde2f71314826f831618eae853205628080afa25cd90faef7a9b761dee4 -F src/shell.c.in b5bc6e14605b8d1e8de729454d98d3353ad7e529a8d2b6e34080d068e99bff99 +F src/shell.c.in fbd48504e6d8953950deddb3ef83de58758a84e58e7994b77792b4568981ea98 F src/sqlite.h.in 1fe1836879ecbb2e28f00f44eb6092db09a2a06bf072af351c6c2466bd515496 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d71a08375aeb525c10037c373b8eeb7e29f7dfaf7c4bfc02f4d99616c5940405 -R aa03a3260eaa51e2eda4cccd90080651 -U dan -Z 28dcdc9b3132851feb893087f0e4ae04 +P 3bc9df82ea5b0fb085c56a326a65e19c9baf98d48d8fa6344c0d7004747594ba +R eb3572b995ebd9a522f40116cadefed1 +U larrybr +Z dfd7917e394baecbbad398b94b2b49b6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a5f6f520a5..fffb99e602 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3bc9df82ea5b0fb085c56a326a65e19c9baf98d48d8fa6344c0d7004747594ba \ No newline at end of file +976c23520fa64b39aa54047f2fd12445e65940206768ce51c0d690d0d0d570ab \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 9c1fe111f5..44105bf1b3 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10328,6 +10328,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ shell_exec(p, zSql, 0); } +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && !defined(SQLITE_OMIT_VIRTUALTABLE) { int lrc; char *zRevText = /* Query for reversible to-blob-to-text check */ @@ -10378,6 +10379,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_free(zRevText); } +#endif /* !defined(*_OMIT_SCHEMA_PRAGMAS) && !defined(*_OMIT_VIRTUALTABLE) */ sqlite3_free(zSql); }else From 209dbab9979628ea1587a07e11c0bcfe78971dd8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 7 Dec 2022 19:51:48 +0000 Subject: [PATCH 10/11] Fix harmless compiler warning in the dynamic continuation prompt of the CLI. FossilOrigin-Name: 68947b0a1147365a29e335d5e4dc55fe5d9afe3562a0709b111067d6017c5a42 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 601f881b94..e1ead94721 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sCLI\suse\sof\spragma_table_xinfo\swhen\sit\sis\snot\sdefined\sin\sthe\sbuild. -D 2022-12-07T19:29:13.407 +C Fix\sharmless\scompiler\swarning\sin\sthe\sdynamic\scontinuation\sprompt\sof\sthe\sCLI. +D 2022-12-07T19:51:48.626 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 56162cde2f71314826f831618eae853205628080afa25cd90faef7a9b761dee4 -F src/shell.c.in fbd48504e6d8953950deddb3ef83de58758a84e58e7994b77792b4568981ea98 +F src/shell.c.in bcf8552c82f2c84650e39a6d638373569c2035942c0497b83eef197169e0305a F src/sqlite.h.in 1fe1836879ecbb2e28f00f44eb6092db09a2a06bf072af351c6c2466bd515496 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3bc9df82ea5b0fb085c56a326a65e19c9baf98d48d8fa6344c0d7004747594ba -R eb3572b995ebd9a522f40116cadefed1 -U larrybr -Z dfd7917e394baecbbad398b94b2b49b6 +P 976c23520fa64b39aa54047f2fd12445e65940206768ce51c0d690d0d0d570ab +R b08d7359f3a0adfc8758cdabe6250a50 +U drh +Z 38d73aa067c1a85ec056e9ed86a84d61 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fffb99e602..070399b39f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -976c23520fa64b39aa54047f2fd12445e65940206768ce51c0d690d0d0d570ab \ No newline at end of file +68947b0a1147365a29e335d5e4dc55fe5d9afe3562a0709b111067d6017c5a42 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 44105bf1b3..e4b054f450 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -537,7 +537,7 @@ static char *dynamicContinuePrompt(void){ if( dynPrompt.zScannerAwaits ){ int ncp = strlen(continuePrompt), ndp = strlen(dynPrompt.zScannerAwaits); if( ndp > ncp-3 ) return continuePrompt; - strncpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits, ndp); + strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits); while( ndp<3 ) dynPrompt.dynamicPrompt[ndp++] = ' '; strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, PROMPT_LEN_MAX-4); From 7f4b066eb2b5ab99ed126809e56e8a9bd41a91e3 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 7 Dec 2022 20:09:54 +0000 Subject: [PATCH 11/11] Reduce the overhead of SQLITE_ENABLE_STMT_SCANSTATUS some. FossilOrigin-Name: 212927e97e7be7d237de08359dce0dfb9211ac406b32009a6e15afd79c006475 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/vdbe.c | 42 ++++++++++++------------------------------ src/vdbe.h | 4 ++++ src/vdbeInt.h | 6 ------ src/vdbeapi.c | 22 +++++++++++++--------- src/vdbeaux.c | 30 ++++++++++-------------------- 7 files changed, 51 insertions(+), 77 deletions(-) diff --git a/manifest b/manifest index e1ead94721..bf71b14d97 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sin\sthe\sdynamic\scontinuation\sprompt\sof\sthe\sCLI. -D 2022-12-07T19:51:48.626 +C Reduce\sthe\soverhead\sof\sSQLITE_ENABLE_STMT_SCANSTATUS\ssome. +D 2022-12-07T20:09:54.641 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -717,11 +717,11 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 313f3154e2b85a447326f5dd15de8d31a4df6ab0c3579bd58f426ff634ec9050 F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c 8fdef91d3c9bfa363e8eb381d362e2f0cadec456975ec5595c0bfdf65da84e53 -F src/vdbe.h 6d921884cf8ec6a53efba99f8b68e32e955367631743e29039840e781aaf547c -F src/vdbeInt.h e83be1eea422758d42302941e96e59d93193c373da655a87befdc03a851e0f95 -F src/vdbeapi.c 5e265ab5165b0dfeac55e6a2ad3929d64584c4e7ca106b438137d7596da91348 -F src/vdbeaux.c 0b81f317c86ed9f4ba822af66a52777fed6e8180edc79d4fc62ffe75c8e52d80 +F src/vdbe.c df9ff3fcaa3c60fa9b0bca340c3f3cfc63a51361901ca06fe18bdeadeb2a1a24 +F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c +F src/vdbeInt.h 8651e4c4e04d1860d0bdcf330cb8294e3778a9d4222be30ce4c490d9220af783 +F src/vdbeapi.c df3f73a4d0a487f2068e3c84776cd6e3fba5ae80ff612659dcfda4307686420b +F src/vdbeaux.c 25691b395bf57ae0a754570fc3ba2d6f8bd2f0a8944f1d2589b9e68114e01c2b F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 6cfed43758d57b6e3b99d9cdedfeccd86e45a07e427b22d8487cbdbebb6c522a F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 976c23520fa64b39aa54047f2fd12445e65940206768ce51c0d690d0d0d570ab -R b08d7359f3a0adfc8758cdabe6250a50 -U drh -Z 38d73aa067c1a85ec056e9ed86a84d61 +P 68947b0a1147365a29e335d5e4dc55fe5d9afe3562a0709b111067d6017c5a42 +R 4a013de93da29e8129599899cb5c1b54 +U dan +Z 4aecefd86ed6f42fbe2469d178e09d33 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 070399b39f..01cc2da2af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -68947b0a1147365a29e335d5e4dc55fe5d9afe3562a0709b111067d6017c5a42 \ No newline at end of file +212927e97e7be7d237de08359dce0dfb9211ac406b32009a6e15afd79c006475 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index cd27aff44f..12b22992b9 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -790,15 +790,12 @@ int sqlite3VdbeExec( assert( pOp>=aOp && pOp<&aOp[p->nOp]); nVmStep++; #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) - if( p->anExec ){ - assert( p->anExec && p->anCycle ); - p->anExec[(int)(pOp-aOp)]++; - pnCycle = &p->anCycle[pOp-aOp]; + pOp->nExec++; + pnCycle = &pOp->nCycle; # ifdef VDBE_PROFILE - if( sqlite3NProfileCnt==0 ) + if( sqlite3NProfileCnt==0 ) # endif - *pnCycle -= sqlite3Hwtime(); - } + *pnCycle -= sqlite3Hwtime(); #endif /* Only allow tracing if SQLITE_DEBUG is defined. @@ -7152,10 +7149,6 @@ case OP_Program: { /* jump */ pFrame->aOp = p->aOp; pFrame->nOp = p->nOp; pFrame->token = pProgram->token; -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) - pFrame->anExec = p->anExec; - pFrame->anCycle = p->anCycle; -#endif #ifdef SQLITE_DEBUG pFrame->iFrameMagic = SQLITE_FRAME_MAGIC; #endif @@ -7192,10 +7185,6 @@ case OP_Program: { /* jump */ memset(pFrame->aOnce, 0, (pProgram->nOp + 7)/8); p->aOp = aOp = pProgram->aOp; p->nOp = pProgram->nOp; -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) - p->anExec = 0; - p->anCycle = 0; -#endif #ifdef SQLITE_DEBUG /* Verify that second and subsequent executions of the same trigger do not ** try to reuse register values from the first use. */ @@ -8732,17 +8721,11 @@ default: { /* This is really OP_Noop, OP_Explain */ } #if defined(VDBE_PROFILE) - assert( pnCycle ); - if( pnCycle ){ - *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); - assert( *pnCycle < (((u64)1) << 48) ); - pnCycle = 0; - } + *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + pnCycle = 0; #elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) - if( pnCycle ){ - *pnCycle += sqlite3Hwtime(); - pnCycle = 0; - } + *pnCycle += sqlite3Hwtime(); + pnCycle = 0; #endif /* The following code adds nothing to the actual functionality @@ -8819,11 +8802,10 @@ abort_due_to_error: ** top. */ vdbe_return: #if defined(VDBE_PROFILE) - if( pnCycle ){ - *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); - assert( *pnCycle < (((u64)1) << 48) ); - pnCycle = 0; - } + if( pnCycle ){ + *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + pnCycle = 0; + } #elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) if( pnCycle ){ *pnCycle += sqlite3Hwtime(); diff --git a/src/vdbe.h b/src/vdbe.h index a199247434..3caf1f7872 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -71,6 +71,10 @@ struct VdbeOp { u32 iSrcLine; /* Source-code line that generated this opcode ** with flags in the upper 8 bits */ #endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + u64 nExec; + u64 nCycle; +#endif }; typedef struct VdbeOp VdbeOp; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index df47e05448..cd112e9c48 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -171,8 +171,6 @@ struct VdbeFrame { Vdbe *v; /* VM this frame belongs to */ VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ Op *aOp; /* Program instructions for parent frame */ - i64 *anExec; /* Event counters from parent frame */ - u64 *anCycle; /* Cycle counters from parent frame */ Mem *aMem; /* Array of memory cells for parent frame */ VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ u8 *aOnce; /* Bitmask used by OP_Once */ @@ -492,10 +490,6 @@ struct Vdbe { u32 expmask; /* Binding to these vars invalidates VM */ SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ AuxData *pAuxData; /* Linked list of auxdata allocations */ -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) - i64 *anExec; /* Number of times each op has been executed */ - u64 *anCycle; /* Sum of sqlite3HwTime() spent in each opcode */ -#endif #ifdef SQLITE_ENABLE_STMT_SCANSTATUS int nScan; /* Entries in aScan[] */ ScanStatus *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 028eb7fe64..414745e8ba 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -2123,14 +2123,12 @@ int sqlite3_stmt_scanstatus_v2( ScanStatus *pScan; int idx; - /* If the v2 flag is clear, then this function must ignore any ScanStatus - ** structures with ScanStatus.addrLoop set to 0. */ if( iScan<0 ){ int ii; if( iScanStatusOp==SQLITE_SCANSTAT_NCYCLE ){ i64 res = 0; for(ii=0; iinOp; ii++){ - res += p->anCycle[ii]; + res += p->aOp[ii].nCycle; } *(i64*)pOut = res; return 0; @@ -2141,6 +2139,8 @@ int sqlite3_stmt_scanstatus_v2( idx = iScan; pScan = &p->aScan[idx]; }else{ + /* If the COMPLEX flag is clear, then this function must ignore any + ** ScanStatus structures with ScanStatus.addrLoop set to 0. */ for(idx=0; idxnScan; idx++){ pScan = &p->aScan[idx]; if( pScan->zName ){ @@ -2154,7 +2154,7 @@ int sqlite3_stmt_scanstatus_v2( switch( iScanStatusOp ){ case SQLITE_SCANSTAT_NLOOP: { if( pScan->addrLoop>0 ){ - *(sqlite3_int64*)pOut = p->anExec[pScan->addrLoop]; + *(sqlite3_int64*)pOut = p->aOp[pScan->addrLoop].nExec; }else{ *(sqlite3_int64*)pOut = -1; } @@ -2162,7 +2162,7 @@ int sqlite3_stmt_scanstatus_v2( } case SQLITE_SCANSTAT_NVISIT: { if( pScan->addrVisit>0 ){ - *(sqlite3_int64*)pOut = p->anExec[pScan->addrVisit]; + *(sqlite3_int64*)pOut = p->aOp[pScan->addrVisit].nExec; }else{ *(sqlite3_int64*)pOut = -1; } @@ -2218,7 +2218,7 @@ int sqlite3_stmt_scanstatus_v2( if( iIns==0 ) break; if( iIns>0 ){ while( iIns<=iEnd ){ - res += p->anCycle[iIns]; + res += p->aOp[iIns].nCycle; iIns++; } }else{ @@ -2229,7 +2229,7 @@ int sqlite3_stmt_scanstatus_v2( if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_NCYCLE)==0 ){ continue; } - res += p->anCycle[iOp]; + res += p->aOp[iOp].nCycle; } } } @@ -2261,7 +2261,11 @@ int sqlite3_stmt_scanstatus( */ void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe*)pStmt; - memset(p->anExec, 0, p->nOp * sizeof(i64)); - memset(p->anCycle, 0, p->nOp * sizeof(u64)); + int ii; + for(ii=0; iinOp; ii++){ + Op *pOp = &p->aOp[ii]; + pOp->nExec = 0; + pOp->nCycle = 0; + } } #endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index e22b741196..8d284746f1 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -262,6 +262,10 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS pOp->zComment = 0; #endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + pOp->nExec = 0; + pOp->nCycle = 0; +#endif #ifdef SQLITE_DEBUG if( p->db->flags & SQLITE_VdbeAddopTrace ){ sqlite3VdbePrintOp(0, i, &p->aOp[i]); @@ -2508,8 +2512,10 @@ void sqlite3VdbeRewind(Vdbe *p){ p->iStatement = 0; p->nFkConstraint = 0; #ifdef VDBE_PROFILE - memset(p->anExec, 0, sizeof(i64)*p->nOp); - memset(p->anCycle, 0, sizeof(u64)*p->nOp); + for(i=0; inOp; i++){ + p->aOp[i].nExec = 0; + p->aOp[i].nCycle = 0; + } #endif } @@ -2617,10 +2623,6 @@ void sqlite3VdbeMakeReady( p->aVar = allocSpace(&x, 0, nVar*sizeof(Mem)); p->apArg = allocSpace(&x, 0, nArg*sizeof(Mem*)); p->apCsr = allocSpace(&x, 0, nCursor*sizeof(VdbeCursor*)); -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) - p->anExec = allocSpace(&x, 0, p->nOp*sizeof(i64)); - p->anCycle = allocSpace(&x, 0, p->nOp*sizeof(u64)); -#endif if( x.nNeeded ){ x.pSpace = p->pFree = sqlite3DbMallocRawNN(db, x.nNeeded); x.nFree = x.nNeeded; @@ -2629,10 +2631,6 @@ void sqlite3VdbeMakeReady( p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) - p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64)); - p->anCycle = allocSpace(&x, p->anCycle, p->nOp*sizeof(u64)); -#endif } } @@ -2647,10 +2645,6 @@ void sqlite3VdbeMakeReady( p->nMem = nMem; initMemArray(p->aMem, nMem, db, MEM_Undefined); memset(p->apCsr, 0, nCursor*sizeof(VdbeCursor*)); -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) - memset(p->anExec, 0, p->nOp*sizeof(i64)); - memset(p->anCycle, 0, p->nOp*sizeof(u64)); -#endif } sqlite3VdbeRewind(p); } @@ -2708,10 +2702,6 @@ static void closeCursorsInFrame(Vdbe *p){ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ Vdbe *v = pFrame->v; closeCursorsInFrame(v); -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) - v->anExec = pFrame->anExec; - v->anCycle = pFrame->anCycle; -#endif v->aOp = pFrame->aOp; v->nOp = pFrame->nOp; v->aMem = pFrame->aMem; @@ -3543,8 +3533,8 @@ int sqlite3VdbeReset(Vdbe *p){ } for(i=0; inOp; i++){ char zHdr[100]; - i64 cnt = p->anExec[i]; - i64 cycles = p->anCycle[i]; + i64 cnt = p->aOp[i].nExec; + i64 cycles = p->aOp[i].nCycle; sqlite3_snprintf(sizeof(zHdr), zHdr, "%6u %12llu %8llu ", cnt, cycles,