From 8c411f7fceffe287b5f94cf449550ee5d6c11601 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 12 Aug 2024 09:49:02 +0000 Subject: [PATCH 01/15] Resolve parameters Bind $int_N and $test_T in fuzzcheck inputs. FossilOrigin-Name: 9c10664416274df6f8da53ddd86f6356c9704ad798fd4034d2784ae433c1c056 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fuzzcheck.c | 25 +++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f7516ccb90..159330c456 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Prevent\sthe\sfts5\sxPhraseNext()\sor\sxPhraseFirst()\sAPIs\sfrom\sreturning\san\sout-of-range\scolumn\snumber,\seven\sif\sthe\sdatabase\sis\scorrupt. -D 2024-08-10T19:57:28.413 +C Resolve\sparameters\sBind\s$int_N\sand\s$test_T\sin\sfuzzcheck\sinputs. +D 2024-08-12T09:49:02.212 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1260,7 +1260,7 @@ F test/fuzz3.test 70ba57260364b83e964707b9d4b5625284239768ab907dd387c740c0370ce3 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c dc159967609d00b0cfe619e735cbbf8482570aca85711397034b0662b6c18fc7 +F test/fuzzcheck.c 6e87c27df3d95c556870187987dff6efdc712b5cea60abedc8ab9215f471907a F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -2204,8 +2204,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3778b2a9ca1cc12a88ef6c32a1ee7c58a0a829ed9715a3d32a225d377d7527ef -R e552f961f7d429a11cf017943b4b9f14 -U dan -Z 0df3f7a0f800270be689e5fef89cec38 +P d4014c87ba9b011a6a04c2bf85879b668dc762ebcbbfb50a2f8a417ce594ef88 +R 33ab7e360692d6ed2c1cd4d18565e8ac +U drh +Z 32b23835ef00f951c0107d2a77ff843c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0b2355bfab..ef73d7fbbd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d4014c87ba9b011a6a04c2bf85879b668dc762ebcbbfb50a2f8a417ce594ef88 +9c10664416274df6f8da53ddd86f6356c9704ad798fd4034d2784ae433c1c056 diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index 6cae348bd6..eaeb54d872 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -1028,6 +1028,30 @@ static int recoverDatabase(sqlite3 *db){ } return rc; } +/* +** Special parameter binding, for testing and debugging purposes. +** +** $int_NNN -> integer value NNN +** $text_TTTT -> floating point value TTT with destructor +*/ +static void bindDebugParameters(sqlite3_stmt *pStmt){ + int nVar = sqlite3_bind_parameter_count(pStmt); + int i; + for(i=0; i=4 ){ From 797fcb8433301d995dbe7731765833c82a74a1da Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 13 Aug 2024 21:04:00 +0000 Subject: [PATCH 02/15] Add the sha3_agg() aggregate to the shathree extension. FossilOrigin-Name: c4d7f9996ce4f5f6d08d83fc3abdb81b6825cfb556318f90c6896736a1da5175 --- ext/misc/shathree.c | 153 ++++++++++++++++++++++++++++++-------------- manifest | 12 ++-- manifest.uuid | 2 +- 3 files changed, 112 insertions(+), 55 deletions(-) diff --git a/ext/misc/shathree.c b/ext/misc/shathree.c index ba3ea581f8..46a864af4e 100644 --- a/ext/misc/shathree.c +++ b/ext/misc/shathree.c @@ -75,6 +75,7 @@ struct SHA3Context { unsigned nRate; /* Bytes of input accepted per Keccak iteration */ unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ + unsigned iSize; /* 224, 256, 358, or 512 */ }; /* @@ -404,6 +405,7 @@ static void KeccakF1600Step(SHA3Context *p){ */ static void SHA3Init(SHA3Context *p, int iSize){ memset(p, 0, sizeof(*p)); + p->iSize = iSize; if( iSize>=128 && iSize<=512 ){ p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; }else{ @@ -547,6 +549,60 @@ static void sha3_step_vformat( SHA3Update(p, (unsigned char*)zBuf, n); } +/* +** Update a SHA3Context using a single sqlite3_value. +*/ +static void sha3UpdateFromValue(SHA3Context *p, sqlite3_value *pVal){ + switch( sqlite3_value_type(pVal) ){ + case SQLITE_NULL: { + SHA3Update(p, (const unsigned char*)"N",1); + break; + } + case SQLITE_INTEGER: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + sqlite3_int64 v = sqlite3_value_int64(pVal); + memcpy(&u, &v, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + SHA3Update(p, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_value_double(pVal); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + SHA3Update(p,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_value_bytes(pVal); + const unsigned char *z2 = sqlite3_value_text(pVal); + sha3_step_vformat(p,"T%d:",n2); + SHA3Update(p, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_value_bytes(pVal); + const unsigned char *z2 = sqlite3_value_blob(pVal); + sha3_step_vformat(p,"B%d:",n2); + SHA3Update(p, z2, n2); + break; + } + } +} + /* ** Implementation of the sha3_query(SQL,SIZE) function. ** @@ -636,54 +692,7 @@ static void sha3QueryFunc( while( SQLITE_ROW==sqlite3_step(pStmt) ){ SHA3Update(&cx,(const unsigned char*)"R",1); for(i=0; i=1; j--){ - x[j] = u & 0xff; - u >>= 8; - } - x[0] = 'I'; - SHA3Update(&cx, x, 9); - break; - } - case SQLITE_FLOAT: { - sqlite3_uint64 u; - int j; - unsigned char x[9]; - double r = sqlite3_column_double(pStmt,i); - memcpy(&u, &r, 8); - for(j=8; j>=1; j--){ - x[j] = u & 0xff; - u >>= 8; - } - x[0] = 'F'; - SHA3Update(&cx,x,9); - break; - } - case SQLITE_TEXT: { - int n2 = sqlite3_column_bytes(pStmt, i); - const unsigned char *z2 = sqlite3_column_text(pStmt, i); - sha3_step_vformat(&cx,"T%d:",n2); - SHA3Update(&cx, z2, n2); - break; - } - case SQLITE_BLOB: { - int n2 = sqlite3_column_bytes(pStmt, i); - const unsigned char *z2 = sqlite3_column_blob(pStmt, i); - sha3_step_vformat(&cx,"B%d:",n2); - SHA3Update(&cx, z2, n2); - break; - } - } + sha3UpdateFromValue(&cx, sqlite3_column_value(pStmt,i)); } } sqlite3_finalize(pStmt); @@ -691,6 +700,44 @@ static void sha3QueryFunc( sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); } +/* +** xStep function for sha3_agg(). +*/ +static void sha3AggStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA3Context *p; + p = (SHA3Context*)sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( p->nRate==0 ){ + int sz = 256; + if( argc==2 ){ + sz = sqlite3_value_int(argv[1]); + if( sz!=224 && sz!=384 && sz!=512 ){ + sz = 256; + } + } + SHA3Init(p, sz); + } + sha3UpdateFromValue(p, argv[0]); +} + + +/* +** xFinal function for sha3_agg(). +*/ +static void sha3AggFinal(sqlite3_context *context){ + SHA3Context *p; + p = (SHA3Context*)sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( p->iSize ){ + sqlite3_result_blob(context, SHA3Final(p), p->iSize/8, SQLITE_TRANSIENT); + } +} + + #ifdef _WIN32 __declspec(dllexport) @@ -711,6 +758,16 @@ int sqlite3_shathree_init( SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, 0, sha3Func, 0, 0); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_agg", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, 0, sha3AggStep, sha3AggFinal); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_agg", 2, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, 0, sha3AggStep, sha3AggFinal); + } if( rc==SQLITE_OK ){ rc = sqlite3_create_function(db, "sha3_query", 1, SQLITE_UTF8 | SQLITE_DIRECTONLY, diff --git a/manifest b/manifest index 159330c456..13c566c83d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Resolve\sparameters\sBind\s$int_N\sand\s$test_T\sin\sfuzzcheck\sinputs. -D 2024-08-12T09:49:02.212 +C Add\sthe\ssha3_agg()\saggregate\sto\sthe\sshathree\sextension. +D 2024-08-13T21:04:00.686 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -415,7 +415,7 @@ F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d385 F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 F ext/misc/series.c d96e5aac21658c6b5d54f918ac140460ec7197734c1a4fba806950831a7b1e7a F ext/misc/sha1.c 4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d -F ext/misc/shathree.c 543af7ce71d391cd3a9ab6924a6a1124efc63211fd0f2e240dc4b56077ba88ac +F ext/misc/shathree.c 24e890cb209272379f8d0823fb278d24bd3dca35d5ea565aa0b2794cd36d6197 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c c0aa7b80d6df45f7da59d912b38752bcac1af53a5766966160e6c5cdd397dbea F ext/misc/sqlar.c a6175790482328171da47095f87608b48a476d4fac78d8a9ff18b03a2454f634 @@ -2204,8 +2204,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d4014c87ba9b011a6a04c2bf85879b668dc762ebcbbfb50a2f8a417ce594ef88 -R 33ab7e360692d6ed2c1cd4d18565e8ac +P 9c10664416274df6f8da53ddd86f6356c9704ad798fd4034d2784ae433c1c056 +R 3605902adbab51926bad4083cbf76d2c U drh -Z 32b23835ef00f951c0107d2a77ff843c +Z fc4daf336fe8a5609f55ae8abc7ca9b2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ef73d7fbbd..6054597565 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9c10664416274df6f8da53ddd86f6356c9704ad798fd4034d2784ae433c1c056 +c4d7f9996ce4f5f6d08d83fc3abdb81b6825cfb556318f90c6896736a1da5175 From 8ff952773df34ee2fa4395e5e1fa8a36717354aa Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 Aug 2024 14:26:02 +0000 Subject: [PATCH 03/15] Fix an assert() in fts5 that might fail with an expression like "... NOT ''". FossilOrigin-Name: 317c91edfc69d1d47e233bb6ac13934241742fda1ba03a88fd51bdf98fcbc463 --- ext/fts5/fts5_expr.c | 7 ++++++- ext/fts5/test/fts5misc.test | 22 ++++++++++++++++++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index a39cc16bbf..1f089d8757 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -54,7 +54,7 @@ struct Fts5Expr { /* ** eType: -** Expression node type. Always one of: +** Expression node type. Usually one of: ** ** FTS5_AND (nChild, apChild valid) ** FTS5_OR (nChild, apChild valid) @@ -62,6 +62,10 @@ struct Fts5Expr { ** FTS5_STRING (pNear valid) ** FTS5_TERM (pNear valid) ** +** An expression node with eType==0 may also exist. It always matches zero +** rows. This is created when a phrase containing no tokens is parsed. +** e.g. "". +** ** iHeight: ** Distance from this node to furthest leaf. This is always 0 for nodes ** of type FTS5_STRING and FTS5_TERM. For all other nodes it is one @@ -3112,6 +3116,7 @@ static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ pNode->iRowid = iRowid; pNode->bEof = 0; switch( pNode->eType ){ + case 0: case FTS5_TERM: case FTS5_STRING: return (pNode->pNear->apPhrase[0]->poslist.n>0); diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test index e4903d067a..abd4fdaf89 100644 --- a/ext/fts5/test/fts5misc.test +++ b/ext/fts5/test/fts5misc.test @@ -21,6 +21,8 @@ ifcapable !fts5 { return } +if 0 { + do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a); } @@ -566,5 +568,25 @@ do_execsql_test 20.5 { SELECT rowid FROM x1 WHERE x1 MATCH 'z' OR (x1 MATCH 'a' AND x1 MATCH 'd'); } {3 1} +#------------------------------------------------------------------------- +} +reset_db +do_execsql_test 21.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none'); + INSERT INTO t1(a) VALUES('a'); +} + +do_execsql_test 21.2 { + SELECT rank FROM ( SELECT rank FROM t1('a NOT "" NOT def') ) ORDER BY 1; +} {-1e-06} + +do_execsql_test 21.3 { + SELECT rank FROM ( SELECT rank FROM t1('a NOT � NOT def') ) ORDER BY 1; +} {-1e-06} + +do_execsql_test 21.4 { + SELECT rank FROM ( SELECT rank FROM t1('a NOT "" NOT def') ); +} {-1e-06} + finish_test diff --git a/manifest b/manifest index 13c566c83d..d30fe768b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssha3_agg()\saggregate\sto\sthe\sshathree\sextension. -D 2024-08-13T21:04:00.686 +C Fix\san\sassert()\sin\sfts5\sthat\smight\sfail\swith\san\sexpression\slike\s"...\sNOT\s''". +D 2024-08-14T14:26:02.443 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -97,7 +97,7 @@ F ext/fts5/fts5Int.h 41fb3a2dd40e818cc96c6f4176dbdf2aaa8f57043cfc9a8f2676e7e6a72 F ext/fts5/fts5_aux.c 4584e88878e54828bf7d4d0d83deedd232ec60628b7731be02bad6adb62304b1 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 F ext/fts5/fts5_config.c 68cb87a49215f8e7028000b681df4057c430a4a6afbd676463886da94c9e1c37 -F ext/fts5/fts5_expr.c 3a24c6ab5b7545312a5ec03085ae705ede820a08f9a63f1d72829ed4a35da6f6 +F ext/fts5/fts5_expr.c 4b7734db98393d6f7fbc5c9c71ebcabe70110f7df08f6b136d096a1eaee0f56a F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe F ext/fts5/fts5_main.c 6ec7a7d005c632d86e510ddfaca56b197a5b20b61848415764b91bd27d1e4f84 @@ -188,7 +188,7 @@ F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fc F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 -F ext/fts5/test/fts5misc.test b88aa28ff20238b394495b0795cddca1a62b98fa09b99e462a8abc572d04ee88 +F ext/fts5/test/fts5misc.test b1682a40061bc58dcc62bbad48938fc5214d2ac6a868a8899c0c3d1930f1115d F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45 F ext/fts5/test/fts5near.test 33d60867581066e5db7016deb5d651628125d7ff4e0233a88175aa5b65874c74 @@ -2204,8 +2204,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9c10664416274df6f8da53ddd86f6356c9704ad798fd4034d2784ae433c1c056 -R 3605902adbab51926bad4083cbf76d2c -U drh -Z fc4daf336fe8a5609f55ae8abc7ca9b2 +P c4d7f9996ce4f5f6d08d83fc3abdb81b6825cfb556318f90c6896736a1da5175 +R 8f6bb730f0c3b63cb61a6b0c78a6a7c9 +U dan +Z 146061b654867fea67c05bd71b2b625a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6054597565..c281d62fd8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c4d7f9996ce4f5f6d08d83fc3abdb81b6825cfb556318f90c6896736a1da5175 +317c91edfc69d1d47e233bb6ac13934241742fda1ba03a88fd51bdf98fcbc463 From 78f9ba9774883d56ff6384c5173ff7e1ac42cb88 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 14 Aug 2024 17:58:46 +0000 Subject: [PATCH 04/15] Improved documentation for the sha3() SQL functions in the shathree.c extension in ext/misc. FossilOrigin-Name: 0ce609103eb35cfa26d9b6fa1769679be7e2b9032c37cc8c00a194fe355b06a4 --- ext/misc/shathree.c | 76 +++++++++++++++++++++++++++++++++++++++++++-- manifest | 14 ++++----- manifest.uuid | 2 +- 3 files changed, 82 insertions(+), 10 deletions(-) diff --git a/ext/misc/shathree.c b/ext/misc/shathree.c index 46a864af4e..5cb3198805 100644 --- a/ext/misc/shathree.c +++ b/ext/misc/shathree.c @@ -15,10 +15,20 @@ ** Two SQL functions are implemented: ** ** sha3(X,SIZE) -** sha3_query(Y,SIZE) +** sha3_agg(Y,SIZE) +** sha3_query(Z,SIZE) ** ** The sha3(X) function computes the SHA3 hash of the input X, or NULL if -** X is NULL. +** X is NULL. If inputs X is text, the UTF-8 rendering of that text is +** used to compute the hash. If X is a BLOB, then the binary data of the +** blob is used to compute the hash. If X is an integer or real number, +** then that number if converted into UTF-8 text and the hash is computed +** over the text. +** +** The sha3_agg(Y) function computes the SHA3 hash of all Y inputs. Since +** order is important for the hash, it is recommended that the Y expression +** by followed by an ORDER BY clause to guarantee that the inputs occur +** in the desired order. ** ** The sha3_query(Y) function evaluates all queries in the SQL statements of Y ** and returns a hash of their results. @@ -26,6 +36,68 @@ ** The SIZE argument is optional. If omitted, the SHA3-256 hash algorithm ** is used. If SIZE is included it must be one of the integers 224, 256, ** 384, or 512, to determine SHA3 hash variant that is computed. +** +** Because the sha3_agg() and sha3_query() functions compute a hash over +** multiple values, the values are encode to use include type information. +** +** In sha3_agg(), the sequence of bytes that gets hashed for each input +** Y depends on the datatype of Y: +** +** typeof(Y)='null' A single "N" is hashed. (One byte) +** +** typeof(Y)='integer' The data hash is the character "I" followed +** by an 8-byte big-endian binary of the +** 64-bit signed integer. (Nine bytes total.) +** +** typeof(Y)='real' The character "F" followed by an 8-byte +** big-ending binary of the double. (Nine +** bytes total.) +** +** typeof(Y)='text' The hash is over prefix "Tnnn:" followed +** by the UTF8 encoding of the text. The "nnn" +** in the prefix is the minimum-length decimal +** representation of the octet_length of the text. +** Notice the ":" at the end of the prefix, which +** is needed to separate the prefix from the +** content in cases where the content starts +** with a digit. +** +** typeof(Y)='blob' The hash is taken over prefix "Bnnn:" followed +** by the binary content of the blob. The "nnn" +** in the prefix is the mimimum-length decimal +** representation of the byte-length of the blob. +** +** According to the rules above, all of the following SELECT statements +** should return TRUE: +** +** SELECT sha3(1) = sha3('1'); +** +** SELECT sha3('hello') = sha3(x'68656c6c6f'); +** +** WITH a(x) AS (VALUES('xyzzy')) +** SELECT sha3_agg(x) = sha3('T5:xyzzy') FROM a; +** +** WITH a(x) AS (VALUES(x'010203')) +** SELECT sha3_agg(x) = sha3(x'42333a010203') FROM a; +** +** WITH a(x) AS (VALUES(0x123456)) +** SELECT sha3_agg(x) = sha3(x'490000000000123456') FROM a; +** +** WITH a(x) AS (VALUES(100.015625)) +** SELECT sha3_agg(x) = sha3(x'464059010000000000') FROM a; +** +** WITH a(x) AS (VALUES(NULL)) +** SELECT sha3_agg(x) = sha3('N') FROM a; +** +** +** In sha3_query(), individual column values are encoded as with +** sha3_agg(), but with the addition that a single "R" character is +** inserted at the start of each row. +** +** Note that sha3_agg() hashes rows for which Y is NULL. Add a FILTER +** clause if NULL rows should be excluded: +** +** SELECT sha3_agg(x ORDER BY rowid) FILTER(x NOT NULL) FROM t1; */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 diff --git a/manifest b/manifest index d30fe768b9..8ae446ea1a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassert()\sin\sfts5\sthat\smight\sfail\swith\san\sexpression\slike\s"...\sNOT\s''". -D 2024-08-14T14:26:02.443 +C Improved\sdocumentation\sfor\sthe\ssha3()\sSQL\sfunctions\sin\sthe\sshathree.c\nextension\sin\sext/misc. +D 2024-08-14T17:58:46.321 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -415,7 +415,7 @@ F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d385 F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 F ext/misc/series.c d96e5aac21658c6b5d54f918ac140460ec7197734c1a4fba806950831a7b1e7a F ext/misc/sha1.c 4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d -F ext/misc/shathree.c 24e890cb209272379f8d0823fb278d24bd3dca35d5ea565aa0b2794cd36d6197 +F ext/misc/shathree.c 811540cb7eb6f34ccf6ce6f6713c3896d38cbed20d46cb3c7c48d90e3251a7f4 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c c0aa7b80d6df45f7da59d912b38752bcac1af53a5766966160e6c5cdd397dbea F ext/misc/sqlar.c a6175790482328171da47095f87608b48a476d4fac78d8a9ff18b03a2454f634 @@ -2204,8 +2204,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c4d7f9996ce4f5f6d08d83fc3abdb81b6825cfb556318f90c6896736a1da5175 -R 8f6bb730f0c3b63cb61a6b0c78a6a7c9 -U dan -Z 146061b654867fea67c05bd71b2b625a +P 317c91edfc69d1d47e233bb6ac13934241742fda1ba03a88fd51bdf98fcbc463 +R 99e8cbe80cd01ac20616f63cdd8dbbda +U drh +Z c4d9af30409bcf77d9189a2bdac49004 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c281d62fd8..e1862fcd06 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -317c91edfc69d1d47e233bb6ac13934241742fda1ba03a88fd51bdf98fcbc463 +0ce609103eb35cfa26d9b6fa1769679be7e2b9032c37cc8c00a194fe355b06a4 From ff7200e75b360bddecfde77f22945925dc643808 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 14 Aug 2024 20:00:03 +0000 Subject: [PATCH 05/15] Doc typo correction reported in the forum. FossilOrigin-Name: 48900a867cc4b6d733d2a3533a553af3dfadf96d3ef7ddf4c72bd432e7bf0f76 --- ext/misc/shathree.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/misc/shathree.c b/ext/misc/shathree.c index 5cb3198805..72ca7278ca 100644 --- a/ext/misc/shathree.c +++ b/ext/misc/shathree.c @@ -97,7 +97,7 @@ ** Note that sha3_agg() hashes rows for which Y is NULL. Add a FILTER ** clause if NULL rows should be excluded: ** -** SELECT sha3_agg(x ORDER BY rowid) FILTER(x NOT NULL) FROM t1; +** SELECT sha3_agg(x ORDER BY rowid) FILTER(WHERE x NOT NULL) FROM t1; */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 diff --git a/manifest b/manifest index 8ae446ea1a..7f02d94ff2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sdocumentation\sfor\sthe\ssha3()\sSQL\sfunctions\sin\sthe\sshathree.c\nextension\sin\sext/misc. -D 2024-08-14T17:58:46.321 +C Doc\stypo\scorrection\sreported\sin\sthe\sforum. +D 2024-08-14T20:00:03.801 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -415,7 +415,7 @@ F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d385 F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 F ext/misc/series.c d96e5aac21658c6b5d54f918ac140460ec7197734c1a4fba806950831a7b1e7a F ext/misc/sha1.c 4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d -F ext/misc/shathree.c 811540cb7eb6f34ccf6ce6f6713c3896d38cbed20d46cb3c7c48d90e3251a7f4 +F ext/misc/shathree.c 1821d90a0040c9accdbe3e3527d378d30569475d758aa70f6848924c0b430e8c F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c c0aa7b80d6df45f7da59d912b38752bcac1af53a5766966160e6c5cdd397dbea F ext/misc/sqlar.c a6175790482328171da47095f87608b48a476d4fac78d8a9ff18b03a2454f634 @@ -2204,8 +2204,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 317c91edfc69d1d47e233bb6ac13934241742fda1ba03a88fd51bdf98fcbc463 -R 99e8cbe80cd01ac20616f63cdd8dbbda -U drh -Z c4d9af30409bcf77d9189a2bdac49004 +P 0ce609103eb35cfa26d9b6fa1769679be7e2b9032c37cc8c00a194fe355b06a4 +R 4c6d39820af9105b6fe92615708e9032 +U stephan +Z 73f13f9927a4c31a02bcb28fc941f7df # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e1862fcd06..cc7e9327e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0ce609103eb35cfa26d9b6fa1769679be7e2b9032c37cc8c00a194fe355b06a4 +48900a867cc4b6d733d2a3533a553af3dfadf96d3ef7ddf4c72bd432e7bf0f76 From 3f138e1d8f7bd97e66bffd41a59c435ffd3bd1d5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 15 Aug 2024 15:23:33 +0000 Subject: [PATCH 06/15] Reinstate the --disable-tcl option on ./configure. [forum:/forumpost/845d4a57becc7f3c|Forum thread 845d4a57becc7f3c]. FossilOrigin-Name: de927016aadd7ee55d947134e3540907a3ea5ab4015034c5d088e3a84905d98a --- configure | 138 +++++++++++++++++++++++++++----------------------- configure.ac | 108 ++++++++++++++++++++------------------- manifest | 16 +++--- manifest.uuid | 2 +- 4 files changed, 142 insertions(+), 122 deletions(-) diff --git a/configure b/configure index 2c57d6ac3a..43952440a8 100755 --- a/configure +++ b/configure @@ -888,6 +888,7 @@ enable_libtool_lock enable_largefile with_tclsh with_tcl +enable_tcl with_wasi_sdk enable_threadsafe enable_releasemode @@ -1543,6 +1544,7 @@ Optional Features: optimize for fast installation [default=yes] --disable-libtool-lock avoid locking (might break parallel builds) --disable-largefile omit support for large files + --disable-tcl do not build TCL extension --disable-threadsafe Disable mutexing --enable-releasemode Support libtool link to release mode --enable-tempstore Use an in-ram database for temporary tables @@ -3936,13 +3938,13 @@ if ${lt_cv_nm_interface+:} false; then : else lt_cv_nm_interface="BSD nm" echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:3939: $ac_compile\"" >&5) + (eval echo "\"\$as_me:3941: $ac_compile\"" >&5) (eval "$ac_compile" 2>conftest.err) cat conftest.err >&5 - (eval echo "\"\$as_me:3942: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval echo "\"\$as_me:3944: $NM \\\"conftest.$ac_objext\\\"\"" >&5) (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) cat conftest.err >&5 - (eval echo "\"\$as_me:3945: output\"" >&5) + (eval echo "\"\$as_me:3947: output\"" >&5) cat conftest.out >&5 if $GREP 'External.*some_variable' conftest.out > /dev/null; then lt_cv_nm_interface="MS dumpbin" @@ -5148,7 +5150,7 @@ ia64-*-hpux*) ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 5151 "configure"' > conftest.$ac_ext + echo '#line 5153 "configure"' > conftest.$ac_ext if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -6673,11 +6675,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:6676: $lt_compile\"" >&5) + (eval echo "\"\$as_me:6678: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:6680: \$? = $ac_status" >&5 + echo "$as_me:6682: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7012,11 +7014,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7015: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7017: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:7019: \$? = $ac_status" >&5 + echo "$as_me:7021: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7117,11 +7119,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7120: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7122: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7124: \$? = $ac_status" >&5 + echo "$as_me:7126: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -7172,11 +7174,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7175: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7177: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7179: \$? = $ac_status" >&5 + echo "$as_me:7181: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -9552,7 +9554,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9555 "configure" +#line 9557 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -9648,7 +9650,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9651 "configure" +#line 9653 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -10320,6 +10322,13 @@ if test "${with_tcl+set}" = set; then : withval=$with_tcl; fi +# Check whether --enable-tcl was given. +if test "${enable_tcl+set}" = set; then : + enableval=$enable_tcl; use_tcl=$enableval +else + use_tcl=yes +fi + if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then for ac_prog in tclsh8.6 tclsh tclsh9.0 do @@ -10370,60 +10379,69 @@ if test x"${with_tclsh}" != x -a x"${with_tclsh}" != xnone; then TCLSH_CMD=${with_tclsh} { $as_echo "$as_me:${as_lineno-$LINENO}: result: using tclsh at \"$TCLSH_CMD\"" >&5 $as_echo "using tclsh at \"$TCLSH_CMD\"" >&6; } - with_tcl=`${with_tclsh} <${srcdir}/tool/find_tclconfig.tcl` - if test x"${with_tcl}" != x; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}" >&5 + if test x"${use_tcl}" = "xyes"; then + with_tcl=`${with_tclsh} <${srcdir}/tool/find_tclconfig.tcl` + if test x"${with_tcl}" != x; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}" >&5 $as_echo "$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}" >&6; } - else - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&5 + else + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&5 $as_echo "$as_me: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&2;} + fi fi fi -if test x"${with_tcl}" != x; then - if test -r ${with_tcl}/tclConfig.sh; then - tclconfig="${with_tcl}/tclConfig.sh" +if test x"${use_tcl}" = "xyes"; then + if test x"${with_tcl}" != x; then + if test -r ${with_tcl}/tclConfig.sh; then + tclconfig="${with_tcl}/tclConfig.sh" + else + for i in tcl8.6 tcl9.0 lib; do + if test -r ${with_tcl}/$i/tclConfig.sh; then + tclconfig=${with_tcl}/$i/tclConfig.sh + break + fi + done + fi + if test ! -r "${tclconfig}"; then + as_fn_error $? "no tclConfig.sh file found under ${with_tcl}" "$LINENO" 5 + fi else - for i in tcl8.6 tcl9.0 lib; do - if test -r ${with_tcl}/$i/tclConfig.sh; then - tclconfig=${with_tcl}/$i/tclConfig.sh - break - fi - done + # If we have not yet found a tclConfig.sh file, look in $libdir whic is + # set automatically by autoconf or by the --prefix command-line option. + # See https://sqlite.org/forum/forumpost/e04e693439a22457 + libdir=${prefix}/lib + if test -r ${libdir}/tclConfig.sh; then + tclconfig=${libdir}/tclConfig.sh + else + for i in tcl8.6 tcl9.0 lib; do + if test -r ${libdir}/$i/tclConfig.sh; then + tclconfig=${libdir}/$i/tclConfig.sh + break + fi + done + fi + if test ! -r "${tclconfig}"; then + as_fn_error $? "cannot find a usable tclConfig.sh file. + Use --with-tcl=DIR to specify a directory where tclConfig.sh can be found. + SQLite does not use TCL internally, but TCL is required to build SQLite + from canonical sources and TCL is required for testing." "$LINENO" 5 + fi fi - if test ! -r "${tclconfig}"; then - as_fn_error $? "no tclConfig.sh file found under ${with_tcl}" "$LINENO" 5 - fi -else - # If we have not yet found a tclConfig.sh file, look in $libdir whic is - # set automatically by autoconf or by the --prefix command-line option. - # See https://sqlite.org/forum/forumpost/e04e693439a22457 - libdir=${prefix}/lib - if test -r ${libdir}/tclConfig.sh; then - tclconfig=${libdir}/tclConfig.sh - else - for i in tcl8.6 tcl9.0 lib; do - if test -r ${libdir}/$i/tclConfig.sh; then - tclconfig=${libdir}/$i/tclConfig.sh - break - fi - done - fi - if test ! -r "${tclconfig}"; then - as_fn_error $? "cannot find a usable tclConfig.sh file. - Use --with-tcl=DIR to specify a directory where tclConfig.sh can be found. - SQLite does not use TCL internally, but TCL is required to build SQLite - from canonical sources and TCL is required for testing." "$LINENO" 5 - fi -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: loading TCL configuration from ${tclconfig}" >&5 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: loading TCL configuration from ${tclconfig}" >&5 $as_echo "loading TCL configuration from ${tclconfig}" >&6; } -. ${tclconfig} + . ${tclconfig} -# There are lots of other configuration variables that are provided by the -# tclConfig.sh file and that could be included here. But as of right now, -# TCL_LIB_SPEC is the only what that the Makefile uses. + # There are lots of other configuration variables that are provided by the + # tclConfig.sh file and that could be included here. But as of right now, + # TCL_LIB_SPEC is the only what that the Makefile uses. + HAVE_TCL=1 +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: unable to run tests because of --disable-tcl" >&5 +$as_echo "unable to run tests because of --disable-tcl" >&6; } + HAVE_TCL=0 +fi if test x"$TCLSH_CMD" == x; then TCLSH_CMD=${TCL_EXEC_PREFIX}/bin/tclsh${TCL_VERSION} @@ -10445,14 +10463,10 @@ if test "$TCLSH_CMD" = "none"; then $as_echo "$as_me: WARNING: Warning: can't find tclsh - defaulting to non-amalgamation build." >&2;} USE_AMALGAMATION=0 TCLSH_CMD="tclsh" - HAVE_TCL=0 -else - HAVE_TCL=1 fi - if test "x${TCLLIBDIR+set}" != "xset" ; then for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` ; do if test -d $i ; then diff --git a/configure.ac b/configure.ac index 095df5e4f6..88a8654fc0 100644 --- a/configure.ac +++ b/configure.ac @@ -121,6 +121,8 @@ USE_AMALGAMATION=1 # AC_ARG_WITH(tclsh, AS_HELP_STRING([--with-tclsh=PATHNAME],[full pathname of a tclsh to use])) AC_ARG_WITH(tcl, AS_HELP_STRING([--with-tcl=DIR],[directory containing (tclConfig.sh)])) +AC_ARG_ENABLE(tcl, AS_HELP_STRING([--disable-tcl],[do not build TCL extension]), + [use_tcl=$enableval],[use_tcl=yes]) if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.6 tclsh tclsh9.0],none) with_tclsh=${TCLSH_CMD} @@ -128,58 +130,66 @@ fi if test x"${with_tclsh}" != x -a x"${with_tclsh}" != xnone; then TCLSH_CMD=${with_tclsh} AC_MSG_RESULT([using tclsh at "$TCLSH_CMD"]) - with_tcl=`${with_tclsh} <${srcdir}/tool/find_tclconfig.tcl` + if test x"${use_tcl}" = "xyes"; then + with_tcl=`${with_tclsh} <${srcdir}/tool/find_tclconfig.tcl` + if test x"${with_tcl}" != x; then + AC_MSG_RESULT([$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}]) + else + AC_MSG_WARN([$TCLSH_CMD is unable to recommend a tclConfig.sh]) + fi + fi +fi +if test x"${use_tcl}" = "xyes"; then if test x"${with_tcl}" != x; then - AC_MSG_RESULT([$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}]) + if test -r ${with_tcl}/tclConfig.sh; then + tclconfig="${with_tcl}/tclConfig.sh" + else + for i in tcl8.6 tcl9.0 lib; do + if test -r ${with_tcl}/$i/tclConfig.sh; then + tclconfig=${with_tcl}/$i/tclConfig.sh + break + fi + done + fi + if test ! -r "${tclconfig}"; then + AC_MSG_ERROR([no tclConfig.sh file found under ${with_tcl}]) + fi else - AC_MSG_WARN([$TCLSH_CMD is unable to recommend a tclConfig.sh]) - fi -fi -if test x"${with_tcl}" != x; then - if test -r ${with_tcl}/tclConfig.sh; then - tclconfig="${with_tcl}/tclConfig.sh" - else - for i in tcl8.6 tcl9.0 lib; do - if test -r ${with_tcl}/$i/tclConfig.sh; then - tclconfig=${with_tcl}/$i/tclConfig.sh - break - fi - done - fi - if test ! -r "${tclconfig}"; then - AC_MSG_ERROR([no tclConfig.sh file found under ${with_tcl}]) + # If we have not yet found a tclConfig.sh file, look in $libdir whic is + # set automatically by autoconf or by the --prefix command-line option. + # See https://sqlite.org/forum/forumpost/e04e693439a22457 + libdir=${prefix}/lib + if test -r ${libdir}/tclConfig.sh; then + tclconfig=${libdir}/tclConfig.sh + else + for i in tcl8.6 tcl9.0 lib; do + if test -r ${libdir}/$i/tclConfig.sh; then + tclconfig=${libdir}/$i/tclConfig.sh + break + fi + done + fi + if test ! -r "${tclconfig}"; then + AC_MSG_ERROR([cannot find a usable tclConfig.sh file. + Use --with-tcl=DIR to specify a directory where tclConfig.sh can be found. + SQLite does not use TCL internally, but TCL is required to build SQLite + from canonical sources and TCL is required for testing.]) + fi fi + AC_MSG_RESULT([loading TCL configuration from ${tclconfig}]) + . ${tclconfig} + AC_SUBST(TCL_INCLUDE_SPEC) + AC_SUBST(TCL_LIB_SPEC) + AC_SUBST(TCL_STUB_LIB_SPEC) + # There are lots of other configuration variables that are provided by the + # tclConfig.sh file and that could be included here. But as of right now, + # TCL_LIB_SPEC is the only what that the Makefile uses. + HAVE_TCL=1 else - # If we have not yet found a tclConfig.sh file, look in $libdir whic is - # set automatically by autoconf or by the --prefix command-line option. - # See https://sqlite.org/forum/forumpost/e04e693439a22457 - libdir=${prefix}/lib - if test -r ${libdir}/tclConfig.sh; then - tclconfig=${libdir}/tclConfig.sh - else - for i in tcl8.6 tcl9.0 lib; do - if test -r ${libdir}/$i/tclConfig.sh; then - tclconfig=${libdir}/$i/tclConfig.sh - break - fi - done - fi - if test ! -r "${tclconfig}"; then - AC_MSG_ERROR([cannot find a usable tclConfig.sh file. - Use --with-tcl=DIR to specify a directory where tclConfig.sh can be found. - SQLite does not use TCL internally, but TCL is required to build SQLite - from canonical sources and TCL is required for testing.]) - fi + AC_MSG_RESULT([unable to run tests because of --disable-tcl]) + HAVE_TCL=0 fi -AC_MSG_RESULT([loading TCL configuration from ${tclconfig}]) -. ${tclconfig} -AC_SUBST(TCL_INCLUDE_SPEC) -AC_SUBST(TCL_LIB_SPEC) -AC_SUBST(TCL_STUB_LIB_SPEC) -# There are lots of other configuration variables that are provided by the -# tclConfig.sh file and that could be included here. But as of right now, -# TCL_LIB_SPEC is the only what that the Makefile uses. - +AC_SUBST(HAVE_TCL) if test x"$TCLSH_CMD" == x; then TCLSH_CMD=${TCL_EXEC_PREFIX}/bin/tclsh${TCL_VERSION} if test ! -x ${TCLSH_CMD}; then @@ -198,12 +208,8 @@ if test "$TCLSH_CMD" = "none"; then AC_MSG_WARN([Warning: can't find tclsh - defaulting to non-amalgamation build.]) USE_AMALGAMATION=0 TCLSH_CMD="tclsh" - HAVE_TCL=0 -else - HAVE_TCL=1 fi AC_SUBST(TCLSH_CMD) -AC_SUBST(HAVE_TCL) AC_ARG_VAR([TCLLIBDIR], [Where to install tcl plugin]) if test "x${TCLLIBDIR+set}" != "xset" ; then diff --git a/manifest b/manifest index 7f02d94ff2..e5f56697c4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Doc\stypo\scorrection\sreported\sin\sthe\sforum. -D 2024-08-14T20:00:03.801 +C Reinstate\sthe\s--disable-tcl\soption\son\s./configure.\n[forum:/forumpost/845d4a57becc7f3c|Forum\sthread\s845d4a57becc7f3c]. +D 2024-08-15T15:23:33.404 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -35,8 +35,8 @@ F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea0034 F autoconf/tea/win/rules.vc 7b3bb2ef32ade0f3f14d951231811678722725e3bca240dd9727ae0dfe10f6a5 F config.guess 883205ddf25b46f10c181818bf42c09da9888884af96f79e1719264345053bd6 F config.sub c2d0260f17f3e4bc0b6808fccf1b291cb5e9126c14fc5890efc77b9fd0175559 -F configure 8c74221e3e3af939270c27f0600281a5485ac2e5014dd7581b056e24a528d625 x -F configure.ac 9a3d92b38ad56456d09c1272f83131794f73110437f0f53fb94c90ce0d7a351a +F configure 6255dfc962e2e5ed0a16ccc5aa39444f693692721ffe1d365ba091d8fe21c7e2 x +F configure.ac bce81466694af143776724dd483cbc9e1f1288e41d2d0f46e56965be3deacfc5 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd F doc/compile-for-windows.md e8635eea9153dcd6a51fd2740666ebc4492b3813cb1ac31cd8e99150df91762d @@ -2204,8 +2204,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0ce609103eb35cfa26d9b6fa1769679be7e2b9032c37cc8c00a194fe355b06a4 -R 4c6d39820af9105b6fe92615708e9032 -U stephan -Z 73f13f9927a4c31a02bcb28fc941f7df +P 48900a867cc4b6d733d2a3533a553af3dfadf96d3ef7ddf4c72bd432e7bf0f76 +R 13d05cfcaef8f5c3cbdcd475a1199700 +U drh +Z 4a60d69549ab00c925ad79ae1e45201c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cc7e9327e6..ac4f295436 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -48900a867cc4b6d733d2a3533a553af3dfadf96d3ef7ddf4c72bd432e7bf0f76 +de927016aadd7ee55d947134e3540907a3ea5ab4015034c5d088e3a84905d98a From 7c5a829025914550de138aab2a8cf3f8090b7c3e Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 15 Aug 2024 15:31:46 +0000 Subject: [PATCH 07/15] Improved help message for --disable-tcl on the configure script. FossilOrigin-Name: 96e6cfb211f7f1aab50f997f4bc61dfb9701beb720c59bc413a4285c1ae5e20d --- configure | 31 ++++++++++++++++--------------- configure.ac | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/configure b/configure index 43952440a8..50d9e2b3d0 100755 --- a/configure +++ b/configure @@ -1544,7 +1544,8 @@ Optional Features: optimize for fast installation [default=yes] --disable-libtool-lock avoid locking (might break parallel builds) --disable-largefile omit support for large files - --disable-tcl do not build TCL extension + --disable-tcl omit building accessory programs that require + TCL-dev --disable-threadsafe Disable mutexing --enable-releasemode Support libtool link to release mode --enable-tempstore Use an in-ram database for temporary tables @@ -3938,13 +3939,13 @@ if ${lt_cv_nm_interface+:} false; then : else lt_cv_nm_interface="BSD nm" echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:3941: $ac_compile\"" >&5) + (eval echo "\"\$as_me:3942: $ac_compile\"" >&5) (eval "$ac_compile" 2>conftest.err) cat conftest.err >&5 - (eval echo "\"\$as_me:3944: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval echo "\"\$as_me:3945: $NM \\\"conftest.$ac_objext\\\"\"" >&5) (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) cat conftest.err >&5 - (eval echo "\"\$as_me:3947: output\"" >&5) + (eval echo "\"\$as_me:3948: output\"" >&5) cat conftest.out >&5 if $GREP 'External.*some_variable' conftest.out > /dev/null; then lt_cv_nm_interface="MS dumpbin" @@ -5150,7 +5151,7 @@ ia64-*-hpux*) ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 5153 "configure"' > conftest.$ac_ext + echo '#line 5154 "configure"' > conftest.$ac_ext if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -6675,11 +6676,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:6678: $lt_compile\"" >&5) + (eval echo "\"\$as_me:6679: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:6682: \$? = $ac_status" >&5 + echo "$as_me:6683: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7014,11 +7015,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7017: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7018: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:7021: \$? = $ac_status" >&5 + echo "$as_me:7022: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7119,11 +7120,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7122: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7123: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7126: \$? = $ac_status" >&5 + echo "$as_me:7127: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -7174,11 +7175,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7177: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7178: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7181: \$? = $ac_status" >&5 + echo "$as_me:7182: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -9554,7 +9555,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9557 "configure" +#line 9558 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -9650,7 +9651,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9653 "configure" +#line 9654 "configure" #include "confdefs.h" #if HAVE_DLFCN_H diff --git a/configure.ac b/configure.ac index 88a8654fc0..4f95a5c08c 100644 --- a/configure.ac +++ b/configure.ac @@ -121,7 +121,7 @@ USE_AMALGAMATION=1 # AC_ARG_WITH(tclsh, AS_HELP_STRING([--with-tclsh=PATHNAME],[full pathname of a tclsh to use])) AC_ARG_WITH(tcl, AS_HELP_STRING([--with-tcl=DIR],[directory containing (tclConfig.sh)])) -AC_ARG_ENABLE(tcl, AS_HELP_STRING([--disable-tcl],[do not build TCL extension]), +AC_ARG_ENABLE(tcl, AS_HELP_STRING([--disable-tcl],[omit building accessory programs that require TCL-dev]), [use_tcl=$enableval],[use_tcl=yes]) if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.6 tclsh tclsh9.0],none) diff --git a/manifest b/manifest index e5f56697c4..22a05bc11b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reinstate\sthe\s--disable-tcl\soption\son\s./configure.\n[forum:/forumpost/845d4a57becc7f3c|Forum\sthread\s845d4a57becc7f3c]. -D 2024-08-15T15:23:33.404 +C Improved\shelp\smessage\sfor\s--disable-tcl\son\sthe\sconfigure\sscript. +D 2024-08-15T15:31:46.143 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -35,8 +35,8 @@ F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea0034 F autoconf/tea/win/rules.vc 7b3bb2ef32ade0f3f14d951231811678722725e3bca240dd9727ae0dfe10f6a5 F config.guess 883205ddf25b46f10c181818bf42c09da9888884af96f79e1719264345053bd6 F config.sub c2d0260f17f3e4bc0b6808fccf1b291cb5e9126c14fc5890efc77b9fd0175559 -F configure 6255dfc962e2e5ed0a16ccc5aa39444f693692721ffe1d365ba091d8fe21c7e2 x -F configure.ac bce81466694af143776724dd483cbc9e1f1288e41d2d0f46e56965be3deacfc5 +F configure 2e5058c2ba2c6b50a01cf50c158180c4ec969b2068c2e361dfbfc875ec3f9d1c x +F configure.ac b7e26a699a2ffc5eb7a25636f32755c27bbb665777b68ef35a90c1ee9bd9d469 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd F doc/compile-for-windows.md e8635eea9153dcd6a51fd2740666ebc4492b3813cb1ac31cd8e99150df91762d @@ -2204,8 +2204,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 48900a867cc4b6d733d2a3533a553af3dfadf96d3ef7ddf4c72bd432e7bf0f76 -R 13d05cfcaef8f5c3cbdcd475a1199700 +P de927016aadd7ee55d947134e3540907a3ea5ab4015034c5d088e3a84905d98a +R fa965b3cf027827ab805def1d12c22e4 U drh -Z 4a60d69549ab00c925ad79ae1e45201c +Z a05ca4fe471b9ed0243c4a5725784485 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ac4f295436..dd4cda6e20 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -de927016aadd7ee55d947134e3540907a3ea5ab4015034c5d088e3a84905d98a +96e6cfb211f7f1aab50f997f4bc61dfb9701beb720c59bc413a4285c1ae5e20d From 235b5d0ac57950437262c40bb92624670dd02a90 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 15 Aug 2024 23:38:52 +0000 Subject: [PATCH 08/15] If a subquery is materialized due to an ORDER BY and that ordering is useful in helping to satisfy the ORDER BY or GROUP BY in the order query without doing an extra sort, then omit the extra sort. FossilOrigin-Name: 2fbb4dc2327ee435cb2b7a4adcddf5a9cee6dff7de96e2ecb761166427b5ddea --- manifest | 29 +++++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 1 + src/test1.c | 1 + src/where.c | 79 +++++++++++++++++++++++++++++++++++++++++-- src/whereInt.h | 4 ++- test/eqp.test | 53 +++++++++++++++++++++++++++++ test/skipscan1.test | 4 +-- test/windowpushd.test | 6 +++- test/with6.test | 44 ++++++++++++++++++++++++ 10 files changed, 203 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 22a05bc11b..e19579238d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\shelp\smessage\sfor\s--disable-tcl\son\sthe\sconfigure\sscript. -D 2024-08-15T15:31:46.143 +C If\sa\ssubquery\sis\smaterialized\sdue\sto\san\sORDER\sBY\sand\sthat\sordering\sis\suseful\nin\shelping\sto\ssatisfy\sthe\sORDER\sBY\sor\sGROUP\sBY\sin\sthe\sorder\squery\swithout\ndoing\san\sextra\ssort,\sthen\somit\sthe\sextra\ssort. +D 2024-08-15T23:38:52.210 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -765,13 +765,13 @@ F src/shell.c.in 94571558b0fb28c37a5cf6dbd6ea27285341023a28a8cb5795cd2768fab6770 F src/sqlite.h.in 1ad9110150773c38ebababbad11b5cb361bcd3997676dec1c91ac5e0416a7b86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h e3f3b3d80a666a7c5c85b4db102d41ca831c5624f0b3001814479d376f00c19d +F src/sqliteInt.h 128b9004698cc79993d5369d7d1763deaf8bbf26e5e8931ec540707e5a7238df F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c c6888598f08dee3d9112a38ef42c8f5c89ca7f3190f4694744d0b84250f4bf8c F src/tclsqlite.h c6af51f31a2b2172d674608763a4b98fdf5cd587e4025053e546fb8077757262 -F src/test1.c be8cc208c0d50b3a7e570049e55f25ae40c1dfec8165b7ce12c2c8ed9f5b3030 +F src/test1.c 3f18399557d954bc85f4564aec8ea1777d2161a81d98a3ff6c9e9046bf3554c1 F src/test2.c 7ebc518e6735939d8979273a6f7b1d9b5702babf059f6ad62499f7f60a9eb9a3 F src/test3.c e7573aa0f78ee4e070a4bc8c3493941c1aa64d5c66d4825c74c0f055451f432b F src/test4.c 13e57ae7ec7a959ee180970aef09deed141252fe9bb07c61054f0dfa4f1dfd5d @@ -846,8 +846,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c 3556ef464ac78f4730f40c085aa0b6729ffe44306c0daaaffe9370c981a66d1d -F src/whereInt.h 002adc3aa2cc10733b9b27958fdbe893987cd989fab25a9853941c1f9b9b0a65 +F src/where.c c59dffd249e5daa335bc43c2b87ffacc25dce995e3600a2b7bcdbcc55a68c985 +F src/whereInt.h 6444b888ce395cb80511284b8a73b63472d34247fcb1b125ee06a54fa6ae878e F src/wherecode.c c9cac0b0b8e809c5e7e79d7796918907fb685ad99be2aaa9737f9787aa47349c F src/whereexpr.c 7d0d34b42b9edfd8e8ca66beb3a6ef63fe211c001af54caf2ccbcd989b783290 F src/window.c 1e40ffc509bae21e466f6106382d238e91eb73edd4ba10e66ca4fd7af2b96896 @@ -1113,7 +1113,7 @@ F test/enc.test 9a7be5479da985381d740b15f432800f65e2c87029ee57a318f42cb2eb43763a F test/enc2.test 848bf05f15b011719f478dddb7b5e9aea35e39e457493cba4c4eef75d849a5ec F test/enc3.test 55ef64416d72975c66167310a51dc9fc544ba3ae4858b8d5ab22f4cb6500b087 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test 815418b69f6be3a27037b1736c54699c72cc3e2e6b0bc878c01464d1dcec65fe +F test/eqp.test 82f221e8cd588434d7f3bba9a0f4c78cbe7a541615a41632e12f50608bfb4a99 F test/eqp2.test 6e8996148de88f0e7670491e92e712a2920a369b4406f21a27c3c9b6a46b68dd F test/errmsg.test eae9f091eb39ce7e20305de45d8e5d115b68fa856fba4ea6757b6ca3705ff7f9 F test/eval.test 73969a2d43a511bf44080c44485a8c4d796b6a4f038d19e491867081155692c0 @@ -1632,7 +1632,7 @@ F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 2668e607dcdfa19c52828c09b69685b38da793856582ae31debf79d90c7bbbdc F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 -F test/skipscan1.test e03ba5b977da6fd71662a4b0a668f04053bda4b187ff3214db7533e28c732279 +F test/skipscan1.test 9cbbb6575517b15292bd87ee85b853bbd3cd4b4735d69b0f083020cec16ff304 F test/skipscan2.test b032ed3e0ba5caa4df6c43ef22c31566aac67783bc031869155989a7ccdb5bd5 F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 0672103fd2c8f96bd114133f356192b35ece45c794fe3677e1d9e5e3104a608e @@ -2067,13 +2067,13 @@ F test/windowE.test 6ba0c8048e4cc02b942e56640f8fcd50fd7ca72c876656c40f6baf42e316 F test/windowerr.tcl f5acd6fbc210d7b5546c0e879d157888455cd4a17a1d3f28f07c1c8a387019e0 F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3cc1819b F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c31660a7c -F test/windowpushd.test d8895d08870b7226f7693665bd292eb177e62ca06799184957b3ca7dc03067df +F test/windowpushd.test c420e2265f0e09a0e798d0513a660d71b51602088d81b3dbd038918ee1339dcc F test/with1.test b93833890e5d2a368e78747f124503a0159aa029b98e9ed4795ebf630b2efd3d F test/with2.test a1df41b987198383b9b70bf5e5fda390582e46398653858dbc6ceb24253b28df F test/with3.test e30369ea27aa27eb1bda4c5e510c8a9f782c8afd2ab99d1a02b8a7f25a5d3e65 F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 F test/with5.test 6248213c41fab36290b5b73aa3f937309dfba337004d9d8434c3fabc8c7d4be8 -F test/with6.test e097a03e5c898a8cd8f3a2d6a994ec510ea4376b5d484c2b669a41001e7758c8 +F test/with6.test 9ff3503c3ff7cd459dc4852a02aaefa998dccace53f4142a0eb726174ad5984a F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64 F test/without_rowid1.test a5210b8770dc4736bca4e74bc96588f43025ad03ad6a80f885afd36d9890e217 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -2204,8 +2204,11 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P de927016aadd7ee55d947134e3540907a3ea5ab4015034c5d088e3a84905d98a -R fa965b3cf027827ab805def1d12c22e4 +P 96e6cfb211f7f1aab50f997f4bc61dfb9701beb720c59bc413a4285c1ae5e20d +R cc9a8ceafd2a9a7a7cbd542af1820055 +T *branch * order-by-subquery +T *sym-order-by-subquery * +T -sym-trunk * U drh -Z a05ca4fe471b9ed0243c4a5725784485 +Z 66dcbf2773ae39411ce78a1d9346e7ac # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dd4cda6e20..8f936c703f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96e6cfb211f7f1aab50f997f4bc61dfb9701beb720c59bc413a4285c1ae5e20d +2fbb4dc2327ee435cb2b7a4adcddf5a9cee6dff7de96e2ecb761166427b5ddea diff --git a/src/sqliteInt.h b/src/sqliteInt.h index dca5bb150d..ebd0f01b26 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1927,6 +1927,7 @@ struct sqlite3 { #define SQLITE_Coroutines 0x02000000 /* Co-routines for subqueries */ #define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */ #define SQLITE_OnePass 0x08000000 /* Single-pass DELETE and UPDATE */ +#define SQLITE_OrderBySubq 0x10000000 /* ORDER BY in subquery helps outer */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* diff --git a/src/test1.c b/src/test1.c index 9def739f4c..177091a0e1 100644 --- a/src/test1.c +++ b/src/test1.c @@ -8100,6 +8100,7 @@ static int SQLITE_TCLAPI optimization_control( { "distinct-opt", SQLITE_DistinctOpt }, { "cover-idx-scan", SQLITE_CoverIdxScan }, { "order-by-idx-join", SQLITE_OrderByIdxJoin }, + { "order-by-subquery", SQLITE_OrderBySubq }, { "transitive", SQLITE_Transitive }, { "omit-noop-join", SQLITE_OmitNoopJoin }, { "stat4", SQLITE_Stat4 }, diff --git a/src/where.c b/src/where.c index 98e9f117f9..b119301e13 100644 --- a/src/where.c +++ b/src/where.c @@ -4010,6 +4010,10 @@ static int whereLoopAddBtree( #endif ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew, rSize); + if( pSrc->pSelect ){ + if( pSrc->fg.viaCoroutine ) pNew->wsFlags |= WHERE_COROUTINE; + pNew->u.btree.pOrderBy = pSrc->pSelect->pOrderBy; + } rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; if( rc ) break; @@ -4843,6 +4847,69 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ return rc; } +/* +** WhereLoop pLoop, which the iLoop-th term of the nested loop, is really +** a subquery or CTE that has an ORDER BY clause. See if any of the terms +** in the subquery ORDER BY clause will satisfy pOrderBy from the outer +** query. Mark off all satisfied terms (by setting bits in *pOBSat) and +** return TRUE if they do. If not, return false. +** +** Example: +** +** CREATE TABLE t1(a,b,c, PRIMARY KEY(a,b)); +** CREATE TABLE t2(x,y); +** WITH t3(p,q) AS MATERIALIZED (SELECT x+y, x-y FROM t2 ORDER BY x+y) +** SELECT * FROM t3 JOIN t1 ON a=q ORDER BY p, b; +** +** The CTE named "t3" comes out in the natural order of "p", so the first +** first them of "ORDER BY p,b" is satisfied by a sequential scan of "t3". +** +*/ +static SQLITE_NOINLINE int wherePathMatchSubqueryOB( + WhereLoop *pLoop, /* The nested loop term that is a subquery */ + int iLoop, /* Which level of the nested loop. 0==outermost */ + int iCur, /* Cursor used by the this loop */ + int wctrlFlags, /* Flags determining sort behavior */ + ExprList *pOrderBy, /* The ORDER BY clause on the whole query */ + Bitmask *pRevMask, /* When loops need to go in reverse order */ + Bitmask *pOBSat /* Which terms of pOrderBy are satisfied so far */ +){ + int i, j; + u8 rev = 0; + u8 revIdx = 0; + Expr *pOBExpr; + ExprList *pSubOB = pLoop->u.btree.pOrderBy; + assert( pSubOB!=0 ); + for(i=0; (MASKBIT(i) & *pOBSat)!=0; i++){} + for(j=0; jnExpr && inExpr; j++, i++){ + if( pSubOB->a[j].u.x.iOrderByCol==0 ) break; + pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[j].pExpr); + if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) break; + if( pOBExpr->iTable!=iCur ) break; + if( pOBExpr->iColumn!=pSubOB->a[j].u.x.iOrderByCol-1 ) break; + if( pSubOB->a[j].fg.sortFlags & KEYINFO_ORDER_BIGNULL ) break; + revIdx = pSubOB->a[j].fg.sortFlags & KEYINFO_ORDER_DESC; + if( wctrlFlags & WHERE_GROUPBY ){ + /* Sort order does not matter for GROUP BY */ + }else if( j>0 ){ + if( (rev ^ revIdx) != (pOrderBy->a[i].fg.sortFlags&KEYINFO_ORDER_DESC) ){ + break; + } + }else{ + rev = revIdx ^ (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC); + if( rev ){ + if( (pLoop->wsFlags & WHERE_COROUTINE)!=0 ){ + /* Cannot run a co-routine in reverse order */ + break; + } + *pRevMask |= MASKBIT(iLoop); + } + } + *pOBSat |= MASKBIT(i); + } + return j>0; +} + /* ** Examine a WherePath (with the addition of the extra WhereLoop of the 6th ** parameters) to see if it outputs rows in the requested ORDER BY @@ -4988,9 +5055,17 @@ static i8 wherePathSatisfiesOrderBy( if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){ if( pLoop->wsFlags & WHERE_IPK ){ + if( pLoop->u.btree.pOrderBy + && OptimizationEnabled(db, SQLITE_OrderBySubq) + && wherePathMatchSubqueryOB(pLoop,iLoop,iCur,wctrlFlags, + pOrderBy,pRevMask, &obSat) + ){ + nColumn = 0; + }else{ + nColumn = 1; + } pIndex = 0; nKeyCol = 0; - nColumn = 1; }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ return 0; }else{ @@ -5085,7 +5160,7 @@ static i8 wherePathSatisfiesOrderBy( } /* Find the ORDER BY term that corresponds to the j-th column - ** of the index and mark that ORDER BY term off + ** of the index and mark that ORDER BY term having been satisfied. */ isMatch = 0; for(i=0; bOnce && i 0 AND nFin > 0; +} { + QUERY PLAN + |--MATERIALIZE sums + | |--MATERIALIZE src + | | |--MATERIALIZE init + | | | `--SCAN raw USING INDEX sqlite_autoindex_raw_1 + | | |--SCAN i + | | |--SEARCH raw USING COVERING INDEX sqlite_autoindex_raw_1 (country=? AND date>?) + | | `--USE TEMP B-TREE FOR ORDER BY + | |--SCAN src + | `--SEARCH raw USING INDEX sqlite_autoindex_raw_1 (country=? AND date>? AND date 0 GROUP BY country), + src(country, date) AS (SELECT raw.country, raw.date + FROM raw JOIN init i on raw.country = i.country AND raw.date > i.date + ORDER BY raw.country, raw.date), + vals(country, date, x, y) AS (SELECT src.country, src.date, julianday(raw.date) - julianday(src.date), log(delta+1) + FROM src JOIN raw on raw.country = src.country AND raw.date > date(src.date,'-7 days') AND raw.date <= src.date AND delta >= 0), + sums(country, date, x2, x, n, xy, y) AS (SELECT country, date, sum(x*x*1.0), sum(x*1.0), sum(1.0), sum(x*y*1.0), sum(y*1.0) FROM vals GROUP BY 1, 2), + mult(country, date, m) AS (SELECT country, date, 1.0/(x2 * n - x * x) FROM sums), + inv(country, date, a,b,c,d) AS (SELECT mult.country, mult.date, n * m, -x * m, -x * m, x2 * m + FROM mult JOIN sums on sums.country=mult.country AND mult.date=sums.date), + fit(country, date, a, b) AS (SELECT inv.country, inv.date, a * xy + b * y, c * xy + d * y + FROM inv + JOIN mult on mult.country = inv.country AND mult.date = inv.date + JOIN sums on sums.country = mult.country AND sums.date = mult.date + ) + SELECT *, nFin/nPrev - 1 AS growth, log(2)/log(nFin/nPrev) AS doubling + FROM (SELECT f.*, exp(b) - 1 AS nFin, exp(a* (-1) + b) - 1 AS nPrev + FROM fit f JOIN init i on i.country = f.country AND f.date <= date(i.fin,'-3 days')) + WHERE nPrev > 0 AND nFin > 0; } { QUERY PLAN |--MATERIALIZE sums From 743188642de006de77e282cac6af4d3fc1baa293 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 16 Aug 2024 00:37:54 +0000 Subject: [PATCH 09/15] Improvements to the subquery ORDER BY decision algorithm. Fewer false-negatives. FossilOrigin-Name: b82421e3f5811946e9d60b845fc882b6ea5c53c648695654c6900754427bf9bf --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/where.c | 27 +++++++++++++++------------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index e19579238d..ce77ec7858 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\ssubquery\sis\smaterialized\sdue\sto\san\sORDER\sBY\sand\sthat\sordering\sis\suseful\nin\shelping\sto\ssatisfy\sthe\sORDER\sBY\sor\sGROUP\sBY\sin\sthe\sorder\squery\swithout\ndoing\san\sextra\ssort,\sthen\somit\sthe\sextra\ssort. -D 2024-08-15T23:38:52.210 +C Improvements\sto\sthe\ssubquery\sORDER\sBY\sdecision\salgorithm.\sFewer\sfalse-negatives. +D 2024-08-16T00:37:54.145 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -846,7 +846,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c c59dffd249e5daa335bc43c2b87ffacc25dce995e3600a2b7bcdbcc55a68c985 +F src/where.c c6b742473a627bb0ff2aa1f05f3daaa7fc974b05fe4f84ae8a2d07fd7b92393c F src/whereInt.h 6444b888ce395cb80511284b8a73b63472d34247fcb1b125ee06a54fa6ae878e F src/wherecode.c c9cac0b0b8e809c5e7e79d7796918907fb685ad99be2aaa9737f9787aa47349c F src/whereexpr.c 7d0d34b42b9edfd8e8ca66beb3a6ef63fe211c001af54caf2ccbcd989b783290 @@ -2204,11 +2204,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 96e6cfb211f7f1aab50f997f4bc61dfb9701beb720c59bc413a4285c1ae5e20d -R cc9a8ceafd2a9a7a7cbd542af1820055 -T *branch * order-by-subquery -T *sym-order-by-subquery * -T -sym-trunk * +P 2fbb4dc2327ee435cb2b7a4adcddf5a9cee6dff7de96e2ecb761166427b5ddea +R 4f42ef8d0868b345800db6a42ea20509 U drh -Z 66dcbf2773ae39411ce78a1d9346e7ac +Z a8bc8a8b808f8550328f0344d9e7735f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8f936c703f..95e076fb40 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2fbb4dc2327ee435cb2b7a4adcddf5a9cee6dff7de96e2ecb761166427b5ddea +b82421e3f5811946e9d60b845fc882b6ea5c53c648695654c6900754427bf9bf diff --git a/src/where.c b/src/where.c index b119301e13..561091e0a0 100644 --- a/src/where.c +++ b/src/where.c @@ -4887,22 +4887,25 @@ static SQLITE_NOINLINE int wherePathMatchSubqueryOB( if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) break; if( pOBExpr->iTable!=iCur ) break; if( pOBExpr->iColumn!=pSubOB->a[j].u.x.iOrderByCol-1 ) break; - if( pSubOB->a[j].fg.sortFlags & KEYINFO_ORDER_BIGNULL ) break; - revIdx = pSubOB->a[j].fg.sortFlags & KEYINFO_ORDER_DESC; - if( wctrlFlags & WHERE_GROUPBY ){ - /* Sort order does not matter for GROUP BY */ - }else if( j>0 ){ - if( (rev ^ revIdx) != (pOrderBy->a[i].fg.sortFlags&KEYINFO_ORDER_DESC) ){ + if( (wctrlFlags & WHERE_GROUPBY)==0 ){ + if( (pSubOB->a[j].fg.sortFlags & KEYINFO_ORDER_BIGNULL) + != (pOrderBy->a[j].fg.sortFlags & KEYINFO_ORDER_BIGNULL) ){ break; } - }else{ - rev = revIdx ^ (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC); - if( rev ){ - if( (pLoop->wsFlags & WHERE_COROUTINE)!=0 ){ - /* Cannot run a co-routine in reverse order */ + revIdx = pSubOB->a[j].fg.sortFlags & KEYINFO_ORDER_DESC; + if( j>0 ){ + if( (rev ^ revIdx)!=(pOrderBy->a[i].fg.sortFlags&KEYINFO_ORDER_DESC) ){ break; } - *pRevMask |= MASKBIT(iLoop); + }else{ + rev = revIdx ^ (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC); + if( rev ){ + if( (pLoop->wsFlags & WHERE_COROUTINE)!=0 ){ + /* Cannot run a co-routine in reverse order */ + break; + } + *pRevMask |= MASKBIT(iLoop); + } } } *pOBSat |= MASKBIT(i); From 42f997d768f3853c9d5bca5e3e07878236ee698b Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 16 Aug 2024 02:19:17 +0000 Subject: [PATCH 10/15] Bug fix in the subquery ORDER BY propagator. FossilOrigin-Name: 5a9a3b8af7ac0aa1c04ad2d735e341c92d67952acb9a1d30217c0471e92cd468 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index ce77ec7858..b8037df583 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\ssubquery\sORDER\sBY\sdecision\salgorithm.\sFewer\sfalse-negatives. -D 2024-08-16T00:37:54.145 +C Bug\sfix\sin\sthe\ssubquery\sORDER\sBY\spropagator. +D 2024-08-16T02:19:17.392 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -846,7 +846,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c c6b742473a627bb0ff2aa1f05f3daaa7fc974b05fe4f84ae8a2d07fd7b92393c +F src/where.c 154533a45da7b54f460d7415b0ea3aea59855a58460cf1356175b0045ff30c26 F src/whereInt.h 6444b888ce395cb80511284b8a73b63472d34247fcb1b125ee06a54fa6ae878e F src/wherecode.c c9cac0b0b8e809c5e7e79d7796918907fb685ad99be2aaa9737f9787aa47349c F src/whereexpr.c 7d0d34b42b9edfd8e8ca66beb3a6ef63fe211c001af54caf2ccbcd989b783290 @@ -2204,8 +2204,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2fbb4dc2327ee435cb2b7a4adcddf5a9cee6dff7de96e2ecb761166427b5ddea -R 4f42ef8d0868b345800db6a42ea20509 +P b82421e3f5811946e9d60b845fc882b6ea5c53c648695654c6900754427bf9bf +R 41797d101dab015774684ce7d95aeefd U drh -Z a8bc8a8b808f8550328f0344d9e7735f +Z c79ab6ee6fe9513aaffefd02f00f8786 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 95e076fb40..77af5ed578 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b82421e3f5811946e9d60b845fc882b6ea5c53c648695654c6900754427bf9bf +5a9a3b8af7ac0aa1c04ad2d735e341c92d67952acb9a1d30217c0471e92cd468 diff --git a/src/where.c b/src/where.c index 561091e0a0..442904a654 100644 --- a/src/where.c +++ b/src/where.c @@ -5064,6 +5064,7 @@ static i8 wherePathSatisfiesOrderBy( pOrderBy,pRevMask, &obSat) ){ nColumn = 0; + isOrderDistinct = 0; }else{ nColumn = 1; } From 2ed4f5016a591b5aaabe4a80fc073faab4ad907d Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 16 Aug 2024 11:26:21 +0000 Subject: [PATCH 11/15] Fix handling of COLLATE. Add test cases for the same. Code cleanup for improved understandability and maintainability. FossilOrigin-Name: 41a41c173a9d15d94f23d73a5c04bfb1616cb9223bc81d41808f9b4d00817fbf --- manifest | 15 ++++---- manifest.uuid | 2 +- src/where.c | 71 ++++++++++++++++++++++------------ test/orderbyB.test | 94 ++++++++++++++++++++++++++++++++++++++++++++++ test/tester.tcl | 23 ++++++++++++ 5 files changed, 174 insertions(+), 31 deletions(-) create mode 100644 test/orderbyB.test diff --git a/manifest b/manifest index b8037df583..3d86a9c04b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bug\sfix\sin\sthe\ssubquery\sORDER\sBY\spropagator. -D 2024-08-16T02:19:17.392 +C Fix\shandling\sof\sCOLLATE.\s\sAdd\stest\scases\sfor\sthe\ssame.\s\sCode\scleanup\sfor\nimproved\sunderstandability\sand\smaintainability. +D 2024-08-16T11:26:21.307 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -846,7 +846,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c 154533a45da7b54f460d7415b0ea3aea59855a58460cf1356175b0045ff30c26 +F src/where.c f5be664f3379c9f930696e339ec4ef4c1187af860cca727411156101fae6b677 F src/whereInt.h 6444b888ce395cb80511284b8a73b63472d34247fcb1b125ee06a54fa6ae878e F src/wherecode.c c9cac0b0b8e809c5e7e79d7796918907fb685ad99be2aaa9737f9787aa47349c F src/whereexpr.c 7d0d34b42b9edfd8e8ca66beb3a6ef63fe211c001af54caf2ccbcd989b783290 @@ -1493,6 +1493,7 @@ F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3 F test/orderbyA.test df608e59efc2ef50c1eddf1a773b272de3252e9401bfec86d04b52fd973866d5 +F test/orderbyB.test 32576c7b138105bc72f7fbf33bd320ca3a7d303641fc939e0e56af6cba884b3d F test/oserror.test 1fc9746b83d778e70d115049747ba19c7fba154afce7cc165b09feb6ca6abbc5 F test/ossfuzz.c 9636dad2092a05a32110df0ca06713038dd0c43dd89a77dabe4b8b0d71096715 F test/ossshell.c f125c5bd16e537a2549aa579b328dd1c59905e7ab1338dfc210e755bb7b69f17 @@ -1706,7 +1707,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 640106bf8f7785d0ac67cda2837577eb9f2d936033bacedf9e705ca5451958ef +F test/tester.tcl e88c498c369cff6bf0898db6d04088685066730be51821ef775ef13fd2b1d077 F test/testrunner.tcl 5d02deeba7a53baeadae6aa7641d90aac58fdfa3a7bcac85cfcfd752b1aab87c F test/testrunner_data.tcl c5ae2b1f9a99210b0600d002fb3af1fee350997cee9416551e83b93501360ebf F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 @@ -2204,8 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b82421e3f5811946e9d60b845fc882b6ea5c53c648695654c6900754427bf9bf -R 41797d101dab015774684ce7d95aeefd +P 5a9a3b8af7ac0aa1c04ad2d735e341c92d67952acb9a1d30217c0471e92cd468 +R 7950e16a6ca37805979b1472a80cc190 U drh -Z c79ab6ee6fe9513aaffefd02f00f8786 +Z 14a0d323dd6fe70b6ed40930f5fa77fd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 77af5ed578..b684817cd4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5a9a3b8af7ac0aa1c04ad2d735e341c92d67952acb9a1d30217c0471e92cd468 +41a41c173a9d15d94f23d73a5c04bfb1616cb9223bc81d41808f9b4d00817fbf diff --git a/src/where.c b/src/where.c index 442904a654..631b7581fb 100644 --- a/src/where.c +++ b/src/where.c @@ -4847,7 +4847,8 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ return rc; } -/* +/* Implementation of the order-by-subquery optimization: +** ** WhereLoop pLoop, which the iLoop-th term of the nested loop, is really ** a subquery or CTE that has an ORDER BY clause. See if any of the terms ** in the subquery ORDER BY clause will satisfy pOrderBy from the outer @@ -4862,43 +4863,67 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ ** SELECT * FROM t3 JOIN t1 ON a=q ORDER BY p, b; ** ** The CTE named "t3" comes out in the natural order of "p", so the first -** first them of "ORDER BY p,b" is satisfied by a sequential scan of "t3". +** first them of "ORDER BY p,b" is satisfied by a sequential scan of "t3" +** and sorting only needs to occur on the second term "b". ** +** Limitations: +** +** (1) The optimization is not applied if the outer ORDER BY contains +** a COLLATE clause. The optimization might be applied if the +** outer ORDER BY uses NULLS FIRST, NULLS LAST, ASC, and/or DESC as +** long as the subquery ORDER BY does the same. But if the +** outer ORDER BY uses COLLATE, even a redundant COLLATE, the +** optimization is bypassed. +** +** (2) The subquery ORDER BY terms must exactly match subquery result +** columns, including any COLLATE annotations. This routine relies +** on iOrderByCol to do matching between order by terms and result +** columns, and iOrderByCol will not be set if the result column +** and ORDER BY collations differ. +** +** (3) The subquery and outer ORDER BY can be in opposite directions as +** long as the subquery is materialized. If the subquery is +** implemented as a co-routine, the sort orders must be in the same +** direction because there is no way to run a co-routine backwards. */ static SQLITE_NOINLINE int wherePathMatchSubqueryOB( + WhereInfo *pWInfo, /* The WHERE clause */ WhereLoop *pLoop, /* The nested loop term that is a subquery */ int iLoop, /* Which level of the nested loop. 0==outermost */ int iCur, /* Cursor used by the this loop */ - int wctrlFlags, /* Flags determining sort behavior */ ExprList *pOrderBy, /* The ORDER BY clause on the whole query */ Bitmask *pRevMask, /* When loops need to go in reverse order */ Bitmask *pOBSat /* Which terms of pOrderBy are satisfied so far */ ){ - int i, j; - u8 rev = 0; - u8 revIdx = 0; - Expr *pOBExpr; - ExprList *pSubOB = pLoop->u.btree.pOrderBy; + int iOB; /* Index into pOrderBy->a[] */ + int jSub; /* Index into pSubOB->a[] */ + u8 rev = 0; /* True if iOB and jSub sort in opposite directions */ + u8 revIdx = 0; /* Sort direction for jSub */ + Expr *pOBExpr; /* Current term of outer ORDER BY */ + ExprList *pSubOB; /* Complete ORDER BY on the subquery */ + + pSubOB = pLoop->u.btree.pOrderBy; assert( pSubOB!=0 ); - for(i=0; (MASKBIT(i) & *pOBSat)!=0; i++){} - for(j=0; jnExpr && inExpr; j++, i++){ - if( pSubOB->a[j].u.x.iOrderByCol==0 ) break; - pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[j].pExpr); + for(iOB=0; (MASKBIT(iOB) & *pOBSat)!=0; iOB++){} + for(jSub=0; jSubnExpr && iOBnExpr; jSub++, iOB++){ + if( pSubOB->a[jSub].u.x.iOrderByCol==0 ) break; + pOBExpr = pOrderBy->a[iOB].pExpr; if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) break; if( pOBExpr->iTable!=iCur ) break; - if( pOBExpr->iColumn!=pSubOB->a[j].u.x.iOrderByCol-1 ) break; - if( (wctrlFlags & WHERE_GROUPBY)==0 ){ - if( (pSubOB->a[j].fg.sortFlags & KEYINFO_ORDER_BIGNULL) - != (pOrderBy->a[j].fg.sortFlags & KEYINFO_ORDER_BIGNULL) ){ + if( pOBExpr->iColumn!=pSubOB->a[jSub].u.x.iOrderByCol-1 ) break; + if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){ + u8 sfOB = pOrderBy->a[iOB].fg.sortFlags; /* sortFlags for iOB */ + u8 sfSub = pSubOB->a[jSub].fg.sortFlags; /* sortFlags for jSub */ + if( (sfSub & KEYINFO_ORDER_BIGNULL) != (sfOB & KEYINFO_ORDER_BIGNULL) ){ break; } - revIdx = pSubOB->a[j].fg.sortFlags & KEYINFO_ORDER_DESC; - if( j>0 ){ - if( (rev ^ revIdx)!=(pOrderBy->a[i].fg.sortFlags&KEYINFO_ORDER_DESC) ){ + revIdx = sfSub & KEYINFO_ORDER_DESC; + if( jSub>0 ){ + if( (rev^revIdx)!=(sfOB & KEYINFO_ORDER_DESC) ){ break; } }else{ - rev = revIdx ^ (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC); + rev = revIdx ^ (sfOB & KEYINFO_ORDER_DESC); if( rev ){ if( (pLoop->wsFlags & WHERE_COROUTINE)!=0 ){ /* Cannot run a co-routine in reverse order */ @@ -4908,9 +4933,9 @@ static SQLITE_NOINLINE int wherePathMatchSubqueryOB( } } } - *pOBSat |= MASKBIT(i); + *pOBSat |= MASKBIT(iOB); } - return j>0; + return jSub>0; } /* @@ -5060,7 +5085,7 @@ static i8 wherePathSatisfiesOrderBy( if( pLoop->wsFlags & WHERE_IPK ){ if( pLoop->u.btree.pOrderBy && OptimizationEnabled(db, SQLITE_OrderBySubq) - && wherePathMatchSubqueryOB(pLoop,iLoop,iCur,wctrlFlags, + && wherePathMatchSubqueryOB(pWInfo,pLoop,iLoop,iCur, pOrderBy,pRevMask, &obSat) ){ nColumn = 0; diff --git a/test/orderbyB.test b/test/orderbyB.test new file mode 100644 index 0000000000..42d2de7982 --- /dev/null +++ b/test/orderbyB.test @@ -0,0 +1,94 @@ +# 2024-08-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# Specifically, it tests cases with order-by-subquery optimization in which +# an ORDER BY in a subquery is used to help resolve an ORDER BY in the +# outer query without having to do an extra sort. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix orderbyb + +db null NULL +do_execsql_test 1.0 { + CREATE TABLE t1(a TEXT, b TEXT, c INT); + INSERT INTO t1 VALUES(NULL,NULL,NULL); + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<7) + INSERT INTO t1(a,b,c) SELECT char(p,p), char(q,q), n FROM + (SELECT ((n-1)%4)+0x61 AS p, abs(n*2-9+(n>=5))+0x60 AS q, n FROM c); + UPDATE t1 SET b=upper(b) WHERE c=1; + CREATE TABLE t2(k TEXT PRIMARY KEY, v INT) WITHOUT ROWID; + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<7) + INSERT INTO t2(k,v) SELECT char(0x60+n,0x60+n), n FROM c; + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<7) + INSERT INTO t2(k,v) SELECT char(0x40+n,0x40+n), n FROM c; + SELECT a,b,c,tx.v AS 'v-a', ty.v AS 'v-b' + FROM t1 LEFT JOIN t2 AS tx ON tx.k=a + LEFT JOIN t2 AS ty ON ty.k=b + ORDER BY c; +} { + NULL NULL NULL NULL NULL + aa GG 1 1 7 + bb ee 2 2 5 + cc cc 3 3 3 + dd aa 4 4 1 + aa bb 5 1 2 + bb dd 6 2 4 + cc ff 7 3 6 +} + +do_eqp_execsql_test 1.1 { + WITH t3(x,y) AS (SELECT a, b FROM t1 ORDER BY a, b LIMIT 8) + SELECT x, y, v FROM t3 LEFT JOIN t2 ON k=t3.y ORDER BY x, y COLLATE nocase; +} { + QUERY PLAN + |--CO-ROUTINE t3 + | |--SCAN t1 + | `--USE TEMP B-TREE FOR ORDER BY + |--SCAN t3 + |--SEARCH t2 USING PRIMARY KEY (k=?) LEFT-JOIN + `--USE TEMP B-TREE FOR LAST TERM OF ORDER BY +} { + NULL NULL NULL + aa bb 2 + aa GG 7 + bb dd 4 + bb ee 5 + cc cc 3 + cc ff 6 + dd aa 1 +} + +do_eqp_execsql_test 1.2 { + WITH t3(x,y) AS MATERIALIZED (SELECT a, b COLLATE nocase FROM t1 ORDER BY 1,2) + SELECT x, y, v FROM t3 LEFT JOIN t2 ON k=t3.y ORDER BY x,y; +} { + QUERY PLAN + |--MATERIALIZE t3 + | |--SCAN t1 + | `--USE TEMP B-TREE FOR ORDER BY + |--SCAN t3 + `--SEARCH t2 USING PRIMARY KEY (k=?) LEFT-JOIN +} { + NULL NULL NULL + aa bb 2 + aa GG 7 + bb dd 4 + bb ee 5 + cc cc 3 + cc ff 6 + dd aa 1 +} + + +finish_test diff --git a/test/tester.tcl b/test/tester.tcl index 63c83187aa..43f7c7a047 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -1056,6 +1056,29 @@ proc do_eqp_test {name sql res} { } } +# Do both an eqp_test and an execsql_test on the same SQL. +# +proc do_eqp_execsql_test {name sql res1 res2} { + if {[regexp {^\s+QUERY PLAN\n} $res1]} { + + set query_plan [query_plan_graph $sql] + + if {[list {*}$query_plan]==[list {*}$res1]} { + uplevel [list do_test ${name}a [list set {} ok] ok] + } else { + uplevel [list \ + do_test ${name}a [list query_plan_graph $sql] $res1 + ] + } + } else { + if {[string index $res 0]!="/"} { + set res1 "/*$res1*/" + } + uplevel do_execsql_test ${name}a [list "EXPLAIN QUERY PLAN $sql"] [list $res1] + } + uplevel do_execsql_test ${name}b [list $sql] [list $res2] +} + #------------------------------------------------------------------------- # Usage: do_select_tests PREFIX ?SWITCHES? TESTLIST From b8123415b2952793279bad794680c14e191d7ca5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 16 Aug 2024 15:35:20 +0000 Subject: [PATCH 12/15] Ensure that memory allocations in ANALYZE are always 8-byte aligned in order to satisfy a new assert() added by [539e4f661767ef90]. FossilOrigin-Name: a1915cbdeccac2ed49fb8a9733abf2c6239396e3174fed2ff275f66b606c48b6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 19 +++++++++++-------- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 22a05bc11b..6e81c8e8bf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\shelp\smessage\sfor\s--disable-tcl\son\sthe\sconfigure\sscript. -D 2024-08-15T15:31:46.143 +C Ensure\sthat\smemory\sallocations\sin\sANALYZE\sare\salways\s8-byte\saligned\sin\sorder\nto\ssatisfy\sa\snew\sassert()\sadded\sby\s[539e4f661767ef90]. +D 2024-08-16T15:35:20.377 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -693,7 +693,7 @@ F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 F src/alter.c bb663fddf1fe0e2e6d8758b2b7fb6374e7c057a6ca3955f37a48986806029765 -F src/analyze.c 5c4e2bfd0aa8e5157f7fb91a17d86905510a74397326dc5767ec4e0588a4eea5 +F src/analyze.c 30bf40ec4208ead9e977bec017bccc8a9681820936e38ca5a4a7443100a6d5c5 F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 @@ -2204,8 +2204,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P de927016aadd7ee55d947134e3540907a3ea5ab4015034c5d088e3a84905d98a -R fa965b3cf027827ab805def1d12c22e4 +P 96e6cfb211f7f1aab50f997f4bc61dfb9701beb720c59bc413a4285c1ae5e20d +R 19ca743d0eb81a004946ea85e6f05b5d U drh -Z a05ca4fe471b9ed0243c4a5725784485 +Z b629a1d86492e4c09b552ad5721d3d89 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dd4cda6e20..edb9927377 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96e6cfb211f7f1aab50f997f4bc61dfb9701beb720c59bc413a4285c1ae5e20d +a1915cbdeccac2ed49fb8a9733abf2c6239396e3174fed2ff275f66b606c48b6 diff --git a/src/analyze.c b/src/analyze.c index 774eeeed26..fc8680d78d 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1797,12 +1797,13 @@ static int loadStatTbl( while( sqlite3_step(pStmt)==SQLITE_ROW ){ int nIdxCol = 1; /* Number of columns in stat4 records */ - char *zIndex; /* Index name */ - Index *pIdx; /* Pointer to the index object */ - int nSample; /* Number of samples */ - int nByte; /* Bytes of space required */ - int i; /* Bytes of space required */ - tRowcnt *pSpace; + char *zIndex; /* Index name */ + Index *pIdx; /* Pointer to the index object */ + int nSample; /* Number of samples */ + int nByte; /* Bytes of space required */ + int i; /* Bytes of space required */ + tRowcnt *pSpace; /* Available allocated memory space */ + u8 *pPtr; /* Available memory as a u8 for easier manipulation */ zIndex = (char *)sqlite3_column_text(pStmt, 0); if( zIndex==0 ) continue; @@ -1822,7 +1823,7 @@ static int loadStatTbl( } pIdx->nSampleCol = nIdxCol; pIdx->mxSample = nSample; - nByte = sizeof(IndexSample) * nSample; + nByte = ROUND8(sizeof(IndexSample) * nSample); nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ @@ -1831,7 +1832,9 @@ static int loadStatTbl( sqlite3_finalize(pStmt); return SQLITE_NOMEM_BKPT; } - pSpace = (tRowcnt*)&pIdx->aSample[nSample]; + pPtr = (u8*)pIdx->aSample; + pPtr += ROUND8(nSample*sizeof(pIdx->aSample[0])); + pSpace = (tRowcnt*)pPtr; assert( EIGHT_BYTE_ALIGNMENT( pSpace ) ); pIdx->aAvgEq = pSpace; pSpace += nIdxCol; pIdx->pTable->tabFlags |= TF_HasStat4; From 52d03102562a4a141357cc3a48f4b903df3aa3bb Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 17 Aug 2024 14:59:26 +0000 Subject: [PATCH 13/15] Fix a typo in an "ifdef" that broke SQLITE_OMIT_ANALYZE builds. FossilOrigin-Name: c09da26cb6f772a18c595ff0eb55d8345a377c3ef5ed02a5e539ffdbe9d07f85 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index dd7e947eb5..704cb20843 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\ssubquery\shas\san\sORDER\sBY\sclause\sand\sthat\sordering\sis\shelpfile\sin\nsatisfying\sthe\sORDER\sBY\sor\sGROUP\sBY\sof\sthe\souter\squery\swithout\sdoing\san\nextra\ssort,\sthen\somit\sor\sreduce\sthe\ssort\sin\sthe\souter\squery.\s\sCall\nthis\sthe\s"order-by-subquery\soptimization". -D 2024-08-16T18:51:46.975 +C Fix\sa\stypo\sin\san\s"ifdef"\sthat\sbroke\sSQLITE_OMIT_ANALYZE\sbuilds. +D 2024-08-17T14:59:26.789 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -831,7 +831,7 @@ F src/upsert.c 2e60567a0e9e8520c18671b30712a88dc73534474304af94f32bb5f3ef65ac65 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e F src/util.c 5d1a0134cf4240648d1c6bb5cc8efaca0ea2b5d5c840985aec7e947271f04375 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 -F src/vdbe.c de13de572eccb688b2b7cf50e2f9005c44bf9ae89e35245ef8eadfc60dfd2764 +F src/vdbe.c be5f58bc29f60252e041a618eae59e8d57d460ba136c5403cf0abf955560c457 F src/vdbe.h c2549a215898a390de6669cfa32adba56f0d7e17ba5a7f7b14506d6fd5f0c36a F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c F src/vdbeapi.c 80235ac380e9467fec1cb0883354d841f2a771976e766995f7e0c77f845406df @@ -2205,9 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a1915cbdeccac2ed49fb8a9733abf2c6239396e3174fed2ff275f66b606c48b6 41a41c173a9d15d94f23d73a5c04bfb1616cb9223bc81d41808f9b4d00817fbf -R a24a84e9d7ceff04c159b78242edac9d -T +closed 41a41c173a9d15d94f23d73a5c04bfb1616cb9223bc81d41808f9b4d00817fbf -U drh -Z 169fa841b16225f819f8996d6765647e +P 7a0cdc7edb704a88a77b748cd28f6e00c49849cc2c1af838b95b34232ecc21f9 +R 501e32d0617414047f040d4ae2e8bb39 +U dan +Z 4ff808b9d2897218daeb3b1363a873d1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d32d2498e4..a4764edcfc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7a0cdc7edb704a88a77b748cd28f6e00c49849cc2c1af838b95b34232ecc21f9 +c09da26cb6f772a18c595ff0eb55d8345a377c3ef5ed02a5e539ffdbe9d07f85 diff --git a/src/vdbe.c b/src/vdbe.c index d097bfd8b9..ea69c759c2 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2096,7 +2096,7 @@ case OP_RealAffinity: { /* in1 */ } #endif -#if !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_ANALYZE) +#if !defined(SQLITE_OMIT_CAST) || !defined(SQLITE_OMIT_ANALYZE) /* Opcode: Cast P1 P2 * * * ** Synopsis: affinity(r[P1]) ** From f87a54ab41de3d7a276059c19f47aa1f9c93ecff Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 17 Aug 2024 15:52:21 +0000 Subject: [PATCH 14/15] Adjust the "configure" script so that it just disables the ability to run tests rather than erroring out if tclsh is unable to recommend a suitable tclConfig.sh. FossilOrigin-Name: 793ff83d09bea165c8cedc978afd3fcfb1a4cee894c5914aa1d92416afce963d --- configure | 8 +++++++- configure.ac | 7 ++++++- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/configure b/configure index 50d9e2b3d0..5f60385c28 100755 --- a/configure +++ b/configure @@ -10330,6 +10330,7 @@ else use_tcl=yes fi +original_use_tcl=${use_tcl} if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then for ac_prog in tclsh8.6 tclsh tclsh9.0 do @@ -10388,6 +10389,7 @@ $as_echo "$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&5 $as_echo "$as_me: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&2;} + use_tcl=no fi fi fi @@ -10438,10 +10440,14 @@ $as_echo "loading TCL configuration from ${tclconfig}" >&6; } # tclConfig.sh file and that could be included here. But as of right now, # TCL_LIB_SPEC is the only what that the Makefile uses. HAVE_TCL=1 -else +elif test x"${original_use_tcl}" = "xno"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: unable to run tests because of --disable-tcl" >&5 $as_echo "unable to run tests because of --disable-tcl" >&6; } HAVE_TCL=0 +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: unable to run tests because no tclConfig.sh file could be located" >&5 +$as_echo "unable to run tests because no tclConfig.sh file could be located" >&6; } + HAVE_TCL=0 fi if test x"$TCLSH_CMD" == x; then diff --git a/configure.ac b/configure.ac index 4f95a5c08c..2936c4ab5a 100644 --- a/configure.ac +++ b/configure.ac @@ -123,6 +123,7 @@ AC_ARG_WITH(tclsh, AS_HELP_STRING([--with-tclsh=PATHNAME],[full pathname of a tc AC_ARG_WITH(tcl, AS_HELP_STRING([--with-tcl=DIR],[directory containing (tclConfig.sh)])) AC_ARG_ENABLE(tcl, AS_HELP_STRING([--disable-tcl],[omit building accessory programs that require TCL-dev]), [use_tcl=$enableval],[use_tcl=yes]) +original_use_tcl=${use_tcl} if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.6 tclsh tclsh9.0],none) with_tclsh=${TCLSH_CMD} @@ -136,6 +137,7 @@ if test x"${with_tclsh}" != x -a x"${with_tclsh}" != xnone; then AC_MSG_RESULT([$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}]) else AC_MSG_WARN([$TCLSH_CMD is unable to recommend a tclConfig.sh]) + use_tcl=no fi fi fi @@ -185,9 +187,12 @@ if test x"${use_tcl}" = "xyes"; then # tclConfig.sh file and that could be included here. But as of right now, # TCL_LIB_SPEC is the only what that the Makefile uses. HAVE_TCL=1 -else +elif test x"${original_use_tcl}" = "xno"; then AC_MSG_RESULT([unable to run tests because of --disable-tcl]) HAVE_TCL=0 +else + AC_MSG_RESULT([unable to run tests because no tclConfig.sh file could be located]) + HAVE_TCL=0 fi AC_SUBST(HAVE_TCL) if test x"$TCLSH_CMD" == x; then diff --git a/manifest b/manifest index 704cb20843..8b5e68507a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\san\s"ifdef"\sthat\sbroke\sSQLITE_OMIT_ANALYZE\sbuilds. -D 2024-08-17T14:59:26.789 +C Adjust\sthe\s"configure"\sscript\sso\sthat\sit\sjust\sdisables\sthe\sability\sto\nrun\stests\srather\sthan\serroring\sout\sif\stclsh\sis\sunable\sto\srecommend\sa\nsuitable\stclConfig.sh. +D 2024-08-17T15:52:21.666 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -35,8 +35,8 @@ F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea0034 F autoconf/tea/win/rules.vc 7b3bb2ef32ade0f3f14d951231811678722725e3bca240dd9727ae0dfe10f6a5 F config.guess 883205ddf25b46f10c181818bf42c09da9888884af96f79e1719264345053bd6 F config.sub c2d0260f17f3e4bc0b6808fccf1b291cb5e9126c14fc5890efc77b9fd0175559 -F configure 2e5058c2ba2c6b50a01cf50c158180c4ec969b2068c2e361dfbfc875ec3f9d1c x -F configure.ac b7e26a699a2ffc5eb7a25636f32755c27bbb665777b68ef35a90c1ee9bd9d469 +F configure 956e132a3e2481e5016e9ecaa0a2bdd9df95c1d923fa4fb2aa2cf9f0b3a54ad9 x +F configure.ac fd9ff064b142682964971fec3d867235e51e1f6de30a031e33f683b7699338ff F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd F doc/compile-for-windows.md e8635eea9153dcd6a51fd2740666ebc4492b3813cb1ac31cd8e99150df91762d @@ -2205,8 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7a0cdc7edb704a88a77b748cd28f6e00c49849cc2c1af838b95b34232ecc21f9 -R 501e32d0617414047f040d4ae2e8bb39 -U dan -Z 4ff808b9d2897218daeb3b1363a873d1 +P c09da26cb6f772a18c595ff0eb55d8345a377c3ef5ed02a5e539ffdbe9d07f85 +R 8082c14d3791ff31a3c70ef600eeef0f +U drh +Z 2bdc019c3dd756a5d860815c6dd9a72b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a4764edcfc..1e3ae45cbc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c09da26cb6f772a18c595ff0eb55d8345a377c3ef5ed02a5e539ffdbe9d07f85 +793ff83d09bea165c8cedc978afd3fcfb1a4cee894c5914aa1d92416afce963d From 21363ac78df6751655c33372a7277512531b9570 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 17 Aug 2024 16:17:16 +0000 Subject: [PATCH 15/15] Adjust Makefile.in so that it outputs a slightly better error message on an attempt to build something that requires TCL libraries when HAVE_TCL is false. FossilOrigin-Name: a4043cbeb8a08fca2fdd2ea703e030d3a5574cc6002292ecc6f0e88c116472a3 --- Makefile.in | 14 +++++++++----- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Makefile.in b/Makefile.in index 7ff6e901e4..44665184fc 100644 --- a/Makefile.in +++ b/Makefile.in @@ -796,6 +796,10 @@ has_tclsh85: sh $(TOP)/tool/cktclsh.sh 8.5 $(TCLSH_CMD) touch has_tclsh85 +has_tclconfig: + @ if test x"$(HAVE_TCL)" != "x1"; then echo 'ERROR: Requires access to "tclConfig.sh" which "configure" was not able to locate'; exit 1; fi + touch has_tclconfig + # This target creates a directory named "tsrc" and fills it with # copies of all of the C source code and header files needed to @@ -1118,7 +1122,7 @@ tclsqlite-shell.lo: $(TOP)/src/tclsqlite.c $(HDR) tclsqlite-stubs.lo: $(TOP)/src/tclsqlite.c $(HDR) $(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c -tclsqlite3$(TEXE): tclsqlite-shell.lo libsqlite3.la +tclsqlite3$(TEXE): has_tclconfig tclsqlite-shell.lo libsqlite3.la $(LTLINK) -o $@ tclsqlite-shell.lo \ libsqlite3.la $(LIBTCL) @@ -1305,7 +1309,7 @@ TESTFIXTURE_SRC1 = sqlite3.c TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION)) -testfixture$(TEXE): has_tclsh85 $(TESTFIXTURE_SRC) +testfixture$(TEXE): has_tclconfig has_tclsh85 $(TESTFIXTURE_SRC) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \ -o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS) @@ -1404,13 +1408,13 @@ shelltest: $(TESTPROGS) sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in has_tclsh85 $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c -sqlite3_analyzer$(TEXE): sqlite3_analyzer.c +sqlite3_analyzer$(TEXE): has_tclconfig sqlite3_analyzer.c $(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS) sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in has_tclsh85 $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c -sqltclsh$(TEXE): sqltclsh.c +sqltclsh$(TEXE): has_tclconfig sqltclsh.c $(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS) sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c @@ -1429,7 +1433,7 @@ CHECKER_DEPS =\ sqlite3_checker.c: $(CHECKER_DEPS) has_tclsh85 $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@ -sqlite3_checker$(TEXE): sqlite3_checker.c +sqlite3_checker$(TEXE): has_tclconfig sqlite3_checker.c $(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS) dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo diff --git a/manifest b/manifest index 8b5e68507a..bd4f4cbe1c 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Adjust\sthe\s"configure"\sscript\sso\sthat\sit\sjust\sdisables\sthe\sability\sto\nrun\stests\srather\sthan\serroring\sout\sif\stclsh\sis\sunable\sto\srecommend\sa\nsuitable\stclConfig.sh. -D 2024-08-17T15:52:21.666 +C Adjust\sMakefile.in\sso\sthat\sit\soutputs\sa\sslightly\sbetter\serror\smessage\son\san\nattempt\sto\sbuild\ssomething\sthat\srequires\sTCL\slibraries\swhen\sHAVE_TCL\sis\sfalse. +D 2024-08-17T16:17:16.045 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in cf96aa7d2682650dec56dc2f03cfe463feb101a1be594e4929bcd38662bc2ea8 +F Makefile.in 209e9c64edf499b0ea7a15448699101c01d15828af35a3439d32fc25fdf1b787 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 F Makefile.msc 6c3fe8b6ce60e73f34a148c957d78b4648745c8d30e792423aa1a8d8bf12d065 F README.md 6358805260a03ebead84e168bbf3740ddf3f683b477e478567186aa7afb490d3 @@ -2205,8 +2205,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c09da26cb6f772a18c595ff0eb55d8345a377c3ef5ed02a5e539ffdbe9d07f85 -R 8082c14d3791ff31a3c70ef600eeef0f +P 793ff83d09bea165c8cedc978afd3fcfb1a4cee894c5914aa1d92416afce963d +R be5ece8b231f01bec02cbe9ef4eda884 U drh -Z 2bdc019c3dd756a5d860815c6dd9a72b +Z 6d429e685f531219ca0d0afba9760eb7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1e3ae45cbc..a5034c9d32 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -793ff83d09bea165c8cedc978afd3fcfb1a4cee894c5914aa1d92416afce963d +a4043cbeb8a08fca2fdd2ea703e030d3a5574cc6002292ecc6f0e88c116472a3