From 3e771c0f077a6a9fad6e560559edb3f501c3892e Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 14 Oct 2022 13:26:18 +0000 Subject: [PATCH 1/5] Remove a resolved TODO comment. FossilOrigin-Name: 32fd4ac382f774189ac34f6fff80e55e6e56dd2aa67b0db88d5a88324f17f6ff --- ext/wasm/sqlite3-opfs-async-proxy.js | 13 ++----------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/ext/wasm/sqlite3-opfs-async-proxy.js b/ext/wasm/sqlite3-opfs-async-proxy.js index fd4c7f2b80..d261dd354c 100644 --- a/ext/wasm/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/sqlite3-opfs-async-proxy.js @@ -25,13 +25,6 @@ access to the sqlite3 JS/WASM bits, so any bits which it needs (most notably SQLITE_xxx integer codes) have to be imported into it via an initialization process. - - Potential TODOs: - - - When idle for "a long time", close the sync access handle in order - to release the lock, then re-open it on demand. Similarly, delay - fetching of the sync access handle until we need it. The intent - would be to help multi-tab access to a db avoid locking issues. */ 'use strict'; const toss = function(...args){throw new Error(args.join(' '))}; @@ -42,8 +35,6 @@ if(self.window === self){ toss("This API requires navigator.storage.getDirectory."); } -//warn("This file is very much experimental and under construction.",self.location.pathname); - /** Will hold state copied to this object from the syncronous side of this API. @@ -161,7 +152,7 @@ const getSyncHandle = async (fh)=>{ try { //if(1===i) toss("Just testing."); //TODO? A config option which tells it to throw here - //randomly every now and then. + //randomly every now and then, for testing purposes. fh.syncHandle = await fh.fileHandle.createSyncAccessHandle(); break; }catch(e){ @@ -170,7 +161,7 @@ const getSyncHandle = async (fh)=>{ "attempts failed. ",fh.filenameAbs, ":", e.message); throw e; } - error("Error getting sync handle. Waiting",ms, + warn("Error getting sync handle. Waiting",ms, "ms and trying again.",fh.filenameAbs,e); Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms); } diff --git a/manifest b/manifest index 13be78b776..2d787a777f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Move\sthe\srest\sof\stesting1.js\sinto\stester1.js\sand\seliminate\sthe\sdependency\son\sjaccwabyt_test.c.\sExtend\sthe\slist\sof\sdefault\sconfig-related\s#defines\sin\ssqlite3-wasm.c\sand\sreorganize\sthem\sfor\smaintainability. -D 2022-10-13T16:48:35.302 +C Remove\sa\sresolved\sTODO\scomment. +D 2022-10-14T13:26:18.756 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -519,7 +519,7 @@ F ext/wasm/speedtest1.html e4cb5d722b494104fc1249e7c008ca018f820a784833c51004c95 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 -F ext/wasm/sqlite3-opfs-async-proxy.js 40fc1f779e18e08b31404ef832381ffd36b9d11447007aaf96223198324e4467 +F ext/wasm/sqlite3-opfs-async-proxy.js e552d9290509644929cd7bfde3381a0eac85d17b4d82dbb11e86fae5336215b6 F ext/wasm/sqlite3-worker1-promiser.js 307d7837420ca6a9d3780dfc81194f1c0715637e6d9540e935514086b96913d8 F ext/wasm/sqlite3-worker1.js 466e9bd39409ab03f3e00999887aaffc11e95b416e2689596e3d7f1516673fdf F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5 @@ -2031,8 +2031,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cb94350185f555c333b628ee846c47bcc9df5f76bb82de569b8322f30dbbe1bc -R 32dab828066657d904ede486bb592456 +P 4e2a8aff2dd4b6e148f45184e2523ebe47815257eca97fa3d32bcbf9625f0def +R 9e71573268d8509a62b15b73d008f5e9 U stephan -Z 3fd1b23c49b42c2df54d60ffd9b89d7f +Z fced5d49252f4340d2afc14fae1e4aa8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7e72bfdd21..6ab0e1f14e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e2a8aff2dd4b6e148f45184e2523ebe47815257eca97fa3d32bcbf9625f0def \ No newline at end of file +32fd4ac382f774189ac34f6fff80e55e6e56dd2aa67b0db88d5a88324f17f6ff \ No newline at end of file From 2bc4a6cad0457fc07138f8c27d0190f3acb33f72 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 14 Oct 2022 15:10:36 +0000 Subject: [PATCH 2/5] Fix a problem in the LIKE and GLOB operators that may occur when the character immediately following a "%" or "*" wildcard is U+80. Reported by [forum:61bf7ccbdf]. FossilOrigin-Name: 2da677c45b643482eec39e4db7079c772760bc966dc71bf6c01658cc468f5823 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/func.c | 2 +- test/like2.test | 8 ++++++++ 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 2d787a777f..5e6a5ac9b8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sresolved\sTODO\scomment. -D 2022-10-14T13:26:18.756 +C Fix\sa\sproblem\sin\sthe\sLIKE\sand\sGLOB\soperators\sthat\smay\soccur\swhen\sthe\scharacter\simmediately\sfollowing\sa\s"%"\sor\s"*"\swildcard\sis\sU+80.\sReported\sby\s[forum:61bf7ccbdf]. +D 2022-10-14T15:10:36.387 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -570,7 +570,7 @@ F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e F src/expr.c d199850a925665df7f8ac29fa938909f65a2bd32354aa206aa8c25fc74ff3d3d F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 -F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1 +F src/func.c fe2a795ad647ce42054873ac91c43beb7b5d204892903a67f6e7e314379b9d4a F src/global.c e83ee571b79ee3adc32e380cf554cf1254bc43763d23786c71721fbcdfbbb965 F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 @@ -1243,7 +1243,7 @@ F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/lemon-test01.y 58b764610fd934e189ffbb0bbfa33d171b9cb06019b55bdc04d090d6767e11d7 F test/like.test 5fe0bc37f307aef0a453ce2de4632bdfc0759448f0421c39f6d53caefe905fac -F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da +F test/like2.test d3be15fefee3e02fc88942a9b98f26c5339bbdef7783c90023c092c4955fe3d3 F test/like3.test a76e5938fadbe6d32807284c796bafd869974a961057bc5fc5a28e06de98745c F test/limit.test 350f5d03c29e7dff9a2cde016f84f8d368d40bcd02fa2b2a52fa10c4bf3cbfaf F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e @@ -2031,8 +2031,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4e2a8aff2dd4b6e148f45184e2523ebe47815257eca97fa3d32bcbf9625f0def -R 9e71573268d8509a62b15b73d008f5e9 -U stephan -Z fced5d49252f4340d2afc14fae1e4aa8 +P 32fd4ac382f774189ac34f6fff80e55e6e56dd2aa67b0db88d5a88324f17f6ff +R ad7e356fc9072ccedd7e8855b564b667 +U dan +Z 16dbeccb3b1fdf0272ff90de38793516 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6ab0e1f14e..8828e08bae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -32fd4ac382f774189ac34f6fff80e55e6e56dd2aa67b0db88d5a88324f17f6ff \ No newline at end of file +2da677c45b643482eec39e4db7079c772760bc966dc71bf6c01658cc468f5823 \ No newline at end of file diff --git a/src/func.c b/src/func.c index ec50a17a37..06581cbe32 100644 --- a/src/func.c +++ b/src/func.c @@ -740,7 +740,7 @@ static int patternCompare( ** c but in the other case and search the input string for either ** c or cx. */ - if( c<=0x80 ){ + if( c<0x80 ){ char zStop[3]; int bMatch; if( noCase ){ diff --git a/test/like2.test b/test/like2.test index 4fd16420ad..2a4474edce 100644 --- a/test/like2.test +++ b/test/like2.test @@ -1006,4 +1006,12 @@ do_test like-2.126.3 { } {126} +do_test like-3.1 { + db eval "SELECT '\u01C0' LIKE '%\x80'" +} {0} +do_test like-3.2 { + db eval "SELECT '\u0080' LIKE '%\x80'" +} {1} + + finish_test From e72ddfd89b05c7ed60b82ac019480d1f3a5c7a21 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 14 Oct 2022 15:52:29 +0000 Subject: [PATCH 3/5] Generic minor cleanups and docs in the OPFS async proxy. FossilOrigin-Name: a4423ca234453c14eb40db7fe5943f63b30fd9dc2207388e8a2966733a004e9d --- ext/wasm/sqlite3-opfs-async-proxy.js | 97 ++++++++++++++++------------ manifest | 14 ++-- manifest.uuid | 2 +- 3 files changed, 64 insertions(+), 49 deletions(-) diff --git a/ext/wasm/sqlite3-opfs-async-proxy.js b/ext/wasm/sqlite3-opfs-async-proxy.js index d261dd354c..5b29aa3ea5 100644 --- a/ext/wasm/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/sqlite3-opfs-async-proxy.js @@ -10,10 +10,10 @@ *********************************************************************** - An INCOMPLETE and UNDER CONSTRUCTION experiment for OPFS: a Worker - which manages asynchronous OPFS handles on behalf of a synchronous - API which controls it via a combination of Worker messages, - SharedArrayBuffer, and Atomics. + An Worker which manages asynchronous OPFS handles on behalf of a + synchronous API which controls it via a combination of Worker + messages, SharedArrayBuffer, and Atomics. It is the asynchronous + counterpart of the API defined in sqlite3-api-opfs.js. Highly indebted to: @@ -102,9 +102,9 @@ const __openFiles = Object.create(null); /** Expects an OPFS file path. It gets resolved, such that ".." - components are properly expanded, and returned. If the 2nd - are is true, it's returned as an array of path elements, - else it's returned as an absolute path string. + components are properly expanded, and returned. If the 2nd arg is + true, the result is returned as an array of path elements, else an + absolute path string is returned. */ const getResolvedPath = function(filename,splitIt){ const p = new URL( @@ -115,9 +115,9 @@ const getResolvedPath = function(filename,splitIt){ /** Takes the absolute path to a filesystem element. Returns an array - of [handleOfContainingDir, filename]. If the 2nd argument is - truthy then each directory element leading to the file is created - along the way. Throws if any creation or resolution fails. + of [handleOfContainingDir, filename]. If the 2nd argument is truthy + then each directory element leading to the file is created along + the way. Throws if any creation or resolution fails. */ const getDirForFilename = async function f(absFilename, createDirs = false){ const path = getResolvedPath(absFilename, true); @@ -171,6 +171,16 @@ const getSyncHandle = async (fh)=>{ return fh.syncHandle; }; +/** + If the given file-holding object has a sync handle attached to it, + that handle is remove and asynchronously closed. Though it may + sound sensible to continue work as soon as the close() returns + (noting that it's asynchronous), doing so can cause operations + performed soon afterwards, e.g. a call to getSyncHandle() to fail + because they may happen out of order from the close(). OPFS does + not guaranty that the actual order of operations is retained in + such cases. i.e. always "await" on the result of this function. +*/ const closeSyncHandle = async (fh)=>{ if(fh.syncHandle){ log("Closing sync handle for",fh.filenameAbs); @@ -197,43 +207,50 @@ const affirmNotRO = function(opName,fh){ if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs); }; - -const opTimer = Object.create(null); -opTimer.op = undefined; -opTimer.start = undefined; +/** + We track 2 different timers: the "metrics" timer records how much + time we spend performing work. The "wait" timer records how much + time we spend waiting on the underlying OPFS timer. See the calls + to mTimeStart(), mTimeEnd(), wTimeStart(), and wTimeEnd() + throughout this file to see how they're used. +*/ +const __mTimer = Object.create(null); +__mTimer.op = undefined; +__mTimer.start = undefined; const mTimeStart = (op)=>{ - opTimer.start = performance.now(); - opTimer.op = op; + __mTimer.start = performance.now(); + __mTimer.op = op; //metrics[op] || toss("Maintenance required: missing metrics for",op); ++metrics[op].count; }; const mTimeEnd = ()=>( - metrics[opTimer.op].time += performance.now() - opTimer.start + metrics[__mTimer.op].time += performance.now() - __mTimer.start ); -const waitTimer = Object.create(null); -waitTimer.op = undefined; -waitTimer.start = undefined; +const __wTimer = Object.create(null); +__wTimer.op = undefined; +__wTimer.start = undefined; const wTimeStart = (op)=>{ - waitTimer.start = performance.now(); - waitTimer.op = op; + __wTimer.start = performance.now(); + __wTimer.op = op; //metrics[op] || toss("Maintenance required: missing metrics for",op); }; const wTimeEnd = ()=>( - metrics[waitTimer.op].wait += performance.now() - waitTimer.start + metrics[__wTimer.op].wait += performance.now() - __wTimer.start ); /** - Set to true by the 'opfs-async-shutdown' command to quite the wait loop. - This is only intended for debugging purposes: we cannot inspect this - file's state while the tight waitLoop() is running. + Gets set to true by the 'opfs-async-shutdown' command to quit the + wait loop. This is only intended for debugging purposes: we cannot + inspect this file's state while the tight waitLoop() is running and + need a way to stop that loop for introspection purposes. */ let flagAsyncShutdown = false; /** Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods - methods. Maintenance reminder: members are in alphabetical order - to simplify finding them. + methods, as well as helpers like mkdir(). Maintenance reminder: + members are in alphabetical order to simplify finding them. */ const vfsAsyncImpls = { 'opfs-async-metrics': async ()=>{ @@ -369,11 +386,13 @@ const vfsAsyncImpls = { const fh = __openFiles[fid]; let rc = 0; if( !fh.syncHandle ){ + wTimeStart('xLock'); try { await getSyncHandle(fh) } catch(e){ state.s11n.storeException(1,e); rc = state.sq3Codes.SQLITE_IOERR; } + wTimeEnd(); } storeAndNotify('xLock',rc); mTimeEnd(); @@ -423,11 +442,11 @@ const vfsAsyncImpls = { }, xRead: async function(fid,n,offset){ mTimeStart('xRead'); - let rc = 0; + let rc = 0, nRead; const fh = __openFiles[fid]; try{ wTimeStart('xRead'); - const nRead = (await getSyncHandle(fh)).read( + nRead = (await getSyncHandle(fh)).read( fh.sabView.subarray(0, n), {at: Number(offset)} ); @@ -437,6 +456,7 @@ const vfsAsyncImpls = { rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; } }catch(e){ + if(undefined===nRead) wTimeEnd(); error("xRead() failed",e,fh); state.s11n.storeException(1,e); rc = state.sq3Codes.SQLITE_IOERR_READ; @@ -454,9 +474,8 @@ const vfsAsyncImpls = { await fh.syncHandle.flush(); }catch(e){ state.s11n.storeException(2,e); - }finally{ - wTimeEnd(); } + wTimeEnd(); } storeAndNotify('xSync',rc); mTimeEnd(); @@ -484,13 +503,13 @@ const vfsAsyncImpls = { const fh = __openFiles[fid]; if( state.sq3Codes.SQLITE_LOCK_NONE===lockType && fh.syncHandle ){ + wTimeStart('xUnlock'); try { await closeSyncHandle(fh) } catch(e){ state.s11n.storeException(1,e); rc = state.sq3Codes.SQLITE_IOERR; - /* Maybe we want to not report this? "Destructors do not - throw." */ } + wTimeEnd(); } storeAndNotify('xUnlock',rc); mTimeEnd(); @@ -511,9 +530,8 @@ const vfsAsyncImpls = { error("xWrite():",e,fh); state.s11n.storeException(1,e); rc = state.sq3Codes.SQLITE_IOERR_WRITE; - }finally{ - wTimeEnd(); } + wTimeEnd(); storeAndNotify('xWrite',rc); mTimeEnd(); } @@ -632,7 +650,7 @@ const waitLoop = async function f(){ const o = Object.create(null); opHandlers[state.opIds[k]] = o; o.key = k; - o.f = vi || toss("No vfsAsyncImpls[",k,"]"); + o.f = vi; } /** waitTime is how long (ms) to wait for each Atomics.wait(). @@ -640,8 +658,6 @@ const waitLoop = async function f(){ to do other things. */ const waitTime = 1000; - let lastOpTime = performance.now(); - let now; while(!flagAsyncShutdown){ try { if('timed-out'===Atomics.wait( @@ -649,7 +665,6 @@ const waitLoop = async function f(){ )){ continue; } - lastOpTime = performance.now(); const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); Atomics.store(state.sabOPView, state.opIds.whichOp, 0); const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); @@ -663,7 +678,7 @@ const waitLoop = async function f(){ }catch(e){ error('in waitLoop():',e); } - }; + } }; navigator.storage.getDirectory().then(function(d){ diff --git a/manifest b/manifest index 5e6a5ac9b8..259868c45d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\sLIKE\sand\sGLOB\soperators\sthat\smay\soccur\swhen\sthe\scharacter\simmediately\sfollowing\sa\s"%"\sor\s"*"\swildcard\sis\sU+80.\sReported\sby\s[forum:61bf7ccbdf]. -D 2022-10-14T15:10:36.387 +C Generic\sminor\scleanups\sand\sdocs\sin\sthe\sOPFS\sasync\sproxy. +D 2022-10-14T15:52:29.231 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -519,7 +519,7 @@ F ext/wasm/speedtest1.html e4cb5d722b494104fc1249e7c008ca018f820a784833c51004c95 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 -F ext/wasm/sqlite3-opfs-async-proxy.js e552d9290509644929cd7bfde3381a0eac85d17b4d82dbb11e86fae5336215b6 +F ext/wasm/sqlite3-opfs-async-proxy.js 206ce6bbc3c30ad51a37d9c25e3a2712e70b586e0f9a2cf8cb0b9619017c2671 F ext/wasm/sqlite3-worker1-promiser.js 307d7837420ca6a9d3780dfc81194f1c0715637e6d9540e935514086b96913d8 F ext/wasm/sqlite3-worker1.js 466e9bd39409ab03f3e00999887aaffc11e95b416e2689596e3d7f1516673fdf F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5 @@ -2031,8 +2031,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 32fd4ac382f774189ac34f6fff80e55e6e56dd2aa67b0db88d5a88324f17f6ff -R ad7e356fc9072ccedd7e8855b564b667 -U dan -Z 16dbeccb3b1fdf0272ff90de38793516 +P 2da677c45b643482eec39e4db7079c772760bc966dc71bf6c01658cc468f5823 +R bd5efaaf882b3ae0c03e0e5a6693b37e +U stephan +Z c8a3d40e488aded9faa8a7e4d2f0c359 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8828e08bae..de485659dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2da677c45b643482eec39e4db7079c772760bc966dc71bf6c01658cc468f5823 \ No newline at end of file +a4423ca234453c14eb40db7fe5943f63b30fd9dc2207388e8a2966733a004e9d \ No newline at end of file From 0388f17988d3b5f0281f92123d9c78732c13c1f2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 14 Oct 2022 19:21:12 +0000 Subject: [PATCH 4/5] Add missing initializer from the extension loader. Fix for check-in [d6d449978245b4fa]. FossilOrigin-Name: 565d74c36f2dd4612860c7ee561682c50db28cfa5384346528292019dbfdf86c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/loadext.c | 4 +++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 259868c45d..7c2b8ac79b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Generic\sminor\scleanups\sand\sdocs\sin\sthe\sOPFS\sasync\sproxy. -D 2022-10-14T15:52:29.231 +C Add\smissing\sinitializer\sfrom\sthe\sextension\sloader.\s\sFix\sfor\s\ncheck-in\s[d6d449978245b4fa]. +D 2022-10-14T19:21:12.300 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -579,7 +579,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c aea5361767817f917b0f0f647a1f0b1621bd858938ae6ae545c3b6b9814b798f F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa -F src/loadext.c 853385cc7a604157e137585097949252d5d0c731768e16b044608e5c95c3614b +F src/loadext.c 8086232d10e51e183a7f64199815bad1c579896354db69435347665f62f481e9 F src/main.c b434b8e4aca38419962de3d1c55fb9279807ba6a8802998a427b1635dc8250f6 F src/malloc.c dfddca1e163496c0a10250cedeafaf56dff47673e0f15888fb0925340a8e3f90 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -2031,8 +2031,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2da677c45b643482eec39e4db7079c772760bc966dc71bf6c01658cc468f5823 -R bd5efaaf882b3ae0c03e0e5a6693b37e -U stephan -Z c8a3d40e488aded9faa8a7e4d2f0c359 +P a4423ca234453c14eb40db7fe5943f63b30fd9dc2207388e8a2966733a004e9d +R 657c639cfdb29ba1717741ed5fd9ad00 +U drh +Z ac4a2afa493b2e100552976a88fb6b99 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index de485659dc..0fb2923565 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4423ca234453c14eb40db7fe5943f63b30fd9dc2207388e8a2966733a004e9d \ No newline at end of file +565d74c36f2dd4612860c7ee561682c50db28cfa5384346528292019dbfdf86c \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index dd15d9a4da..421ccca804 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -508,7 +508,9 @@ static const sqlite3_api_routines sqlite3Apis = { 0, 0, #endif - sqlite3_db_name + sqlite3_db_name, + /* Version 3.40.0 and later */ + sqlite3_value_type }; /* True if x is the directory separator character From 879164ed74846a4bdfcd513c200c70debf0b9409 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 14 Oct 2022 19:30:34 +0000 Subject: [PATCH 5/5] Fix a problem with using the push-down optimization on compound SELECTs where component SELECTs use different collation sequences. dbsqlfuzz 11516f050100243e5a845f5a2b48a90ed2efaf2e. FossilOrigin-Name: ed14863dd72e35fa3a23320c3d5a8166515faea39a555c28a27b2d35e701eac4 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 33 +++++++++++++++++++++++++++------ test/collate5.test | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 7c2b8ac79b..df43c1449f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smissing\sinitializer\sfrom\sthe\sextension\sloader.\s\sFix\sfor\s\ncheck-in\s[d6d449978245b4fa]. -D 2022-10-14T19:21:12.300 +C Fix\sa\sproblem\swith\susing\sthe\spush-down\soptimization\son\scompound\sSELECTs\swhere\scomponent\sSELECTs\suse\sdifferent\scollation\ssequences.\sdbsqlfuzz\s11516f050100243e5a845f5a2b48a90ed2efaf2e. +D 2022-10-14T19:30:34.777 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -617,7 +617,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 546d6feb15ec69c1aafe9bb351a277cbb498fd5410e646add673acb805714960 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c bb18acf4eded647fef88d4d543c673874dbebff516fbeba90a85e6c13f2a58cd +F src/select.c b795f31a326564ffc448bb26f863c64f0796b2a5e0585a3358aff7877b72ed82 F src/shell.c.in 2915eaf22bda89ad6533851a051de4773c249185360fe1fc7b4477b8f9063b2c F src/sqlite.h.in d9c8a6243fc0a1c270d69db33758e34b810af3462f9bc5b4af113b347e07c69d F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -848,7 +848,7 @@ F test/collate1.test 71a6f27fdc93a92f14d8ab80c05e1937656a5a03197e1a10157314554d6 F test/collate2.test 471c6f74573382b89b0f8b88a05256faa52f7964f9e4799e76708a3b1ece6ba4 F test/collate3.test 89defc49983ddfbf0a0555aca8c0521a676f56a5 F test/collate4.test c953715fb498b87163e3e73dd94356bff1f317bd -F test/collate5.test 65d928034d30d2d263a80f6359f7549ee1598ec6 +F test/collate5.test b1dfeff239ea69ee9225832553f423d37a6184eb730cee06f6846ab4e3c6dbef F test/collate6.test 8be65a182abaac8011a622131486dafb8076e907 F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 F test/collate8.test cd9b3d3f999b8520ffaa7cc1647061fc5bab1334 @@ -2031,8 +2031,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a4423ca234453c14eb40db7fe5943f63b30fd9dc2207388e8a2966733a004e9d -R 657c639cfdb29ba1717741ed5fd9ad00 -U drh -Z ac4a2afa493b2e100552976a88fb6b99 +P 565d74c36f2dd4612860c7ee561682c50db28cfa5384346528292019dbfdf86c +R 75e7f9b91b017a681d1ae11ac5186379 +U dan +Z 42c84fae9e00b1c8a3d198c324edcd95 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0fb2923565..65ef760f0e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -565d74c36f2dd4612860c7ee561682c50db28cfa5384346528292019dbfdf86c \ No newline at end of file +ed14863dd72e35fa3a23320c3d5a8166515faea39a555c28a27b2d35e701eac4 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 2876f48ab8..e4424024c8 100644 --- a/src/select.c +++ b/src/select.c @@ -3750,6 +3750,7 @@ typedef struct SubstContext { int iNewTable; /* New table number */ int isOuterJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */ ExprList *pEList; /* Replacement expressions */ + ExprList *pCList; /* Collation sequences for replacement expr */ } SubstContext; /* Forward Declarations */ @@ -3791,9 +3792,10 @@ static Expr *substExpr( #endif { Expr *pNew; - Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr; + int iColumn = pExpr->iColumn; + Expr *pCopy = pSubst->pEList->a[iColumn].pExpr; Expr ifNullRow; - assert( pSubst->pEList!=0 && pExpr->iColumnpEList->nExpr ); + assert( pSubst->pEList!=0 && iColumnpEList->nExpr ); assert( pExpr->pRight==0 ); if( sqlite3ExprIsVector(pCopy) ){ sqlite3VectorErrorMsg(pSubst->pParse, pCopy); @@ -3831,11 +3833,16 @@ static Expr *substExpr( /* Ensure that the expression now has an implicit collation sequence, ** just as it did when it was a column of a view or sub-query. */ - if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE ){ - CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, pExpr); - pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, - (pColl ? pColl->zName : "BINARY") + { + CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pExpr); + CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, + pSubst->pCList->a[iColumn].pExpr ); + if( pNat!=pColl || (pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE) ){ + pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, + (pColl ? pColl->zName : "BINARY") + ); + } } ExprClearProperty(pExpr, EP_Collate); } @@ -4028,6 +4035,18 @@ static void renumberCursors( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +/* +** If pSel is not part of a compound SELECT, return a pointer to its +** expression list. Otherwise, return a pointer to the expression list +** of the leftmost SELECT in the compound. +*/ +static ExprList *findLeftmostExprlist(Select *pSel){ + while( pSel->pPrior ){ + pSel = pSel->pPrior; + } + return pSel->pEList; +} + #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* ** This routine attempts to flatten subqueries as a performance optimization. @@ -4580,6 +4599,7 @@ static int flattenSubquery( x.iNewTable = iNewParent; x.isOuterJoin = isOuterJoin; x.pEList = pSub->pEList; + x.pCList = findLeftmostExprlist(pSub); substSelect(&x, pParent, 0); } @@ -5062,6 +5082,7 @@ static int pushDownWhereTerms( x.iNewTable = pSrc->iCursor; x.isOuterJoin = 0; x.pEList = pSubq->pEList; + x.pCList = findLeftmostExprlist(pSubq); pNew = substExpr(&x, pNew); #ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){ diff --git a/test/collate5.test b/test/collate5.test index 5f8697ea51..71d4efe255 100644 --- a/test/collate5.test +++ b/test/collate5.test @@ -19,6 +19,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix collate5 + # # Tests are organised as follows: @@ -26,6 +28,7 @@ source $testdir/tester.tcl # collate5-2.* - Compound SELECT # collate5-3.* - ORDER BY on compound SELECT # collate5-4.* - GROUP BY +# collate5-5.* - Collation sequence cases # Create the collation sequence 'TEXT', purely for asthetic reasons. The # test cases in this script could just as easily use BINARY. @@ -289,4 +292,37 @@ do_test collate5-4.3 { } } {} +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 5.0 { + CREATE TABLE t1(a, b COLLATE nocase); + CREATE TABLE t2(c, d); + INSERT INTO t2 VALUES(1, 'bbb'); +} +do_execsql_test 5.1 { + SELECT * FROM ( + SELECT a, b FROM t1 UNION ALL SELECT c, d FROM t2 + ) WHERE b='BbB'; +} {1 bbb} + +reset_db +do_execsql_test 5.2 { + CREATE TABLE t1(a,b,c COLLATE NOCASE); + INSERT INTO t1 VALUES(NULL,'C','c'); + CREATE VIEW v2 AS + SELECT a,b,c FROM t1 INTERSECT SELECT a,b,b FROM t1 + WHERE 'eT"3qRkL+oJMJjQ9z0'>=b + ORDER BY a,b,c; +} + +do_execsql_test 5.3 { + SELECT * FROM v2; +} { {} C c } + +do_execsql_test 5.4 { + SELECT * FROM v2 WHERE c='c'; +} { {} C c } + + finish_test