From ad1285c5c0be7eb92cc44a3357be71507c3c07f2 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 16:39:07 +0000 Subject: [PATCH 1/5] Initial infrastructure for adding a mode to the OPFS VFS which causes implicit locks to be released ASAP, which increases concurrency at the cost of performance. FossilOrigin-Name: c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 --- ext/wasm/api/sqlite3-api-oo1.js | 5 +- ext/wasm/api/sqlite3-api-opfs.js | 1 + ext/wasm/api/sqlite3-opfs-async-proxy.js | 123 ++++++++++++++++------ ext/wasm/tests/opfs/concurrency/worker.js | 6 +- manifest | 21 ++-- manifest.uuid | 2 +- 6 files changed, 112 insertions(+), 46 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 02ce9c0ced..bc32ec1067 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -200,9 +200,8 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ dbCtorHelper.normalizeArgs = function(filename=':memory:',flags = 'c',vfs = null){ const arg = {}; - if(1===arguments.length && 'object'===typeof arguments[0]){ - const x = arguments[0]; - Object.keys(x).forEach((k)=>arg[k] = x[k]); + if(1===arguments.length && arguments[0] && 'object'===typeof arguments[0]){ + Object.assign(arg, arguments[0]); if(undefined===arg.flags) arg.flags = 'c'; if(undefined===arg.vfs) arg.vfs = null; if(undefined===arg.filename) arg.filename = ':memory:'; diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index deb4c923ab..53f68a1d58 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -348,6 +348,7 @@ const installOpfsVfs = function callee(options){ 'SQLITE_NOTFOUND', 'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE', + 'SQLITE_OPEN_MAIN_DB', 'SQLITE_OPEN_READONLY' ].forEach((k)=>{ if(undefined === (state.sq3Codes[k] = capi[k])){ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index c208932e17..cbe0549b3e 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -105,7 +105,7 @@ metrics.dump = ()=>{ */ const __openFiles = Object.create(null); /** - __autoLocks is a Set of sqlite3_file pointers (integers) which were + __implicitLocks is a Set of sqlite3_file pointers (integers) which were "auto-locked". i.e. those for which we obtained a sync access handle without an explicit xLock() call. Such locks will be released during db connection idle time, whereas a sync access @@ -117,7 +117,7 @@ const __openFiles = Object.create(null); penalty: speedtest1 benchmarks take up to 4x as long. By delaying the lock release until idle time, the hit is negligible. */ -const __autoLocks = new Set(); +const __implicitLocks = new Set(); /** Expects an OPFS file path. It gets resolved, such that ".." @@ -166,7 +166,7 @@ const closeSyncHandle = async (fh)=>{ const h = fh.syncHandle; delete fh.syncHandle; delete fh.xLock; - __autoLocks.delete(fh.fid); + __implicitLocks.delete(fh.fid); return h.close(); } }; @@ -190,10 +190,10 @@ const closeSyncHandleNoThrow = async (fh)=>{ }; /* Release all auto-locks. */ -const closeAutoLocks = async ()=>{ - if(__autoLocks.size){ +const releaseImplicitLocks = async ()=>{ + if(__implicitLocks.size){ /* Release all auto-locks. */ - for(const fid of __autoLocks){ + for(const fid of __implicitLocks){ const fh = __openFiles[fid]; await closeSyncHandleNoThrow(fh); log("Auto-unlocked",fid,fh.filenameAbs); @@ -201,6 +201,32 @@ const closeAutoLocks = async ()=>{ } }; +/** + If true, any routine which implicitly acquires a sync access handle + (i.e. an OPFS lock) will release that locks at the end of the call + which acquires it. If false, such "autolocks" are not released + until the VFS is idle for some brief amount of time. + + The benefit of enabling this is much higher concurrency. The + down-side is much-reduced performance (as much as a 4x decrease + in speedtest1). +*/ +state.defaultReleaseImplicitLocks = false; + +/** + An experiment in improving concurrency by freeing up implicit locks + sooner. This is known to impact performance dramatically but it has + also shown to improve concurrency considerably. + + If fh.releaseImplicitLocks is truthy and fh is in __implicitLocks, + this routine returns closeSyncHandleNoThrow(), else it is a no-op. +*/ +const releaseImplicitLock = async (fh)=>{ + if(fh.releaseImplicitLocks && __implicitLocks.has(fh.fid)){ + return closeSyncHandleNoThrow(fh); + } +}; + /** An error class specifically for use with getSyncHandle(), the goal of which is to eventually be able to distinguish unambiguously @@ -246,7 +272,7 @@ GetSyncHandleError.convertRc = (e,rc)=>{ still fails at that point it will give up and propagate the exception. */ -const getSyncHandle = async (fh)=>{ +const getSyncHandle = async (fh,opName)=>{ if(!fh.syncHandle){ const t = performance.now(); log("Acquiring sync handle for",fh.filenameAbs); @@ -262,20 +288,21 @@ const getSyncHandle = async (fh)=>{ }catch(e){ if(i === maxTries){ throw new GetSyncHandleError( - e, "Error getting sync handle.",maxTries, + e, "Error getting sync handle for",opName+"().",maxTries, "attempts failed.",fh.filenameAbs ); } - warn("Error getting sync handle. Waiting",ms, + warn("Error getting sync handle for",opName+"(). Waiting",ms, "ms and trying again.",fh.filenameAbs,e); - await closeAutoLocks(); + //await releaseImplicitLocks(); Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); } } - log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms'); + log("Got",opName+"() sync handle for",fh.filenameAbs, + 'in',performance.now() - t,'ms'); if(!fh.xLock){ - __autoLocks.add(fh.fid); - log("Auto-locked",fh.fid,fh.filenameAbs); + __implicitLocks.add(fh.fid); + log("Auto-locked for",opName+"()",fh.fid,fh.filenameAbs); } } return fh.syncHandle; @@ -409,7 +436,7 @@ const vfsAsyncImpls = { xClose: async function(fid/*sqlite3_file pointer*/){ const opName = 'xClose'; mTimeStart(opName); - __autoLocks.delete(fid); + __implicitLocks.delete(fid); const fh = __openFiles[fid]; let rc = 0; wTimeStart(opName); @@ -474,13 +501,14 @@ const vfsAsyncImpls = { wTimeStart('xFileSize'); try{ affirmLocked('xFileSize',fh); - const sz = await (await getSyncHandle(fh)).getSize(); + const sz = await (await getSyncHandle(fh,'xFileSize')).getSize(); state.s11n.serialize(Number(sz)); rc = 0; }catch(e){ state.s11n.storeException(2,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR); } + await releaseImplicitLock(fh); wTimeEnd(); storeAndNotify('xFileSize', rc); mTimeEnd(); @@ -495,8 +523,8 @@ const vfsAsyncImpls = { if( !fh.syncHandle ){ wTimeStart('xLock'); try { - await getSyncHandle(fh); - __autoLocks.delete(fid); + await getSyncHandle(fh,'xLock'); + __implicitLocks.delete(fid); }catch(e){ state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); @@ -511,7 +539,6 @@ const vfsAsyncImpls = { flags/*SQLITE_OPEN_...*/){ const opName = 'xOpen'; mTimeStart(opName); - const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags); const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); wTimeStart('xOpen'); try{ @@ -526,14 +553,8 @@ const vfsAsyncImpls = { return; } const hFile = await hDir.getFileHandle(filenamePart, {create}); - /** - wa-sqlite, at this point, grabs a SyncAccessHandle and - assigns it to the syncHandle prop of the file state - object, but only for certain cases and it's unclear why it - places that limitation on it. - */ wTimeEnd(); - __openFiles[fid] = Object.assign(Object.create(null),{ + const fh = Object.assign(Object.create(null),{ fid: fid, filenameAbs: filename, filenamePart: filenamePart, @@ -542,8 +563,47 @@ const vfsAsyncImpls = { sabView: state.sabFileBufView, readOnly: create ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), - deleteOnClose: deleteOnClose + deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags) }); + fh.releaseImplicitLocks = + state.defaultReleaseImplicitLocks + /* TODO: check URI flags for "opfs-auto-unlock". First we need to + reshape the API a bit to be able to pass those on to here + from the other half of the proxy. */; + /*if(fh.releaseImplicitLocks){ + console.warn("releaseImplicitLocks is ON for",fh); + }*/ + if(0 /* this block is modelled after something wa-sqlite + does but it leads to horrible contention on journal files. */ + && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ + /* sqlite does not lock these files, so go ahead and grab an OPFS + lock. + + Regarding "immutable": that flag is not _really_ applicable + here. It's intended for use on read-only media. If, + however, a file is opened with that flag but changes later + (which can happen if we _don't_ grab a sync handle here) + then sqlite may misbehave. + + Regarding "nolock": ironically, the nolock flag forces us + to lock the file up front. "nolock" tells sqlite to _not_ + use its locking API, but OPFS requires a lock to perform + most of the operations performed in this file. If we don't + grab that lock up front, another handle could end up grabbing + it and mutating the database out from under our nolocked'd + handle. In the interest of preventing corruption, at the cost + of decreased concurrency, we have to lock it for the duration + of this file handle. + + https://www.sqlite.org/uri.html + */ + fh.xLock = "atOpen"/* Truthy value to keep entry from getting + flagged as auto-locked. String value so + that we can easily distinguish is later + if needed. */; + await getSyncHandle(fh,'xOpen'); + } + __openFiles[fid] = fh; storeAndNotify(opName, 0); }catch(e){ wTimeEnd(); @@ -560,7 +620,7 @@ const vfsAsyncImpls = { try{ affirmLocked('xRead',fh); wTimeStart('xRead'); - nRead = (await getSyncHandle(fh)).read( + nRead = (await getSyncHandle(fh,'xRead')).read( fh.sabView.subarray(0, n), {at: Number(offset64)} ); @@ -575,6 +635,7 @@ const vfsAsyncImpls = { state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); } + await releaseImplicitLock(fh); storeAndNotify('xRead',rc); mTimeEnd(); }, @@ -603,12 +664,13 @@ const vfsAsyncImpls = { try{ affirmLocked('xTruncate',fh); affirmNotRO('xTruncate', fh); - await (await getSyncHandle(fh)).truncate(size); + await (await getSyncHandle(fh,'xTruncate')).truncate(size); }catch(e){ error("xTruncate():",e,fh); state.s11n.storeException(2,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); } + await releaseImplicitLock(fh); wTimeEnd(); storeAndNotify('xTruncate',rc); mTimeEnd(); @@ -640,7 +702,7 @@ const vfsAsyncImpls = { affirmLocked('xWrite',fh); affirmNotRO('xWrite', fh); rc = ( - n === (await getSyncHandle(fh)) + n === (await getSyncHandle(fh,'xWrite')) .write(fh.sabView.subarray(0, n), {at: Number(offset64)}) ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; @@ -649,6 +711,7 @@ const vfsAsyncImpls = { state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); } + await releaseImplicitLock(fh); wTimeEnd(); storeAndNotify('xWrite',rc); mTimeEnd(); @@ -783,7 +846,7 @@ const waitLoop = async function f(){ if('timed-out'===Atomics.wait( state.sabOPView, state.opIds.whichOp, 0, waitTime )){ - await closeAutoLocks(); + await releaseImplicitLocks(); continue; } const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 19b0a068e7..9b4898f4c3 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -43,7 +43,11 @@ self.sqlite3InitModule().then(async function(sqlite3){ } }; const run = async function(){ - db = new sqlite3.opfs.OpfsDb(dbName,'c'); + db = new sqlite3.oo1.DB({ + filename: 'file:'+dbName, + flags: 'c', + vfs: 'opfs' + }); sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); db.transaction((db)=>{ db.exec([ diff --git a/manifest b/manifest index 3ecb281109..d6e43cfa43 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sMakefile.in\sto\sinclude\snew\starget\s"sqlite3r.c".\sFor\sgenerating\s"sqlite3r.c"\sand\s"sqlite3r.h",\sversions\sof\sthe\samalgamation\sthat\sinclude\sthe\srecover\sextension.\sTo\sbuild\sthe\sshell\stool\sagainst\sthese\sfiles,\sadd\s-DSQLITE_HAVE_SQLITE3R. -D 2022-11-23T16:08:49.765 +C Initial\sinfrastructure\sfor\sadding\sa\smode\sto\sthe\sOPFS\sVFS\swhich\scauses\simplicit\slocks\sto\sbe\sreleased\sASAP,\swhich\sincreases\sconcurrency\sat\sthe\scost\sof\sperformance. +D 2022-11-23T16:39:07.866 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -501,12 +501,12 @@ F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b387181 F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 -F ext/wasm/api/sqlite3-api-oo1.js e9a83489bbb4838ce0aee46eaaa9350e0e25a5b926b565e4f5ae8e840e4fbaed -F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233c345a246f1ad4c26d3b +F ext/wasm/api/sqlite3-api-oo1.js e4df25e7fd1a0b67a9f3df9eea8cbcbcdecab55be481c903488a9e8dcaf356e4 +F ext/wasm/api/sqlite3-api-opfs.js 69a897eae705816a002f44e8a37693e2ceb7b3e32f9889df2302aeba7df24c70 F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 1ec10873f1d59d305f6f3b435c50a1b75d693d5fb739b226f3da46fcbb11261a +F ext/wasm/api/sqlite3-opfs-async-proxy.js 3142ed3a636d9a1a3f4793c6af6373996593c615a3a5fa29de59ba3e0ea45bee F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -554,7 +554,7 @@ F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff -F ext/wasm/tests/opfs/concurrency/worker.js afccb78082b57edb17d5aba0754c823772553395df6f1aed92f82b4d9e3c32de +F ext/wasm/tests/opfs/concurrency/worker.js cc43ea47bb59582523e3f27c2198247c4adca2fae9a891f84dd3bccfccef2833 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,9 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 220cc4c6399b772b4ece03305a41b437ef0654d586a8a1c3dc5e7606fd36d655 59a837cfc7f9f96509491c8fc45355d2e8892af25246955e22adec1cbf37327b -R 6b5395a59674684ae7409c1a67752bd1 -T +closed 59a837cfc7f9f96509491c8fc45355d2e8892af25246955e22adec1cbf37327b -U dan -Z 08a51ae8157c87edcee2b45d299f4155 +P 5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 +R 5ffa2a11b06c4a44d92d27455a2fc3e8 +U stephan +Z 70f5adf76d87ddf8a488ff2bcd290afb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4c19a5133e..54a2776a7a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 \ No newline at end of file +c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 \ No newline at end of file From e79cb67c3575d8254b36eba3f35dba2d62124e07 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 19:03:22 +0000 Subject: [PATCH 2/5] Add an experimental OPFS VFS-specific URI flag, opfs-unlock-asap, which tells the VFS to release implicit locks ASAP. This permits higher concurrency but hurts performance considerably. This may or may not be obsoleted by other concurrency-related experimentation. FossilOrigin-Name: d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 --- ext/wasm/api/sqlite3-api-opfs.js | 27 ++++++++++++++++++++++- ext/wasm/api/sqlite3-opfs-async-proxy.js | 25 +++++---------------- ext/wasm/tests/opfs/concurrency/worker.js | 2 +- manifest | 16 +++++++------- manifest.uuid | 2 +- 5 files changed, 41 insertions(+), 31 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 53f68a1d58..deb8bf04e8 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -355,6 +355,25 @@ const installOpfsVfs = function callee(options){ toss("Maintenance required: not found:",k); } }); + state.opfsFlags = Object.assign(Object.create(null),{ + /** + Flag for use with xOpen(). "opfs-unlock-asap=1" enables + this. See defaultUnlockAsap, below. + */ + OPFS_UNLOCK_ASAP: 0x01, + /** + If true, any async routine which implicitly acquires a sync + access handle (i.e. an OPFS lock) will release that locks at + the end of the call which acquires it. If false, such + "autolocks" are not released until the VFS is idle for some + brief amount of time. + + The benefit of enabling this is much higher concurrency. The + down-side is much-reduced performance (as much as a 4x decrease + in speedtest1). + */ + defaultUnlockAsap: false + }); /** Runs the given operation (by name) in the async worker @@ -845,9 +864,15 @@ const installOpfsVfs = function callee(options){ //xSleep is optionally defined below xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ mTimeStart('xOpen'); + let opfsFlags = 0; if(0===zName){ zName = randomFilename(); }else if('number'===typeof zName){ + if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){ + /* -----------------------^^^^^ MUST pass the untranslated + C-string here. */ + opfsFlags |= state.opfsFlags.OPFS_UNLOCK_ASAP; + } zName = wasm.cstringToJs(zName); } const fh = Object.create(null); @@ -855,7 +880,7 @@ const installOpfsVfs = function callee(options){ fh.filename = zName; fh.sab = new SharedArrayBuffer(state.fileBufferSize); fh.flags = flags; - const rc = opRun('xOpen', pFile, zName, flags); + const rc = opRun('xOpen', pFile, zName, flags, opfsFlags); if(!rc){ /* Recall that sqlite3_vfs::xClose() will be called, even on error, unless pFile->pMethods is NULL. */ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index cbe0549b3e..b14494a0c9 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -201,18 +201,6 @@ const releaseImplicitLocks = async ()=>{ } }; -/** - If true, any routine which implicitly acquires a sync access handle - (i.e. an OPFS lock) will release that locks at the end of the call - which acquires it. If false, such "autolocks" are not released - until the VFS is idle for some brief amount of time. - - The benefit of enabling this is much higher concurrency. The - down-side is much-reduced performance (as much as a 4x decrease - in speedtest1). -*/ -state.defaultReleaseImplicitLocks = false; - /** An experiment in improving concurrency by freeing up implicit locks sooner. This is known to impact performance dramatically but it has @@ -536,7 +524,8 @@ const vfsAsyncImpls = { mTimeEnd(); }, xOpen: async function(fid/*sqlite3_file pointer*/, filename, - flags/*SQLITE_OPEN_...*/){ + flags/*SQLITE_OPEN_...*/, + opfsFlags/*OPFS_...*/){ const opName = 'xOpen'; mTimeStart(opName); const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); @@ -566,13 +555,8 @@ const vfsAsyncImpls = { deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags) }); fh.releaseImplicitLocks = - state.defaultReleaseImplicitLocks - /* TODO: check URI flags for "opfs-auto-unlock". First we need to - reshape the API a bit to be able to pass those on to here - from the other half of the proxy. */; - /*if(fh.releaseImplicitLocks){ - console.warn("releaseImplicitLocks is ON for",fh); - }*/ + (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) + || state.opfsFlags.defaultUnlockAsap; if(0 /* this block is modelled after something wa-sqlite does but it leads to horrible contention on journal files. */ && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ @@ -887,6 +871,7 @@ navigator.storage.getDirectory().then(function(d){ state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); state.opIds = opt.opIds; state.sq3Codes = opt.sq3Codes; + state.opfsFlags = opt.opfsFlags; Object.keys(vfsAsyncImpls).forEach((k)=>{ if(!Number.isFinite(state.opIds[k])){ toss("Maintenance required: missing state.opIds[",k,"]"); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 9b4898f4c3..1a896e714e 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -44,7 +44,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ }; const run = async function(){ db = new sqlite3.oo1.DB({ - filename: 'file:'+dbName, + filename: 'file:'+dbName,//+'?opfs-unlock-asap=1'/*EXPERIMENTAL*/, flags: 'c', vfs: 'opfs' }); diff --git a/manifest b/manifest index d6e43cfa43..d9cc7cb9d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\sinfrastructure\sfor\sadding\sa\smode\sto\sthe\sOPFS\sVFS\swhich\scauses\simplicit\slocks\sto\sbe\sreleased\sASAP,\swhich\sincreases\sconcurrency\sat\sthe\scost\sof\sperformance. -D 2022-11-23T16:39:07.866 +C Add\san\sexperimental\sOPFS\sVFS-specific\sURI\sflag,\sopfs-unlock-asap,\swhich\stells\sthe\sVFS\sto\srelease\simplicit\slocks\sASAP.\sThis\spermits\shigher\sconcurrency\sbut\shurts\sperformance\sconsiderably.\sThis\smay\sor\smay\snot\sbe\sobsoleted\sby\sother\sconcurrency-related\sexperimentation. +D 2022-11-23T19:03:22.450 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,11 +502,11 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e4df25e7fd1a0b67a9f3df9eea8cbcbcdecab55be481c903488a9e8dcaf356e4 -F ext/wasm/api/sqlite3-api-opfs.js 69a897eae705816a002f44e8a37693e2ceb7b3e32f9889df2302aeba7df24c70 +F ext/wasm/api/sqlite3-api-opfs.js 23b5c51d7c48134eb5a2d23c1dca06315c738aa365f1c9620e649805c62e5781 F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 3142ed3a636d9a1a3f4793c6af6373996593c615a3a5fa29de59ba3e0ea45bee +F ext/wasm/api/sqlite3-opfs-async-proxy.js 20030993ccc04e42b4c7111db31d8f22ca02a00879f183a9067738d7bc9f10b9 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -554,7 +554,7 @@ F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff -F ext/wasm/tests/opfs/concurrency/worker.js cc43ea47bb59582523e3f27c2198247c4adca2fae9a891f84dd3bccfccef2833 +F ext/wasm/tests/opfs/concurrency/worker.js e1b10dc5d96117ac58f4eedde97a970816adc60e007f0a0f20a41d880ab59ca9 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5f135575b923cb59947667071c6af9ff14063c933cedf37d6c2a0a1b86c85032 -R 5ffa2a11b06c4a44d92d27455a2fc3e8 +P c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 +R 37d44560e41f4eab685876e5b8c66c33 U stephan -Z 70f5adf76d87ddf8a488ff2bcd290afb +Z 1cab17b5c5d56b08d5be52571107da20 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 54a2776a7a..8365ac9b1a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 \ No newline at end of file +d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 \ No newline at end of file From 056a71562f192c7fcd4ca73a3133b60dcf5a83fd Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Nov 2022 20:49:08 +0000 Subject: [PATCH 3/5] OPFS concurrency test: add a URL flag to enable/disable unlock-asap mode. FossilOrigin-Name: 1c1bf22eadae2a5a7d4358e7cdd22641c2efb9296f42e7376749293b3a58b114 --- ext/wasm/tests/opfs/concurrency/test.js | 6 +++++- ext/wasm/tests/opfs/concurrency/worker.js | 11 +++++++---- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index 27bc47b19d..cb9d4275be 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -63,6 +63,9 @@ options.interval = ( urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 750 ) || 750; + options.unlockAsap = ( + urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0 + ) || 0; const workers = []; workers.post = (type,...args)=>{ for(const w of workers) w.postMessage({type, payload:args}); @@ -92,12 +95,13 @@ } }; - stdout("Launching",options.workerCount,"workers..."); + stdout("Launching",options.workerCount,"workers. Options:",options); workers.uri = ( 'worker.js?' + 'sqlite3.dir='+options.sqlite3Dir + '&interval='+options.interval + '&opfs-verbose='+options.opfsVerbose + + '&opfs-unlock-asap='+options.unlockAsap ); for(let i = 0; i < options.workerCount; ++i){ stdout("Launching worker..."); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 1a896e714e..91aa0fa6fa 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -3,9 +3,12 @@ importScripts( ); self.sqlite3InitModule().then(async function(sqlite3){ const urlArgs = new URL(self.location.href).searchParams; - const wName = urlArgs.get('workerId') || Math.round(Math.random()*10000); + const options = { + workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000), + unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/ + }; const wPost = (type,...payload)=>{ - postMessage({type, worker: wName, payload}); + postMessage({type, worker: options.workerName, payload}); }; const stdout = (...args)=>wPost('stdout',...args); const stderr = (...args)=>wPost('stderr',...args); @@ -44,7 +47,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ }; const run = async function(){ db = new sqlite3.oo1.DB({ - filename: 'file:'+dbName,//+'?opfs-unlock-asap=1'/*EXPERIMENTAL*/, + filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, flags: 'c', vfs: 'opfs' }); @@ -66,7 +69,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ try{ db.exec({ sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)", - bind: [wName, new Date().getTime()] + bind: [options.workerName, new Date().getTime()] }); //stdout("Set",prefix); }catch(e){ diff --git a/manifest b/manifest index d9cc7cb9d0..a6b0539bab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sexperimental\sOPFS\sVFS-specific\sURI\sflag,\sopfs-unlock-asap,\swhich\stells\sthe\sVFS\sto\srelease\simplicit\slocks\sASAP.\sThis\spermits\shigher\sconcurrency\sbut\shurts\sperformance\sconsiderably.\sThis\smay\sor\smay\snot\sbe\sobsoleted\sby\sother\sconcurrency-related\sexperimentation. -D 2022-11-23T19:03:22.450 +C OPFS\sconcurrency\stest:\sadd\sa\sURL\sflag\sto\senable/disable\sunlock-asap\smode. +D 2022-11-23T20:49:08.427 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -553,8 +553,8 @@ F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b51073 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 -F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff -F ext/wasm/tests/opfs/concurrency/worker.js e1b10dc5d96117ac58f4eedde97a970816adc60e007f0a0f20a41d880ab59ca9 +F ext/wasm/tests/opfs/concurrency/test.js 9a937068b66a0cfbb9cb6833cb001ce22f9d0f8f765775e3456860b05db21797 +F ext/wasm/tests/opfs/concurrency/worker.js 2a275c4983016365cac18c9105f1ac7bd2adbc7ad89cc91b363d722f2bb55cb5 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c5b7a9715a13b696ab3ee965aa0a310f59b65f07cecd72faa2e3504bfd8eb632 -R 37d44560e41f4eab685876e5b8c66c33 +P d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 +R ff8c3df82102d217683515f7779a95c4 U stephan -Z 1cab17b5c5d56b08d5be52571107da20 +Z 269769e48cdf286c8fb19455cdae3a52 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8365ac9b1a..bb0d09842b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 \ No newline at end of file +1c1bf22eadae2a5a7d4358e7cdd22641c2efb9296f42e7376749293b3a58b114 \ No newline at end of file From df5d06d03eca407aa84f12fce477a0c210bc4375 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 24 Nov 2022 17:53:09 +0000 Subject: [PATCH 4/5] More work on the OPFS concurrency testing app. FossilOrigin-Name: c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 --- ext/wasm/api/sqlite3-api-opfs.js | 3 +++ ext/wasm/api/sqlite3-opfs-async-proxy.js | 28 +++++----------------- ext/wasm/tests/opfs/concurrency/index.html | 11 +++++---- ext/wasm/tests/opfs/concurrency/test.js | 28 ++++++++++++++++++---- ext/wasm/tests/opfs/concurrency/worker.js | 8 +++---- manifest | 20 ++++++++-------- manifest.uuid | 2 +- 7 files changed, 54 insertions(+), 46 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index deb8bf04e8..5537df89fb 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -1171,6 +1171,9 @@ const installOpfsVfs = function callee(options){ //TODO to support fiddle and worker1 db upload: //opfsUtil.createFile = function(absName, content=undefined){...} + //We have sqlite3.wasm.sqlite3_wasm_vfs_create_file() for this + //purpose but its interface and name are still under + //consideration. if(sqlite3.oo1){ opfsUtil.OpfsDb = function(...args){ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index b14494a0c9..8a3db9c648 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -558,33 +558,17 @@ const vfsAsyncImpls = { (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) || state.opfsFlags.defaultUnlockAsap; if(0 /* this block is modelled after something wa-sqlite - does but it leads to horrible contention on journal files. */ + does but it leads to immediate contention on journal files. */ && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ /* sqlite does not lock these files, so go ahead and grab an OPFS lock. - Regarding "immutable": that flag is not _really_ applicable - here. It's intended for use on read-only media. If, - however, a file is opened with that flag but changes later - (which can happen if we _don't_ grab a sync handle here) - then sqlite may misbehave. - - Regarding "nolock": ironically, the nolock flag forces us - to lock the file up front. "nolock" tells sqlite to _not_ - use its locking API, but OPFS requires a lock to perform - most of the operations performed in this file. If we don't - grab that lock up front, another handle could end up grabbing - it and mutating the database out from under our nolocked'd - handle. In the interest of preventing corruption, at the cost - of decreased concurrency, we have to lock it for the duration - of this file handle. - https://www.sqlite.org/uri.html */ - fh.xLock = "atOpen"/* Truthy value to keep entry from getting - flagged as auto-locked. String value so - that we can easily distinguish is later - if needed. */; + fh.xLock = "xOpen"/* Truthy value to keep entry from getting + flagged as auto-locked. String value so + that we can easily distinguish is later + if needed. */; await getSyncHandle(fh,'xOpen'); } __openFiles[fid] = fh; @@ -824,7 +808,7 @@ const waitLoop = async function f(){ to do other things. If this is too high (e.g. 500ms) then even two workers/tabs can easily run into locking errors. */ - const waitTime = 150; + const waitTime = 100; while(!flagAsyncShutdown){ try { if('timed-out'===Atomics.wait( diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html index a082dfe997..e19f6a8da6 100644 --- a/ext/wasm/tests/opfs/concurrency/index.html +++ b/ext/wasm/tests/opfs/concurrency/index.html @@ -24,10 +24,13 @@

URL flags: pass a number of workers using - the workers=N URL flag and the worker work interval - as interval=N (milliseconds). Enable OPFS VFS - verbosity with verbose=1-3 (output goes to the - dev console). + the workers=N URL flag. Set the time between each + workload with interval=N (milliseconds). Set the + number of worker iterations with iterations=N. + Enable OPFS VFS verbosity with verbose=1-3 (output + goes to the dev console). Enable/disable "unlock ASAP" mode + (higher concurrency, lower speed) + with unlock-asap=0-1.

Achtung: if it does not start to do anything within a couple of seconds, check the dev console: Chrome often fails with "cannot allocate diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index cb9d4275be..044d343745 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -56,13 +56,16 @@ options.sqlite3Dir = urlArgsJs.get('sqlite3.dir'); options.workerCount = ( urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3 - ) || 3; + ) || 4; options.opfsVerbose = ( urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1 ) || 1; options.interval = ( urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 750 - ) || 750; + ) || 1000; + options.iterations = ( + urlArgsHtml.has('iterations') ? +urlArgsHtml.get('iterations') : 10 + ) || 10; options.unlockAsap = ( urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0 ) || 0; @@ -70,15 +73,25 @@ workers.post = (type,...args)=>{ for(const w of workers) w.postMessage({type, payload:args}); }; - workers.loadedCount = 0; + workers.counts = {loaded: 0, passed: 0, failed: 0}; + const checkFinished = function(){ + if(workers.counts.passed + workers.counts.failed !== workers.length){ + return; + } + if(workers.counts.failed>0){ + logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s)."); + }else{ + logCss('tests-pass',"All",workers.length,"workers finished."); + } + }; workers.onmessage = function(msg){ msg = msg.data; const prefix = 'Worker #'+msg.worker+':'; switch(msg.type){ case 'loaded': stdout(prefix,"loaded"); - if(++workers.loadedCount === workers.length){ - stdout("All workers loaded. Telling them to run..."); + if(++workers.counts.loaded === workers.length){ + stdout("All",workers.length,"workers loaded. Telling them to run..."); workers.post('run'); } break; @@ -86,10 +99,14 @@ case 'stderr': stderr(prefix,...msg.payload); break; case 'error': stderr(prefix,"ERROR:",...msg.payload); break; case 'finished': + ++workers.counts.passed; logCss('tests-pass',prefix,...msg.payload); + checkFinished(); break; case 'failed': + ++workers.counts.failed; logCss('tests-fail',prefix,"FAILED:",...msg.payload); + checkFinished(); break; default: logCss('error',"Unhandled message type:",msg); break; } @@ -100,6 +117,7 @@ 'worker.js?' + 'sqlite3.dir='+options.sqlite3Dir + '&interval='+options.interval + + '&iterations='+options.iterations + '&opfs-verbose='+options.opfsVerbose + '&opfs-unlock-asap='+options.unlockAsap ); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 91aa0fa6fa..95fefa1954 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -46,10 +46,9 @@ self.sqlite3InitModule().then(async function(sqlite3){ } }; const run = async function(){ - db = new sqlite3.oo1.DB({ + db = new sqlite3.opfs.OpfsDb({ filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, - flags: 'c', - vfs: 'opfs' + flags: 'c' }); sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); db.transaction((db)=>{ @@ -59,7 +58,8 @@ self.sqlite3InitModule().then(async function(sqlite3){ ]); }); - const maxIterations = 10; + const maxIterations = + urlArgs.has('iterations') ? (+urlArgs.get('iterations') || 10) : 10; stdout("Starting interval-based db updates with delay of",interval.delay,"ms."); const doWork = async ()=>{ const tm = new Date().getTime(); diff --git a/manifest b/manifest index a6b0539bab..602f6943ac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sconcurrency\stest:\sadd\sa\sURL\sflag\sto\senable/disable\sunlock-asap\smode. -D 2022-11-23T20:49:08.427 +C More\swork\son\sthe\sOPFS\sconcurrency\stesting\sapp. +D 2022-11-24T17:53:09.937 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,11 +502,11 @@ F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34 F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5 F ext/wasm/api/sqlite3-api-oo1.js e4df25e7fd1a0b67a9f3df9eea8cbcbcdecab55be481c903488a9e8dcaf356e4 -F ext/wasm/api/sqlite3-api-opfs.js 23b5c51d7c48134eb5a2d23c1dca06315c738aa365f1c9620e649805c62e5781 +F ext/wasm/api/sqlite3-api-opfs.js e98a8bd67dea8c20b0ec17332698b44658a6fbc4be18dd87fab2ce05284a10d7 F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 20030993ccc04e42b4c7111db31d8f22ca02a00879f183a9067738d7bc9f10b9 +F ext/wasm/api/sqlite3-opfs-async-proxy.js d933d4a3c84e6dc614b57355fc9029645cfcdfbfde096ed42f4c979a6f60c18a F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -552,9 +552,9 @@ F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d96 F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d -F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 -F ext/wasm/tests/opfs/concurrency/test.js 9a937068b66a0cfbb9cb6833cb001ce22f9d0f8f765775e3456860b05db21797 -F ext/wasm/tests/opfs/concurrency/worker.js 2a275c4983016365cac18c9105f1ac7bd2adbc7ad89cc91b363d722f2bb55cb5 +F ext/wasm/tests/opfs/concurrency/index.html e8fec75ea6eddc600c8a382da7ea2579feece2263a2fb4417f2cf3e9d451744c +F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a +F ext/wasm/tests/opfs/concurrency/worker.js 0eff027cbd3a495acb2ac94f57ca9e4d21125ab9fda07d45f3701b0efe82d450 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d23c917013485ec2793125221f3936b05c39d6eca941629fb819b6b4aa714520 -R ff8c3df82102d217683515f7779a95c4 +P 1c1bf22eadae2a5a7d4358e7cdd22641c2efb9296f42e7376749293b3a58b114 +R b591bd74ceba5bd36575b86f818be346 U stephan -Z 269769e48cdf286c8fb19455cdae3a52 +Z f64db814b4972f9f3353d092900f9ddf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bb0d09842b..11f565569e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c1bf22eadae2a5a7d4358e7cdd22641c2efb9296f42e7376749293b3a58b114 \ No newline at end of file +c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 \ No newline at end of file From 743c9ec4a5f3f00a85ad99c582b012c189a0d34f Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 26 Nov 2022 20:12:02 +0000 Subject: [PATCH 5/5] Fix an incorrect assertion-fault in the "TREETRACE" debugging logic on select.c. The problem does not affect production builds. FossilOrigin-Name: f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/select.c | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f14f6424d5..a2d2f52121 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sVFS:\sadd\sthe\sopfs-unlock-asap=1\sURI\sflag\swhich\stells\sthe\sVFS\sto\srelease\simplicit\slocks\sASAP\sinstead\sof\sduring\sVFS\sidle\stime.\sThis\simproves\sconcurrency\snotably\sin\sthe\stest\sapp\sbut\sbrings\sa\ssignificant\sperformance\spenalty\sin\sspeedtest1\s(roughly\s4x\sslowdown).\sThis\sis\snot\sthe\sfinal\sword\sin\sOPFS\sconcurrency,\sbut\sgets\sus\sa\sstep\sfurther. -D 2022-11-26T15:24:58.092 +C Fix\san\sincorrect\sassertion-fault\sin\sthe\s"TREETRACE"\sdebugging\slogic\son\nselect.c.\s\sThe\sproblem\sdoes\snot\saffect\sproduction\sbuilds. +D 2022-11-26T20:12:02.257 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -641,7 +641,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 934c37455762579dcef5ce37f6abf04facaa61ef04cde207ac170b28c4780113 +F src/select.c c1eb8f3ee25152327f2e7e87db8cea549e57c104b63638bff4fc584d479c33f0 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2060,9 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 -R ef0be46e6ac8ee603582dd679fae7434 -T +closed c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 Closed\sby\sintegrate-merge. -U stephan -Z 3b8060cf3f2afe36aab99fa36bba89db +P 9542f9ce9e023b489e2d93661f719fb0751c1e28f72fded9d3c2156d5777e7b1 +R 581898efb4aefcef8555315298dbd7f7 +U drh +Z f6493a79b992a3696ede08a261d497d4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e97840e0fc..d4e96fca06 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9542f9ce9e023b489e2d93661f719fb0751c1e28f72fded9d3c2156d5777e7b1 \ No newline at end of file +f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e \ No newline at end of file diff --git a/src/select.c b/src/select.c index 2bedfb9da0..10498310d4 100644 --- a/src/select.c +++ b/src/select.c @@ -6259,7 +6259,7 @@ static void printAggInfo(AggInfo *pAggInfo){ "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" " iSorterColumn=%d %s\n", ii, pCol->pTab ? pCol->pTab->zName : "NULL", - pCol->iTable, pCol->iColumn, AggInfoColumnReg(pAggInfo,ii), + pCol->iTable, pCol->iColumn, pAggInfo->iFirstReg+ii, pCol->iSorterColumn, ii>=pAggInfo->nAccumulator ? "" : " Accumulator"); sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0);