From 5e8bb0aa13628208b7baedd720ba4aaaac2cc239 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 20 Sep 2022 08:27:57 +0000 Subject: [PATCH 1/5] An alternative messaging strategy for the OPFS VFS proxy which uses only SharedArrayBuffer and Atomics, instead of worker messages, for communication (only the initial one-time handshake during initialization uses worker messages). It runs speedtest1 approx. 15-20% faster but still 20-ish% slower than WASMFS. FossilOrigin-Name: a83ee3082d89439ea3ad5737e63e25bebb0f91895aca006ce5fecf5b93a2651a --- ext/wasm/GNUmakefile | 15 +-- ext/wasm/api/sqlite3-api-opfs.js | 60 ++++++----- ext/wasm/speedtest1-worker.html | 101 +++++++++--------- ext/wasm/sqlite3-opfs-async-proxy.js | 146 +++++++++++++++++---------- manifest | 21 ++-- manifest.uuid | 2 +- 6 files changed, 200 insertions(+), 145 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index e1b6784d38..ae416cb469 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -65,7 +65,7 @@ SQLITE_OPT = \ .PHONY: release release: - $(MAKE) 'emcc_opt=-Os -g3 -flto' + $(MAKE) "emcc_opt=-Os -g3 -flto" fiddle_opt=-Os # ^^^^^ target-specific vars, e.g.: # release: emcc_opt=... # apparently only work for file targets, not PHONY targets? @@ -432,19 +432,20 @@ push-fiddle: $(fiddle_files) # experimentation shows -O2 to be the clear winner in terms of speed. # Note that build times with anything higher than -O0 are somewhat # painful. + .PHONY: o0 o1 o2 o3 os oz o0: - $(MAKE) clean; $(MAKE) -e emcc_opt=-O0 + $((MAKE) clean; $(MAKE) -e "emcc_opt=-O0 -flto" fiddle_opt=-O0 o1: - $(MAKE) clean; $(MAKE) -e emcc_opt=-O1 + $(MAKE) clean; $(MAKE) -e "emcc_opt=-O1 -flto" fiddle_opt=-O1 o2: - $(MAKE) clean; $(MAKE) -e emcc_opt=-O2 + $(MAKE) clean; $(MAKE) -e "emcc_opt=-O2 -flto" fiddle_opt=-O2 o3: - $(MAKE) clean; $(MAKE) -e emcc_opt=-O3 + $(MAKE) clean; $(MAKE) -e "emcc_opt=-O3 -flto" fiddle_opt=-O3 os: - $(MAKE) clean; $(MAKE) -e emcc_opt=-Os + $(MAKE) clean; $(MAKE) -e "emcc_opt=-Os -flto" fiddle_opt=-Os oz: - $(MAKE) clean; $(MAKE) -e emcc_opt=-Oz + $(MAKE) clean; $(MAKE) -e "emcc_opt=-Oz -flto" fiddle_opt=-Oz ######################################################################## # Sub-makes... diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index c6b38fa93e..52777c5f3d 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -128,7 +128,6 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) // failure is, e.g., that the remote script is 404. promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); }; - const wMsg = (type,args)=>W.postMessage({type,args}); /** Generic utilities for working with OPFS. This will get filled out by the Promise setup and, on success, installed as sqlite3.opfs. @@ -203,7 +202,6 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) { let i = 0; state.opIds.whichOp = i++; - state.opIds.nothing = i++; state.opIds.xAccess = i++; state.rcIds.xAccess = i++; state.opIds.xClose = i++; @@ -228,8 +226,9 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) state.rcIds.xWrite = i++; state.opIds.mkdir = i++; state.rcIds.mkdir = i++; + state.opIds.xFileControl = i++; + state.rcIds.xFileControl = i++; state.sabOP = new SharedArrayBuffer(i * 4/*sizeof int32*/); - state.opIds.xFileControl = state.opIds.xSync /* special case */; opfsUtil.metrics.reset(); } @@ -260,12 +259,17 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) given operation's signature in the async API counterpart. */ const opRun = (op,...args)=>{ + const rcNdx = state.rcIds[op] || toss("Invalid rc ID:",op); + const opNdx = state.opIds[op] || toss("Invalid op ID:",op); + state.s11n.serialize(...args); + Atomics.store(state.sabOPView, rcNdx, -1); + Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); + Atomics.notify(state.sabOPView, state.opIds.whichOp) /* async thread will take over here */; const t = performance.now(); - Atomics.store(state.sabOPView, state.opIds[op], -1); - wMsg(op, args); - Atomics.wait(state.sabOPView, state.opIds[op], -1); + Atomics.wait(state.sabOPView, rcNdx, -1); + const rc = Atomics.load(state.sabOPView, rcNdx); metrics[op].wait += performance.now() - t; - return Atomics.load(state.sabOPView, state.opIds[op]); + return rc; }; const initS11n = ()=>{ @@ -297,11 +301,25 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) serialization for simplicy of implementation, but if that proves imperformant then a lower-level approach will be created. + + If passed "too much data" (more that the shared buffer size + it will either throw or truncate the data (not certain + which)). This routine is only intended for serializing OPFS + VFS arguments and (in at least one special case) result + values, and the buffer is sized to be able to comfortably + handle those. + + If passed no arguments then it zeroes out the serialization + state. */ state.s11n.serialize = function(...args){ - const json = jsonEncoder.encode(JSON.stringify(args)); - viewSz.setInt32(0, json.byteLength, state.littleEndian); - viewJson.set(json); + if(args.length){ + const json = jsonEncoder.encode(JSON.stringify(args)); + viewSz.setInt32(0, json.byteLength, state.littleEndian); + viewJson.set(json); + }else{ + viewSz.setInt32(0, 0, state.littleEndian); + } }; return state.s11n; }; @@ -552,9 +570,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) const vfsSyncWrappers = { xAccess: function(pVfs,zName,flags,pOut){ mTimeStart('xAccess'); - wasm.setMemValue( - pOut, (opRun('xAccess', wasm.cstringToJs(zName)) ? 0 : 1), 'i32' - ); + const rc = opRun('xAccess', wasm.cstringToJs(zName)); + wasm.setMemValue( pOut, (rc ? 0 : 1), 'i32' ); mTimeEnd(); return 0; }, @@ -686,22 +703,12 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) opfsUtil.deleteEntry = function(fsEntryName,recursive=false){ return 0===opRun('xDelete', fsEntryName, 0, recursive); }; - /** - Exactly like deleteEntry() but runs asynchronously. This is a - "fire and forget" operation: it does not return a promise - because the counterpart operation happens in another thread and - waiting on that result in a Promise would block the OPFS VFS - from acting until it completed. - */ - opfsUtil.deleteEntryAsync = function(fsEntryName,recursive=false){ - wMsg('xDeleteNoWait', [fsEntryName, 0, recursive]); - }; /** Synchronously creates the given directory name, recursively, in the OPFS filesystem. Returns true if it succeeds or the directory already exists, else false. */ - opfsUtil.mkdir = async function(absDirName){ + opfsUtil.mkdir = function(absDirName){ return 0===opRun('mkdir', absDirName); }; /** @@ -736,7 +743,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) features like getting a directory listing. */ - const sanityCheck = async function(){ + const sanityCheck = function(){ const scope = wasm.scopedAllocPush(); const sq3File = new sqlite3_file(); try{ @@ -791,6 +798,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); rc = wasm.getMemValue(pOut,'i32'); if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); + log("End of OPFS sanity checks."); }finally{ sq3File.dispose(); wasm.scopedAllocPop(scope); @@ -803,7 +811,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) switch(data.type){ case 'opfs-async-loaded': /*Pass our config and shared state on to the async worker.*/ - wMsg('opfs-async-init',state); + W.postMessage({type: 'opfs-async-init',args: state}); break; case 'opfs-async-inited':{ /*Indicates that the async partner has received the 'init', diff --git a/ext/wasm/speedtest1-worker.html b/ext/wasm/speedtest1-worker.html index fef673e5e2..bcbd473422 100644 --- a/ext/wasm/speedtest1-worker.html +++ b/ext/wasm/speedtest1-worker.html @@ -212,70 +212,71 @@ /* TODO? Flags which require values need custom UI controls and some of them make little sense here (e.g. --script FILE). */ - flags["autovacuum"] = "Enable AUTOVACUUM mode"; - flags["big-transactions"] = "Important for tests 410 and 510!"; - //flags["cachesize"] = "N Set the cache size to N"; - flags["checkpoint"] = "Run PRAGMA wal_checkpoint after each test case"; - flags["exclusive"] = "Enable locking_mode=EXCLUSIVE"; - flags["explain"] = "Like --sqlonly but with added EXPLAIN keywords"; - //flags["heap"] = "SZ MIN Memory allocator uses SZ bytes & min allocation MIN"; - flags["incrvacuum"] = "Enable incremenatal vacuum mode"; - //flags["journal"] = "M Set the journal_mode to M"; - //flags["key"] = "KEY Set the encryption key to KEY"; - //flags["lookaside"] = "N SZ Configure lookaside for N slots of SZ bytes each"; - flags["memdb"] = "Use an in-memory database"; - //flags["mmap"] = "SZ MMAP the first SZ bytes of the database file"; - flags["multithread"] = "Set multithreaded mode"; - flags["nomemstat"] = "Disable memory statistics"; - flags["nomutex"] = "Open db with SQLITE_OPEN_NOMUTEX"; - flags["nosync"] = "Set PRAGMA synchronous=OFF"; - flags["notnull"] = "Add NOT NULL constraints to table columns"; - //flags["output"] = "FILE Store SQL output in FILE"; - //flags["pagesize"] = "N Set the page size to N"; - //flags["pcache"] = "N SZ Configure N pages of pagecache each of size SZ bytes"; - //flags["primarykey"] = "Use PRIMARY KEY instead of UNIQUE where appropriate"; - //flags["repeat"] = "N Repeat each SELECT N times (default: 1)"; - flags["reprepare"] = "Reprepare each statement upon every invocation"; - //flags["reserve"] = "N Reserve N bytes on each database page"; - //flags["script"] = "FILE Write an SQL script for the test into FILE"; - flags["serialized"] = "Set serialized threading mode"; - flags["singlethread"] = "Set single-threaded mode - disables all mutexing"; - flags["sqlonly"] = "No-op. Only show the SQL that would have been run."; - flags["shrink"] = "memory Invoke sqlite3_db_release_memory() frequently."; - //flags["size"] = "N Relative test size. Default=100"; - flags["strict"] = "Use STRICT table where appropriate"; - flags["stats"] = "Show statistics at the end"; - //flags["temp"] = "N N from 0 to 9. 0: no temp table. 9: all temp tables"; - //flags["testset"] = "T Run test-set T (main, cte, rtree, orm, fp, debug)"; - flags["trace"] = "Turn on SQL tracing"; - //flags["threads"] = "N Use up to N threads for sorting"; + flags["--autovacuum"] = "Enable AUTOVACUUM mode"; + flags["--big-transactions"] = "Important for tests 410 and 510!"; + //flags["--cachesize"] = "N Set the cache size to N"; + flags["--checkpoint"] = "Run PRAGMA wal_checkpoint after each test case"; + flags["--exclusive"] = "Enable locking_mode=EXCLUSIVE"; + flags["--explain"] = "Like --sqlonly but with added EXPLAIN keywords"; + //flags["--heap"] = "SZ MIN Memory allocator uses SZ bytes & min allocation MIN"; + flags["--incrvacuum"] = "Enable incremenatal vacuum mode"; + //flags["--journal"] = "M Set the journal_mode to M"; + //flags["--key"] = "KEY Set the encryption key to KEY"; + //flags["--lookaside"] = "N SZ Configure lookaside for N slots of SZ bytes each"; + flags["--memdb"] = "Use an in-memory database"; + //flags["--mmap"] = "SZ MMAP the first SZ bytes of the database file"; + flags["--multithread"] = "Set multithreaded mode"; + flags["--nomemstat"] = "Disable memory statistics"; + flags["--nomutex"] = "Open db with SQLITE_OPEN_NOMUTEX"; + flags["--nosync"] = "Set PRAGMA synchronous=OFF"; + flags["--notnull"] = "Add NOT NULL constraints to table columns"; + //flags["--output"] = "FILE Store SQL output in FILE"; + //flags["--pagesize"] = "N Set the page size to N"; + //flags["--pcache"] = "N SZ Configure N pages of pagecache each of size SZ bytes"; + //flags["--primarykey"] = "Use PRIMARY KEY instead of UNIQUE where appropriate"; + //flags["--repeat"] = "N Repeat each SELECT N times (default: 1)"; + flags["--reprepare"] = "Reprepare each statement upon every invocation"; + //flags["--reserve"] = "N Reserve N bytes on each database page"; + //flags["--script"] = "FILE Write an SQL script for the test into FILE"; + flags["--serialized"] = "Set serialized threading mode"; + flags["--singlethread"] = "Set single-threaded mode - disables all mutexing"; + flags["--sqlonly"] = "No-op. Only show the SQL that would have been run."; + flags["--shrink"] = "memory Invoke sqlite3_db_release_memory() frequently."; + //flags["--size"] = "N Relative test size. Default=100"; + flags["--strict"] = "Use STRICT table where appropriate"; + flags["--stats"] = "Show statistics at the end"; + //flags["--temp"] = "N N from 0 to 9. 0: no temp table. 9: all temp tables"; + //flags["--testset"] = "T Run test-set T (main, cte, rtree, orm, fp, debug)"; + flags["--trace"] = "Turn on SQL tracing"; + //flags["--threads"] = "N Use up to N threads for sorting"; /* The core API's WASM build does not support UTF16, but in this app it's not an issue because the data are not crossing JS/WASM boundaries. */ - flags["utf16be"] = "Set text encoding to UTF-16BE"; - flags["utf16le"] = "Set text encoding to UTF-16LE"; - flags["verify"] = "Run additional verification steps."; - flags["without"] = "rowid Use WITHOUT ROWID where appropriate"; + flags["--utf16be"] = "Set text encoding to UTF-16BE"; + flags["--utf16le"] = "Set text encoding to UTF-16LE"; + flags["--verify"] = "Run additional verification steps."; + flags["--without"] = "rowid Use WITHOUT ROWID where appropriate"; const preselectedFlags = [ - 'big-transactions', - 'singlethread' + '--big-transactions', + '--singlethread' ]; - if('opfs'!==urlParams.get('vfs')){ - preselectedFlags.push('memdb'); + if(urlParams.has('flags')){ + preselectedFlags.push(...urlParams.get('flags').split(',')); + } + if('opfs'!==urlParams.get('vfs')){ + preselectedFlags.push('--memdb'); } - Object.keys(flags).sort().forEach(function(f){ const opt = document.createElement('option'); eFlags.appendChild(opt); - const lbl = nbspPad('--'+f)+flags[f]; + const lbl = nbspPad(f)+flags[f]; //opt.innerText = lbl; opt.innerHTML = lbl; - opt.value = '--'+f; + opt.value = f; if(preselectedFlags.indexOf(f) >= 0) opt.selected = true; - }); - + }); const cbReverseLog = E('#cb-reverse-log-order'); const lblReverseLog = E('#lbl-reverse-log-order'); if(cbReverseLog.checked){ diff --git a/ext/wasm/sqlite3-opfs-async-proxy.js b/ext/wasm/sqlite3-opfs-async-proxy.js index 19d71512c9..a2e0dc5233 100644 --- a/ext/wasm/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/sqlite3-opfs-async-proxy.js @@ -63,7 +63,7 @@ const error = (...args)=>logImpl(0, ...args); const metrics = Object.create(null); metrics.reset = ()=>{ let k; - const r = (m)=>(m.count = m.time = 0); + const r = (m)=>(m.count = m.time = m.wait = 0); for(k in state.opIds){ r(metrics[k] = Object.create(null)); } @@ -74,11 +74,15 @@ metrics.dump = ()=>{ const m = metrics[k]; n += m.count; t += m.time; + w += m.wait; m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; } console.log(self.location.href, - "metrics for",self.location.href,":",metrics, - "\nTotal of",n,"op(s) for",t,"ms"); + "metrics for",self.location.href,":\n", + JSON.stringify(metrics,0,2) + /*dev console can't expand this object!*/, + "\nTotal of",n,"op(s) for",t,"ms", + "approx",w,"ms spent waiting on OPFS APIs."); }; warn("This file is very much experimental and under construction.", @@ -130,9 +134,9 @@ const getDirForPath = async function f(absFilename, createDirs = false){ and then Atomics.notify()'s it. */ const storeAndNotify = (opName, value)=>{ - log(opName+"() is notify()ing w/ value:",value); - Atomics.store(state.sabOPView, state.opIds[opName], value); - Atomics.notify(state.sabOPView, state.opIds[opName]); + log(opName+"() => notify(",state.rcIds[opName],",",value,")"); + Atomics.store(state.sabOPView, state.rcIds[opName], value); + Atomics.notify(state.sabOPView, state.rcIds[opName]); }; /** @@ -155,6 +159,17 @@ const mTimeStart = (op)=>{ const mTimeEnd = ()=>( metrics[opTimer.op].time += performance.now() - opTimer.start ); +const waitTimer = Object.create(null); +waitTimer.op = undefined; +waitTimer.start = undefined; +const wTimeStart = (op)=>{ + waitTimer.start = performance.now(); + waitTimer.op = op; + //metrics[op] || toss("Maintenance required: missing metrics for",op); +}; +const wTimeEnd = ()=>( + metrics[waitTimer.op].wait += performance.now() - waitTimer.start +); /** Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods @@ -163,17 +178,20 @@ const mTimeEnd = ()=>( */ const vfsAsyncImpls = { mkdir: async function(dirname){ + mTimeStart('mkdir'); let rc = 0; + wTimeStart('mkdir'); try { await getDirForPath(dirname+"/filepart", true); }catch(e){ //error("mkdir failed",filename, e.message); rc = state.sq3Codes.SQLITE_IOERR; } + wTimeEnd(); storeAndNotify('mkdir', rc); + mTimeEnd(); }, xAccess: async function(filename){ - log("xAccess(",arguments[0],")"); mTimeStart('xAccess'); /* OPFS cannot support the full range of xAccess() queries sqlite3 calls for. We can essentially just tell if the file is @@ -187,20 +205,23 @@ const vfsAsyncImpls = { accessible, non-0 means not accessible. */ let rc = 0; + wTimeStart('xAccess'); try{ const [dh, fn] = await getDirForPath(filename); await dh.getFileHandle(fn); }catch(e){ rc = state.sq3Codes.SQLITE_IOERR; } + wTimeEnd(); storeAndNotify('xAccess', rc); mTimeEnd(); }, xClose: async function(fid){ const opName = 'xClose'; mTimeStart(opName); - log(opName+"(",arguments[0],")"); const fh = __openFiles[fid]; + let rc = 0; + wTimeStart('xClose'); if(fh){ delete __openFiles[fid]; if(fh.accessHandle) await fh.accessHandle.close(); @@ -208,10 +229,11 @@ const vfsAsyncImpls = { try{ await fh.dirHandle.removeEntry(fh.filenamePart) } catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) } } - storeAndNotify(opName, 0); }else{ - storeAndNotify(opName, state.sq3Codes.SQLITE_NOFOUND); + rc = state.sq3Codes.SQLITE_NOTFOUND; } + wTimeEnd(); + storeAndNotify(opName, rc); mTimeEnd(); }, xDelete: async function(...args){ @@ -233,12 +255,11 @@ const vfsAsyncImpls = { presumably it will fail if the dir is not empty and that flag is false. */ - log("xDelete(",arguments[0],")"); let rc = 0; + wTimeStart('xDelete'); try { while(filename){ const [hDir, filenamePart] = await getDirForPath(filename, false); - //log("Removing:",hDir, filenamePart); if(!filenamePart) break; await hDir.removeEntry(filenamePart, {recursive}); if(0x1234 !== syncDir) break; @@ -252,13 +273,14 @@ const vfsAsyncImpls = { //error("Delete failed",filename, e.message); rc = state.sq3Codes.SQLITE_IOERR_DELETE; } + wTimeEnd(); return rc; }, xFileSize: async function(fid){ mTimeStart('xFileSize'); - log("xFileSize(",arguments,")"); const fh = __openFiles[fid]; let sz; + wTimeStart('xFileSize'); try{ sz = await fh.accessHandle.getSize(); state.s11n.serialize(Number(sz)); @@ -267,15 +289,16 @@ const vfsAsyncImpls = { error("xFileSize():",e, fh); sz = state.sq3Codes.SQLITE_IOERR; } + wTimeEnd(); storeAndNotify('xFileSize', sz); mTimeEnd(); }, xOpen: async function(fid/*sqlite3_file pointer*/, filename, flags){ const opName = 'xOpen'; mTimeStart(opName); - log(opName+"(",arguments[0],")"); const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags); const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); + wTimeStart('xOpen'); try{ let hDir, filenamePart; try { @@ -283,6 +306,7 @@ const vfsAsyncImpls = { }catch(e){ storeAndNotify(opName, state.sql3Codes.SQLITE_NOTFOUND); mTimeEnd(); + wTimeEnd(); return; } const hFile = await hDir.getFileHandle(filenamePart, {create}); @@ -294,6 +318,7 @@ const vfsAsyncImpls = { places that limitation on it. */ fobj.accessHandle = await hFile.createSyncAccessHandle(); + wTimeEnd(); __openFiles[fid] = fobj; fobj.filenameAbs = filename; fobj.filenamePart = filenamePart; @@ -304,6 +329,7 @@ const vfsAsyncImpls = { fobj.deleteOnClose = deleteOnClose; storeAndNotify(opName, 0); }catch(e){ + wTimeEnd(); error(opName,e); storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR); } @@ -311,14 +337,15 @@ const vfsAsyncImpls = { }, xRead: async function(fid,n,offset){ mTimeStart('xRead'); - log("xRead(",arguments[0],")"); let rc = 0; try{ const fh = __openFiles[fid]; + wTimeStart('xRead'); const nRead = fh.accessHandle.read( fh.sabView.subarray(0, n), {at: Number(offset)} ); + wTimeEnd(); if(nRead < n){/* Zero-fill remaining bytes */ fh.sabView.fill(0, nRead, n); rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; @@ -332,17 +359,20 @@ const vfsAsyncImpls = { }, xSync: async function(fid,flags/*ignored*/){ mTimeStart('xSync'); - log("xSync(",arguments[0],")"); const fh = __openFiles[fid]; - if(!fh.readOnly && fh.accessHandle) await fh.accessHandle.flush(); + if(!fh.readOnly && fh.accessHandle){ + wTimeStart('xSync'); + await fh.accessHandle.flush(); + wTimeEnd(); + } storeAndNotify('xSync',0); mTimeEnd(); }, xTruncate: async function(fid,size){ mTimeStart('xTruncate'); - log("xTruncate(",arguments[0],")"); let rc = 0; const fh = __openFiles[fid]; + wTimeStart('xTruncate'); try{ affirmNotRO('xTruncate', fh); await fh.accessHandle.truncate(size); @@ -350,13 +380,14 @@ const vfsAsyncImpls = { error("xTruncate():",e,fh); rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE; } + wTimeEnd(); storeAndNotify('xTruncate',rc); mTimeEnd(); }, xWrite: async function(fid,n,offset){ mTimeStart('xWrite'); - log("xWrite(",arguments[0],")"); let rc; + wTimeStart('xWrite'); try{ const fh = __openFiles[fid]; affirmNotRO('xWrite', fh); @@ -367,13 +398,14 @@ const vfsAsyncImpls = { }catch(e){ error("xWrite():",e,fh); rc = state.sq3Codes.SQLITE_IOERR_WRITE; + }finally{ + wTimeEnd(); } storeAndNotify('xWrite',rc); mTimeEnd(); } }; - const initS11n = ()=>{ // Achtung: this code is 100% duplicated in the other half of this proxy! if(state.s11n) return state.s11n; @@ -403,46 +435,69 @@ const initS11n = ()=>{ serialization for simplicy of implementation, but if that proves imperformant then a lower-level approach will be created. + + If passed "too much data" (more that the shared buffer size + it will either throw or truncate the data (not certain + which)). This routine is only intended for serializing OPFS + VFS arguments and (in at least one special case) result + values, and the buffer is sized to be able to comfortably + handle those. + + If passed no arguments then it zeroes out the serialization + state. */ state.s11n.serialize = function(...args){ - const json = jsonEncoder.encode(JSON.stringify(args)); - viewSz.setInt32(0, json.byteLength, state.littleEndian); - viewJson.set(json); + if(args.length){ + const json = jsonEncoder.encode(JSON.stringify(args)); + viewSz.setInt32(0, json.byteLength, state.littleEndian); + viewJson.set(json); + }else{ + viewSz.setInt32(0, 0, state.littleEndian); + } }; return state.s11n; }; -const waitLoop = function(){ +const waitLoop = async function f(){ const opHandlers = Object.create(null); - for(let k of Object.keys(state.opIds)){ + for(let k of Object.keys(state.rcIds)){ const o = Object.create(null); opHandlers[state.opIds[k]] = o; o.key = k; + o.f = vfsAsyncImpls[k];// || toss("No vfsAsyncImpls[",k,"]"); } - const sabOP = state.sabOP; - for(;;){ + let metricsTimer = self.location.port>=1024 ? performance.now() : 0; + // ^^^ in dev environment, dump out these metrics one time after a delay. + while(true){ try { - Atomics.store(sabOP, state.opIds.whichOp, 0); - Atomic.wait(sabOP, state.opIds.whichOp); - const opId = Atomics.load(sabOP, state.opIds.whichOp); + if('timed-out'===Atomics.wait(state.sabOPView, state.opIds.whichOp, 0, 150)){ + continue; + } + const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); + Atomics.store(state.sabOPView, state.opIds.whichOp, 0); const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); const args = state.s11n.deserialize(); - log("whichOp =",opId,hnd,args); - const rc = 0/*TODO: run op*/; - Atomics.store(sabOP, state.rcIds[hnd.key], rc); - Atomics.notify(sabOP, state.rcIds[hnd.key]); + //warn("waitLoop() whichOp =",opId, hnd, args); + if(hnd.f) await hnd.f(...args); + else error("Missing callback for opId",opId); }catch(e){ error('in waitLoop():',e.message); + }finally{ + // We can't call metrics.dump() from the dev console because this + // thread is continually tied up in Atomics.wait(), so let's + // do, for dev purposes only, a dump one time after 60 seconds. + if(metricsTimer && (performance.now() > metricsTimer + 60000)){ + metrics.dump(); + metricsTimer = 0; + } } - } + }; }; navigator.storage.getDirectory().then(function(d){ const wMsg = (type)=>postMessage({type}); state.rootDir = d; - log("state.rootDir =",state.rootDir); - self.onmessage = async function({data}){ - log("self.onmessage()",data); + self.onmessage = function({data}){ switch(data.type){ case 'opfs-async-init':{ /* Receive shared state from synchronous partner */ @@ -469,20 +524,7 @@ navigator.storage.getDirectory().then(function(d){ metrics.reset(); log("init state",state); wMsg('opfs-async-inited'); - break; - } - default:{ - let err; - const m = vfsAsyncImpls[data.type] || toss("Unknown message type:",data.type); - try { - await m(...data.args).catch((e)=>err=e); - }catch(e){ - err = e; - } - if(err){ - error("Error handling",data.type+"():",e); - storeAndNotify(data.type, state.sq3Codes.SQLITE_ERROR); - } + waitLoop(); break; } } diff --git a/manifest b/manifest index 13980df6d3..dadba80e1a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sVFS:\sfurther\sinternal\srefactoring\stowards\sexperimenting\swith\sa\snew\scomms\smodel. -D 2022-09-20T03:31:02.139 +C An\salternative\smessaging\sstrategy\sfor\sthe\sOPFS\sVFS\sproxy\swhich\suses\sonly\sSharedArrayBuffer\sand\sAtomics,\sinstead\sof\sworker\smessages,\sfor\scommunication\s(only\sthe\sinitial\sone-time\shandshake\sduring\sinitialization\suses\sworker\smessages).\sIt\sruns\sspeedtest1\sapprox.\s15-20%\sfaster\sbut\sstill\s20-ish%\sslower\sthan\sWASMFS. +D 2022-09-20T08:27:57.073 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -474,7 +474,7 @@ F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 -F ext/wasm/GNUmakefile b6a5b642e8b3e587d3edcfeb6b6275acbe4730293f4ad46c4997cd932d57aec5 +F ext/wasm/GNUmakefile b65cd280059febd4f034b856139fde52f7edb7326133f7e0588b8efcf396709c F ext/wasm/README.md e1ee1e7c321c6a250bf78a84ca6f5882890a237a450ba5a0649c7a8399194c52 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 150a793a47205b8009ac934f3b6d6ebf67b965c072339aaa25ce808a19e116cc F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 @@ -484,7 +484,7 @@ F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77 F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30 F ext/wasm/api/sqlite3-api-oo1.js 2d13dddf0d2b4168a9249f124134d37924331e5b55e05dba18b6d661fbeefe48 -F ext/wasm/api/sqlite3-api-opfs.js 351459d571166ff4cebaccd6b8aad2b0fe5eac54a8c777ba52c31c931a3eb2e2 +F ext/wasm/api/sqlite3-api-opfs.js 1df64b2a11b7f71ba5c7a5807b4d62727e2c712e537eb563caedc67c5a38d149 F ext/wasm/api/sqlite3-api-prologue.js 0d2639387b94c30f492d4aea6e44fb7b16720808678464559458fd2ae3759655 F ext/wasm/api/sqlite3-api-worker1.js ee4cf149cbacb63d06b536674f822aa5088b7e022cdffc69f1f36cebe2f9fea0 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 @@ -513,13 +513,13 @@ F ext/wasm/jaccwabyt/jaccwabyt_test.exports 5ff001ef975c426ffe88d7d8a6e96ec725e5 F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06 F ext/wasm/scratchpad-wasmfs-main.js f0836e3576df7a89390d777bb53e142e559e8a79becfb2a5a976490b05a1c4fa F ext/wasm/speedtest1-wasmfs.html 9d8cd19eab8854d17f7129aa11607cae6f6d9857c505a4aef13000588583d93e -F ext/wasm/speedtest1-worker.html 7bd3ede33e08067f59e8d6e3dc25016fc930222b059d66b2554203bf262aba02 +F ext/wasm/speedtest1-worker.html ede59f2c1884bf72e3d650064604b48703c81848250b19b8063d260aa3a2201d F ext/wasm/speedtest1-worker.js 11e7f68cedd2a83b0e638f94c1d2f58406ba672a7e88b66bff5d4f4284e8ba16 F ext/wasm/speedtest1.html 512addeb3c27c94901178b7bcbde83a6f95c093f9ebe16a2959a0aa0d828cf1d F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 -F ext/wasm/sqlite3-opfs-async-proxy.js 462081970a6a46d9b2c386474aacad2d81e6629bb554d6cad5c58515f08c8a38 +F ext/wasm/sqlite3-opfs-async-proxy.js 038ecd8558abc3f46cfedd560093fef4d460af8c0d0009ab84f2abdc10916a6a F ext/wasm/sqlite3-worker1-promiser.js 4fd0465688a28a75f1d4ee4406540ba494f49844e3cad0670d0437a001943365 F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5 @@ -2026,8 +2026,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d4d63e4580ad8d497310608175308c03c517e051d7865cb66aa0b10356612d7d -R 7fef73d3d6085edd0d87db08b80a36c1 +P 5ca412ced24b4e3af5f467e710a597ed440badf7b8335346aade11d3cad3d1a1 +R eef8f82c266e71b1fc83e097c56f095e +T *branch * opfs-proxy-atomics +T *sym-opfs-proxy-atomics * +T -sym-fiddle-opfs * Cancelled\sby\sbranch. U stephan -Z d9dc6ab88cc2471cfc1f140af8fec063 +Z 1c049de5a4413363c0d5cd656309295d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b2152187c4..cf2c4e7642 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ca412ced24b4e3af5f467e710a597ed440badf7b8335346aade11d3cad3d1a1 \ No newline at end of file +a83ee3082d89439ea3ad5737e63e25bebb0f91895aca006ce5fecf5b93a2651a \ No newline at end of file From c9e2602e8f55d7a40d36ef6080529c7fa96ed11f Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 20 Sep 2022 10:11:52 +0000 Subject: [PATCH 2/5] Remove extraneous result-code SAB slots. Doc additions. FossilOrigin-Name: 88de20712bbe3e11ea74af95edc34e9ab9f346f9aa0a30931e5a9e1f96ce57b0 --- ext/wasm/GNUmakefile | 14 +-- ext/wasm/api/sqlite3-api-opfs.js | 133 ++++++++++++++------------- ext/wasm/sqlite3-opfs-async-proxy.js | 15 +-- manifest | 19 ++-- manifest.uuid | 2 +- 5 files changed, 96 insertions(+), 87 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index ae416cb469..cff5af5944 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -434,18 +434,20 @@ push-fiddle: $(fiddle_files) # painful. .PHONY: o0 o1 o2 o3 os oz +o-xtra := +#o-xtra += -flto o0: - $((MAKE) clean; $(MAKE) -e "emcc_opt=-O0 -flto" fiddle_opt=-O0 + $(MAKE) clean; $(MAKE) -e "emcc_opt=-O0 $(o-xtra)" fiddle_opt=-O0 o1: - $(MAKE) clean; $(MAKE) -e "emcc_opt=-O1 -flto" fiddle_opt=-O1 + $(MAKE) clean; $(MAKE) -e "emcc_opt=-O1 $(o-xtra)" fiddle_opt=-O1 o2: - $(MAKE) clean; $(MAKE) -e "emcc_opt=-O2 -flto" fiddle_opt=-O2 + $(MAKE) clean; $(MAKE) -e "emcc_opt=-O2 $(o-xtra)" fiddle_opt=-O2 o3: - $(MAKE) clean; $(MAKE) -e "emcc_opt=-O3 -flto" fiddle_opt=-O3 + $(MAKE) clean; $(MAKE) -e "emcc_opt=-O3 $(o-xtra)" fiddle_opt=-O3 os: - $(MAKE) clean; $(MAKE) -e "emcc_opt=-Os -flto" fiddle_opt=-Os + $(MAKE) clean; $(MAKE) -e "emcc_opt=-Os $(o-xtra)" fiddle_opt=-Os oz: - $(MAKE) clean; $(MAKE) -e "emcc_opt=-Oz -flto" fiddle_opt=-Oz + $(MAKE) clean; $(MAKE) -e "emcc_opt=-Oz $(o-xtra)" fiddle_opt=-Oz ######################################################################## # Sub-makes... diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 52777c5f3d..a0630d8e76 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -165,6 +165,38 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) } }/*metrics*/; + const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; + const dVfs = pDVfs + ? new sqlite3_vfs(pDVfs) + : null /* dVfs will be null when sqlite3 is built with + SQLITE_OS_OTHER. Though we cannot currently handle + that case, the hope is to eventually be able to. */; + const opfsVfs = new sqlite3_vfs(); + const opfsIoMethods = new sqlite3_io_methods(); + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = 1024/*sure, why not?*/; + opfsVfs.$zName = wasm.allocCString("opfs"); + // All C-side memory of opfsVfs is zeroed out, but just to be explicit: + opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; + opfsVfs.ondispose = [ + '$zName', opfsVfs.$zName, + 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null), + 'cleanup opfsIoMethods', ()=>opfsIoMethods.dispose() + ]; + /** + Pedantic sidebar about opfsVfs.ondispose: the entries in that array + are items to clean up when opfsVfs.dispose() is called, but in this + environment it will never be called. The VFS instance simply + hangs around until the WASM module instance is cleaned up. We + "could" _hypothetically_ clean it up by "importing" an + sqlite3_os_end() impl into the wasm build, but the shutdown order + of the wasm engine and the JS one are undefined so there is no + guaranty that the opfsVfs instance would be available in one + environment or the other when sqlite3_os_end() is called (_if_ it + gets called at all in a wasm build, which is undefined). + */ + /** State which we send to the async-api Worker or share with it. This object must initially contain only cloneable or sharable @@ -187,54 +219,64 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) const state = Object.create(null); state.littleEndian = true; state.verbose = options.verbose; + /* Size of file I/O buffer block. 64k = max sqlite3 page size. */ state.fileBufferSize = - 1024 * 64 /* size of aFileHandle.sab. 64k = max sqlite3 page - size. */; + 1024 * 64; state.sabS11nOffset = state.fileBufferSize; - state.sabS11nSize = 2048; + /** + The size of the block in our SAB for serializing arguments and + result values. Need to be large enough to hold serialized + values of any of the proxied APIs. Filenames are the largest + part but are limited to opfsVfs.$mxPathname bytes. + */ + state.sabS11nSize = opfsVfs.$mxPathname * 2; + /** + The SAB used for all data I/O (files and arg/result s11n). + */ state.sabIO = new SharedArrayBuffer( - state.fileBufferSize - + state.sabS11nSize/* arg/result serialization block */ + state.fileBufferSize/* file i/o block */ + + state.sabS11nSize/* argument/result serialization block */ ); state.opIds = Object.create(null); - state.rcIds = Object.create(null); const metrics = Object.create(null); { + /* Indexes for use in our SharedArrayBuffer... */ let i = 0; + /* SAB slot used to communicate which operation is desired + between both workers. This worker writes to it and the other + listens for changes. */ state.opIds.whichOp = i++; + /* Slot for storing return values. This work listens to that + slot and the other worker writes to it. */ + state.opIds.rc = i++; + /* Each function gets an ID which this worker writes to + the whichOp slot. The async-api worker uses Atomic.wait() + on the whichOp slot to figure out which operation to run + next. */ state.opIds.xAccess = i++; - state.rcIds.xAccess = i++; state.opIds.xClose = i++; - state.rcIds.xClose = i++; state.opIds.xDelete = i++; - state.rcIds.xDelete = i++; state.opIds.xDeleteNoWait = i++; - state.rcIds.xDeleteNoWait = i++; state.opIds.xFileSize = i++; - state.rcIds.xFileSize = i++; state.opIds.xOpen = i++; - state.rcIds.xOpen = i++; state.opIds.xRead = i++; - state.rcIds.xRead = i++; state.opIds.xSleep = i++; - state.rcIds.xSleep = i++; state.opIds.xSync = i++; - state.rcIds.xSync = i++; state.opIds.xTruncate = i++; - state.rcIds.xTruncate = i++; state.opIds.xWrite = i++; - state.rcIds.xWrite = i++; state.opIds.mkdir = i++; - state.rcIds.mkdir = i++; state.opIds.xFileControl = i++; - state.rcIds.xFileControl = i++; state.sabOP = new SharedArrayBuffer(i * 4/*sizeof int32*/); opfsUtil.metrics.reset(); } + /** + SQLITE_xxx constants to export to the async worker + counterpart... + */ state.sq3Codes = Object.create(null); state.sq3Codes._reverse = Object.create(null); - [ // SQLITE_xxx constants to export to the async worker counterpart... + [ 'SQLITE_ERROR', 'SQLITE_IOERR', 'SQLITE_NOTFOUND', 'SQLITE_MISUSE', 'SQLITE_IOERR_READ', 'SQLITE_IOERR_SHORT_READ', @@ -252,22 +294,21 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) const isWorkerErrCode = (n)=>!!state.sq3Codes._reverse[n]; /** - Runs the given operation in the async worker counterpart, waits - for its response, and returns the result which the async worker - writes to the given op's index in state.sabOPView. The 2nd argument - must be a single object or primitive value, depending on the - given operation's signature in the async API counterpart. + Runs the given operation (by name) in the async worker + counterpart, waits for its response, and returns the result + which the async worker writes to SAB[state.opIds.rc]. The + 2nd and subsequent arguments must be the aruguments for the + async op. */ const opRun = (op,...args)=>{ - const rcNdx = state.rcIds[op] || toss("Invalid rc ID:",op); const opNdx = state.opIds[op] || toss("Invalid op ID:",op); state.s11n.serialize(...args); - Atomics.store(state.sabOPView, rcNdx, -1); + Atomics.store(state.sabOPView, state.opIds.rc, -1); Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); Atomics.notify(state.sabOPView, state.opIds.whichOp) /* async thread will take over here */; const t = performance.now(); - Atomics.wait(state.sabOPView, rcNdx, -1); - const rc = Atomics.load(state.sabOPView, rcNdx); + Atomics.wait(state.sabOPView, state.opIds.rc, -1); + const rc = Atomics.load(state.sabOPView, state.opIds.rc); metrics[op].wait += performance.now() - t; return rc; }; @@ -348,38 +389,6 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) Map of sqlite3_file pointers to objects constructed by xOpen(). */ const __openFiles = Object.create(null); - - const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; - const dVfs = pDVfs - ? new sqlite3_vfs(pDVfs) - : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. Though we cannot currently handle - that case, the hope is to eventually be able to. */; - const opfsVfs = new sqlite3_vfs(); - const opfsIoMethods = new sqlite3_io_methods(); - opfsVfs.$iVersion = 2/*yes, two*/; - opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; - opfsVfs.$mxPathname = 1024/*sure, why not?*/; - opfsVfs.$zName = wasm.allocCString("opfs"); - // All C-side memory of opfsVfs is zeroed out, but just to be explicit: - opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; - opfsVfs.ondispose = [ - '$zName', opfsVfs.$zName, - 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null), - 'cleanup opfsIoMethods', ()=>opfsIoMethods.dispose() - ]; - /** - Pedantic sidebar about opfsVfs.ondispose: the entries in that array - are items to clean up when opfsVfs.dispose() is called, but in this - environment it will never be called. The VFS instance simply - hangs around until the WASM module instance is cleaned up. We - "could" _hypothetically_ clean it up by "importing" an - sqlite3_os_end() impl into the wasm build, but the shutdown order - of the wasm engine and the JS one are undefined so there is no - guaranty that the opfsVfs instance would be available in one - environment or the other when sqlite3_os_end() is called (_if_ it - gets called at all in a wasm build, which is undefined). - */ /** Installs a StructBinder-bound function pointer member of the @@ -798,7 +807,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); rc = wasm.getMemValue(pOut,'i32'); if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); - log("End of OPFS sanity checks."); + warn("End of OPFS sanity checks."); }finally{ sq3File.dispose(); wasm.scopedAllocPop(scope); diff --git a/ext/wasm/sqlite3-opfs-async-proxy.js b/ext/wasm/sqlite3-opfs-async-proxy.js index a2e0dc5233..6caf21597a 100644 --- a/ext/wasm/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/sqlite3-opfs-async-proxy.js @@ -134,9 +134,9 @@ const getDirForPath = async function f(absFilename, createDirs = false){ and then Atomics.notify()'s it. */ const storeAndNotify = (opName, value)=>{ - log(opName+"() => notify(",state.rcIds[opName],",",value,")"); - Atomics.store(state.sabOPView, state.rcIds[opName], value); - Atomics.notify(state.sabOPView, state.rcIds[opName]); + log(opName+"() => notify(",state.opIds.rc,",",value,")"); + Atomics.store(state.sabOPView, state.opIds.rc, value); + Atomics.notify(state.sabOPView, state.opIds.rc); }; /** @@ -460,17 +460,19 @@ const initS11n = ()=>{ const waitLoop = async function f(){ const opHandlers = Object.create(null); - for(let k of Object.keys(state.rcIds)){ + for(let k of Object.keys(state.opIds)){ + const vi = vfsAsyncImpls[k]; + if(!vi) continue; const o = Object.create(null); opHandlers[state.opIds[k]] = o; o.key = k; - o.f = vfsAsyncImpls[k];// || toss("No vfsAsyncImpls[",k,"]"); + o.f = vi;// || toss("No vfsAsyncImpls[",k,"]"); } let metricsTimer = self.location.port>=1024 ? performance.now() : 0; // ^^^ in dev environment, dump out these metrics one time after a delay. while(true){ try { - if('timed-out'===Atomics.wait(state.sabOPView, state.opIds.whichOp, 0, 150)){ + if('timed-out'===Atomics.wait(state.sabOPView, state.opIds.whichOp, 0, 500)){ continue; } const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); @@ -513,7 +515,6 @@ navigator.storage.getDirectory().then(function(d){ state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); state.opIds = opt.opIds; - state.rcIds = opt.rcIds; state.sq3Codes = opt.sq3Codes; Object.keys(vfsAsyncImpls).forEach((k)=>{ if(!Number.isFinite(state.opIds[k])){ diff --git a/manifest b/manifest index dadba80e1a..4943ef8ef2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C An\salternative\smessaging\sstrategy\sfor\sthe\sOPFS\sVFS\sproxy\swhich\suses\sonly\sSharedArrayBuffer\sand\sAtomics,\sinstead\sof\sworker\smessages,\sfor\scommunication\s(only\sthe\sinitial\sone-time\shandshake\sduring\sinitialization\suses\sworker\smessages).\sIt\sruns\sspeedtest1\sapprox.\s15-20%\sfaster\sbut\sstill\s20-ish%\sslower\sthan\sWASMFS. -D 2022-09-20T08:27:57.073 +C Remove\sextraneous\sresult-code\sSAB\sslots.\sDoc\sadditions. +D 2022-09-20T10:11:52.007 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -474,7 +474,7 @@ F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 -F ext/wasm/GNUmakefile b65cd280059febd4f034b856139fde52f7edb7326133f7e0588b8efcf396709c +F ext/wasm/GNUmakefile 5b848d4d20c31b2aac3dcfabaf5deaf56fe9db9ca1299a4dede1bcd27821426b F ext/wasm/README.md e1ee1e7c321c6a250bf78a84ca6f5882890a237a450ba5a0649c7a8399194c52 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 150a793a47205b8009ac934f3b6d6ebf67b965c072339aaa25ce808a19e116cc F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 @@ -484,7 +484,7 @@ F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77 F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30 F ext/wasm/api/sqlite3-api-oo1.js 2d13dddf0d2b4168a9249f124134d37924331e5b55e05dba18b6d661fbeefe48 -F ext/wasm/api/sqlite3-api-opfs.js 1df64b2a11b7f71ba5c7a5807b4d62727e2c712e537eb563caedc67c5a38d149 +F ext/wasm/api/sqlite3-api-opfs.js 8611680362619cac2425fd77fa9bff00690c80a17a6b95cdd1a39c2cc33e11d0 F ext/wasm/api/sqlite3-api-prologue.js 0d2639387b94c30f492d4aea6e44fb7b16720808678464559458fd2ae3759655 F ext/wasm/api/sqlite3-api-worker1.js ee4cf149cbacb63d06b536674f822aa5088b7e022cdffc69f1f36cebe2f9fea0 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 @@ -519,7 +519,7 @@ F ext/wasm/speedtest1.html 512addeb3c27c94901178b7bcbde83a6f95c093f9ebe16a2959a0 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 -F ext/wasm/sqlite3-opfs-async-proxy.js 038ecd8558abc3f46cfedd560093fef4d460af8c0d0009ab84f2abdc10916a6a +F ext/wasm/sqlite3-opfs-async-proxy.js e3c5e1b6416e9c08c713c43fa98319b06fac622ecb813f294b047072b089fba6 F ext/wasm/sqlite3-worker1-promiser.js 4fd0465688a28a75f1d4ee4406540ba494f49844e3cad0670d0437a001943365 F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5 @@ -2026,11 +2026,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5ca412ced24b4e3af5f467e710a597ed440badf7b8335346aade11d3cad3d1a1 -R eef8f82c266e71b1fc83e097c56f095e -T *branch * opfs-proxy-atomics -T *sym-opfs-proxy-atomics * -T -sym-fiddle-opfs * Cancelled\sby\sbranch. +P a83ee3082d89439ea3ad5737e63e25bebb0f91895aca006ce5fecf5b93a2651a +R bb080f9a3495e6fa47cacc332af06e26 U stephan -Z 1c049de5a4413363c0d5cd656309295d +Z e5fc885edbb6664192b4f06eefccf3e5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cf2c4e7642..16e232c3fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a83ee3082d89439ea3ad5737e63e25bebb0f91895aca006ce5fecf5b93a2651a \ No newline at end of file +88de20712bbe3e11ea74af95edc34e9ab9f346f9aa0a30931e5a9e1f96ce57b0 \ No newline at end of file From c2ccd676890392b1850bdb17d1f22233fb9c0570 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 20 Sep 2022 10:47:36 +0000 Subject: [PATCH 3/5] OPFS proxy: remove one sanity-checking-only level of proxy function to shave off a few microseconds. FossilOrigin-Name: b534831f3efb8910a17e29956e3e87cc80055ea66e15dbef992b6a556ff042f8 --- ext/wasm/api/sqlite3-api-opfs.js | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index a0630d8e76..dd61ee0a26 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -443,7 +443,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) } const memKey = tgt.memberKey(name); //log("installMethod",tgt, name, sigN); - const fProxy = 1 + const fProxy = 0 // We can remove this proxy middle-man once the VFS is working ? callee.argcProxy(func, sigN) : func; diff --git a/manifest b/manifest index 4943ef8ef2..a80252bfbe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sextraneous\sresult-code\sSAB\sslots.\sDoc\sadditions. -D 2022-09-20T10:11:52.007 +C OPFS\sproxy:\sremove\sone\ssanity-checking-only\slevel\sof\sproxy\sfunction\sto\sshave\soff\sa\sfew\smicroseconds. +D 2022-09-20T10:47:36.917 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -484,7 +484,7 @@ F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77 F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30 F ext/wasm/api/sqlite3-api-oo1.js 2d13dddf0d2b4168a9249f124134d37924331e5b55e05dba18b6d661fbeefe48 -F ext/wasm/api/sqlite3-api-opfs.js 8611680362619cac2425fd77fa9bff00690c80a17a6b95cdd1a39c2cc33e11d0 +F ext/wasm/api/sqlite3-api-opfs.js 10063ff6bc4065a27c81a71f41eb429b53390b5bc885a6305243bf83e3bd94c4 F ext/wasm/api/sqlite3-api-prologue.js 0d2639387b94c30f492d4aea6e44fb7b16720808678464559458fd2ae3759655 F ext/wasm/api/sqlite3-api-worker1.js ee4cf149cbacb63d06b536674f822aa5088b7e022cdffc69f1f36cebe2f9fea0 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 @@ -2026,8 +2026,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a83ee3082d89439ea3ad5737e63e25bebb0f91895aca006ce5fecf5b93a2651a -R bb080f9a3495e6fa47cacc332af06e26 +P 88de20712bbe3e11ea74af95edc34e9ab9f346f9aa0a30931e5a9e1f96ce57b0 +R e087e3859232a8d2cb7d73aa59ed1981 U stephan -Z e5fc885edbb6664192b4f06eefccf3e5 +Z c97570003be4379736d19c807db44fd0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 16e232c3fa..3e977d7987 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -88de20712bbe3e11ea74af95edc34e9ab9f346f9aa0a30931e5a9e1f96ce57b0 \ No newline at end of file +b534831f3efb8910a17e29956e3e87cc80055ea66e15dbef992b6a556ff042f8 \ No newline at end of file From b8c8d4e4a077d0f877b189a739fc3293bf585aec Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 20 Sep 2022 13:25:39 +0000 Subject: [PATCH 4/5] Speed up de/serialization of func args and return values in the OPFS VFS proxy. FossilOrigin-Name: 5bf235bbe035e4ace7a54851e190742528af6b4266328a1b8bbb9fb3dd7f2118 --- ext/wasm/api/sqlite3-api-opfs.js | 125 ++++++++++++++++++++------- ext/wasm/sqlite3-opfs-async-proxy.js | 125 +++++++++++++++++++++------ manifest | 14 +-- manifest.uuid | 2 +- 4 files changed, 200 insertions(+), 66 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index dd61ee0a26..1f50b4631a 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -152,6 +152,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) "metrics for",self.location.href,":",metrics, "\nTotal of",n,"op(s) for",t, "ms (incl. "+w+" ms of waiting on the async side)"); + console.log("Serialization metrics:",JSON.stringify(metrics.s11n,0,2)); }, reset: function(){ let k; @@ -159,6 +160,11 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) for(k in state.opIds){ r(metrics[k] = Object.create(null)); } + let s = metrics.s11n = Object.create(null); + s = s.serialize = Object.create(null); + s.count = s.time = 0; + s = metrics.s11n.deserialize = Object.create(null); + s.count = s.time = 0; //[ // timed routines which are not in state.opIds // 'xFileControl' //].forEach((k)=>r(metrics[k] = Object.create(null))); @@ -314,56 +320,115 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) }; const initS11n = ()=>{ - // Achtung: this code is 100% duplicated in the other half of this proxy! + /** + ACHTUNG: this code is 100% duplicated in the other half of + this proxy! + + Historical note: this impl was initially about 5% this size by using + using JSON.stringify/parse(), but using fit-to-purpose serialization + saves considerable runtime. + */ if(state.s11n) return state.s11n; - const jsonDecoder = new TextDecoder(), - jsonEncoder = new TextEncoder('utf-8'), - viewSz = new DataView(state.sabIO, state.sabS11nOffset, 4), - viewJson = new Uint8Array(state.sabIO, state.sabS11nOffset+4, state.sabS11nSize-4); + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); state.s11n = Object.create(null); + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + const getTypeId = (v)=>{ + return TypeIds[typeof v] || toss("This value type cannot be serialized.",v); + }; + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); + } + }; /** Returns an array of the state serialized by the most recent serialize() operation (here or in the counterpart thread), or null if the serialization buffer is empty. */ state.s11n.deserialize = function(){ - const sz = viewSz.getInt32(0, state.littleEndian); - const json = sz ? jsonDecoder.decode( - viewJson.slice(0, sz) - /* slice() (copy) needed, instead of subarray() (reference), - because TextDecoder throws if asked to decode from an - SAB. */ - ) : null; - return JSON.parse(json); - } + ++metrics.s11n.deserialize.count; + const t = performance.now(); + let rc = null; + const argc = viewU8[0]; + if(argc){ + rc = []; + let offset = 1, i, n, v, typeIds = []; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); + } + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; + } + rc.push(v); + } + } + //log("deserialize:",argc, rc); + metrics.s11n.deserialize.time += performance.now() - t; + return rc; + }; /** Serializes all arguments to the shared buffer for consumption - by the counterpart thread. This impl currently uses JSON for - serialization for simplicy of implementation, but if that - proves imperformant then a lower-level approach will be - created. + by the counterpart thread. - If passed "too much data" (more that the shared buffer size - it will either throw or truncate the data (not certain - which)). This routine is only intended for serializing OPFS - VFS arguments and (in at least one special case) result - values, and the buffer is sized to be able to comfortably - handle those. + This routine is only intended for serializing OPFS VFS + arguments and (in at least one special case) result values, + and the buffer is sized to be able to comfortably handle + those. If passed no arguments then it zeroes out the serialization state. */ state.s11n.serialize = function(...args){ + ++metrics.s11n.serialize.count; + const t = performance.now(); if(args.length){ - const json = jsonEncoder.encode(JSON.stringify(args)); - viewSz.setInt32(0, json.byteLength, state.littleEndian); - viewJson.set(json); + //log("serialize():",args); + let i = 0, offset = 1, typeIds = []; + viewU8[0] = args.length & 0xff; + for(; i < args.length; ++i, ++offset){ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); }else{ - viewSz.setInt32(0, 0, state.littleEndian); + viewU8[0] = 0; } + metrics.s11n.serialize.time += performance.now() - t; }; return state.s11n; - }; + }/*initS11n()*/; /** Generates a random ASCII string len characters long, intended for @@ -762,7 +827,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) //| capi.SQLITE_OPEN_DELETEONCLOSE | capi.SQLITE_OPEN_MAIN_DB; const pOut = wasm.scopedAlloc(8); - const dbFile = "/sanity/check/file"; + const dbFile = "/sanity/check/file"+randomFilename(8); const zDbFile = wasm.scopedAllocCString(dbFile); let rc; vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); diff --git a/ext/wasm/sqlite3-opfs-async-proxy.js b/ext/wasm/sqlite3-opfs-async-proxy.js index 6caf21597a..7c5e8430b6 100644 --- a/ext/wasm/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/sqlite3-opfs-async-proxy.js @@ -67,6 +67,11 @@ metrics.reset = ()=>{ for(k in state.opIds){ r(metrics[k] = Object.create(null)); } + let s = metrics.s11n = Object.create(null); + s = s.serialize = Object.create(null); + s.count = s.time = 0; + s = metrics.s11n.deserialize = Object.create(null); + s.count = s.time = 0; }; metrics.dump = ()=>{ let k, n = 0, t = 0, w = 0; @@ -83,6 +88,7 @@ metrics.dump = ()=>{ /*dev console can't expand this object!*/, "\nTotal of",n,"op(s) for",t,"ms", "approx",w,"ms spent waiting on OPFS APIs."); + console.log("Serialization metrics:",JSON.stringify(metrics.s11n,0,2)); }; warn("This file is very much experimental and under construction.", @@ -408,55 +414,118 @@ const vfsAsyncImpls = { const initS11n = ()=>{ // Achtung: this code is 100% duplicated in the other half of this proxy! + + /** + Historical note: this impl was initially about 1% this size by using + using JSON.stringify/parse(), but using fit-to-purpose serialization + saves considerable runtime. + */ + if(state.s11n) return state.s11n; - const jsonDecoder = new TextDecoder(), - jsonEncoder = new TextEncoder('utf-8'), - viewSz = new DataView(state.sabIO, state.sabS11nOffset, 4), - viewJson = new Uint8Array(state.sabIO, state.sabS11nOffset+4, state.sabS11nSize-4); + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); state.s11n = Object.create(null); + + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + + const getTypeId = (v)=>{ + return TypeIds[typeof v] || toss("This value type cannot be serialized.",v); + }; + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); + } + }; + /** Returns an array of the state serialized by the most recent serialize() operation (here or in the counterpart thread), or null if the serialization buffer is empty. */ state.s11n.deserialize = function(){ - const sz = viewSz.getInt32(0, state.littleEndian); - const json = sz ? jsonDecoder.decode( - viewJson.slice(0, sz) - /* slice() (copy) needed, instead of subarray() (reference), - because TextDecoder throws if asked to decode from an - SAB. */ - ) : null; - return JSON.parse(json); - } + ++metrics.s11n.deserialize.count; + const t = performance.now(); + let rc = null; + const argc = viewU8[0]; + if(argc){ + rc = []; + let offset = 1, i, n, v, typeIds = []; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); + } + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; + } + rc.push(v); + } + } + //log("deserialize:",argc, rc); + metrics.s11n.deserialize.time += performance.now() - t; + return rc; + }; + /** Serializes all arguments to the shared buffer for consumption - by the counterpart thread. This impl currently uses JSON for - serialization for simplicy of implementation, but if that - proves imperformant then a lower-level approach will be - created. + by the counterpart thread. - If passed "too much data" (more that the shared buffer size - it will either throw or truncate the data (not certain - which)). This routine is only intended for serializing OPFS - VFS arguments and (in at least one special case) result - values, and the buffer is sized to be able to comfortably - handle those. + This routine is only intended for serializing OPFS VFS + arguments and (in at least one special case) result values, + and the buffer is sized to be able to comfortably handle + those. If passed no arguments then it zeroes out the serialization state. */ state.s11n.serialize = function(...args){ + ++metrics.s11n.serialize.count; + const t = performance.now(); if(args.length){ - const json = jsonEncoder.encode(JSON.stringify(args)); - viewSz.setInt32(0, json.byteLength, state.littleEndian); - viewJson.set(json); + //log("serialize():",args); + let i = 0, offset = 1, typeIds = []; + viewU8[0] = args.length & 0xff; + for(; i < args.length; ++i, ++offset){ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); }else{ - viewSz.setInt32(0, 0, state.littleEndian); + viewU8[0] = 0; } + metrics.s11n.serialize.time += performance.now() - t; }; return state.s11n; -}; +}/*initS11n()*/; const waitLoop = async function f(){ const opHandlers = Object.create(null); diff --git a/manifest b/manifest index a80252bfbe..d5e6c8b22a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C OPFS\sproxy:\sremove\sone\ssanity-checking-only\slevel\sof\sproxy\sfunction\sto\sshave\soff\sa\sfew\smicroseconds. -D 2022-09-20T10:47:36.917 +C Speed\sup\sde/serialization\sof\sfunc\sargs\sand\sreturn\svalues\sin\sthe\sOPFS\sVFS\sproxy. +D 2022-09-20T13:25:39.815 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -484,7 +484,7 @@ F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77 F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30 F ext/wasm/api/sqlite3-api-oo1.js 2d13dddf0d2b4168a9249f124134d37924331e5b55e05dba18b6d661fbeefe48 -F ext/wasm/api/sqlite3-api-opfs.js 10063ff6bc4065a27c81a71f41eb429b53390b5bc885a6305243bf83e3bd94c4 +F ext/wasm/api/sqlite3-api-opfs.js ce75aba0cbfb600cf839362012d17b7b2984aeac5189586c9a5a8f37a573a929 F ext/wasm/api/sqlite3-api-prologue.js 0d2639387b94c30f492d4aea6e44fb7b16720808678464559458fd2ae3759655 F ext/wasm/api/sqlite3-api-worker1.js ee4cf149cbacb63d06b536674f822aa5088b7e022cdffc69f1f36cebe2f9fea0 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 @@ -519,7 +519,7 @@ F ext/wasm/speedtest1.html 512addeb3c27c94901178b7bcbde83a6f95c093f9ebe16a2959a0 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 -F ext/wasm/sqlite3-opfs-async-proxy.js e3c5e1b6416e9c08c713c43fa98319b06fac622ecb813f294b047072b089fba6 +F ext/wasm/sqlite3-opfs-async-proxy.js 9305d92f32d02983c4528b9c801096cfd8295ca7d24e357d90de9bbcb201d035 F ext/wasm/sqlite3-worker1-promiser.js 4fd0465688a28a75f1d4ee4406540ba494f49844e3cad0670d0437a001943365 F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5 @@ -2026,8 +2026,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 88de20712bbe3e11ea74af95edc34e9ab9f346f9aa0a30931e5a9e1f96ce57b0 -R e087e3859232a8d2cb7d73aa59ed1981 +P b534831f3efb8910a17e29956e3e87cc80055ea66e15dbef992b6a556ff042f8 +R 28e069ac7c71e059841784989b21e62c U stephan -Z c97570003be4379736d19c807db44fd0 +Z 483126717a4194dd1997ada5cd19f259 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3e977d7987..44047c5b0f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b534831f3efb8910a17e29956e3e87cc80055ea66e15dbef992b6a556ff042f8 \ No newline at end of file +5bf235bbe035e4ace7a54851e190742528af6b4266328a1b8bbb9fb3dd7f2118 \ No newline at end of file From 0e0aafc69842302d3167e8f64f36529ad58c3c3e Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 20 Sep 2022 14:21:58 +0000 Subject: [PATCH 5/5] Remove an obsolete file. FossilOrigin-Name: 1e43855498b4329f733bd6e0731841dc884e885de7d2d338d402d05d54309427 --- ext/wasm/GNUmakefile | 3 +- ext/wasm/batch-runner-kvvfs.html | 65 -------------------------------- ext/wasm/index.html | 1 - manifest | 15 ++++---- manifest.uuid | 2 +- 5 files changed, 9 insertions(+), 77 deletions(-) delete mode 100644 ext/wasm/batch-runner-kvvfs.html diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index cff5af5944..d8cc845230 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -311,8 +311,7 @@ dir.sql := sql speedtest1 := ../../speedtest1 speedtest1.c := ../../test/speedtest1.c speedtest1.sql := $(dir.sql)/speedtest1.sql -speedtest1.cliflags := --size 1 --big-transactions -# --------------------^^^^^^^^ small size is needed for batch-runner-kvvfs.html +speedtest1.cliflags := --size 100 --big-transactions $(speedtest1): $(MAKE) -C ../.. speedtest1 $(speedtest1.sql): $(speedtest1) $(MAKEFILE) diff --git a/ext/wasm/batch-runner-kvvfs.html b/ext/wasm/batch-runner-kvvfs.html deleted file mode 100644 index 420bdf4f20..0000000000 --- a/ext/wasm/batch-runner-kvvfs.html +++ /dev/null @@ -1,65 +0,0 @@ - - - - - - - - - sqlite3-api kvvfs batch SQL runner - - -
sqlite3-api kvvfs batch SQL runner
- -
- -
Initializing app...
-
- On a slow internet connection this may take a moment. If this - message displays for "a long time", intialization may have - failed and the JavaScript console may contain clues as to why. -
-
-
Downloading...
-
- -
-

- This page is for batch-running extracts from the output - of speedtest1 --script, as well as other standalone SQL - scripts. This variant runs with KVVFS and - has tiny size limits. -

-

ACHTUNG: this file requires a generated input list - file. Run "make batch" from this directory to generate it. -

-

WARNING: if the WASMFS/OPFS layer crashes, this page may - become completely unresponsive and need to be closed and - reloaded to recover. -

-
-
- - - - - - - - - - - -
-
- -
- - - - - - diff --git a/ext/wasm/index.html b/ext/wasm/index.html index 06dd0f31e4..e9a8d7bebb 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -47,7 +47,6 @@
  • testing-worker1-promiser: tests for the Promise-based wrapper of the Worker-based API.
  • batch-runner: runs batches of SQL exported from speedtest1.
    • -
  • batch-runner-kvvfs: KVVFS-specific variant of batch-runner.html.
  • speedtest1: a main-thread WASM build of speedtest1.
  • speedtest1-worker: an interactive Worker-thread variant of speedtest1.
  • speedtest1-wasmfs: a variant of speedtest1 built solely for the wasmfs/opfs feature.
    • diff --git a/manifest b/manifest index d5e6c8b22a..a6a7d0a7be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Speed\sup\sde/serialization\sof\sfunc\sargs\sand\sreturn\svalues\sin\sthe\sOPFS\sVFS\sproxy. -D 2022-09-20T13:25:39.815 +C Remove\san\sobsolete\sfile. +D 2022-09-20T14:21:58.678 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -474,7 +474,7 @@ F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 -F ext/wasm/GNUmakefile 5b848d4d20c31b2aac3dcfabaf5deaf56fe9db9ca1299a4dede1bcd27821426b +F ext/wasm/GNUmakefile e6359a72f044a877ac6ca8f2b04fa643664157f84d81a6d9ad09f384a65160d2 F ext/wasm/README.md e1ee1e7c321c6a250bf78a84ca6f5882890a237a450ba5a0649c7a8399194c52 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 150a793a47205b8009ac934f3b6d6ebf67b965c072339aaa25ce808a19e116cc F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 @@ -489,7 +489,6 @@ F ext/wasm/api/sqlite3-api-prologue.js 0d2639387b94c30f492d4aea6e44fb7b167208086 F ext/wasm/api/sqlite3-api-worker1.js ee4cf149cbacb63d06b536674f822aa5088b7e022cdffc69f1f36cebe2f9fea0 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 4130e2df9587f4e4c3afc04c3549d682c8a5c0cfe5b22819a0a86edb7f01b9bd -F ext/wasm/batch-runner-kvvfs.html ef3b2f553abad4f17a2a29ce6526023793a88e8597b7a24b8c7855a030b90a16 F ext/wasm/batch-runner.html 2857a6db7292ac83d1581af865d643fd34235db2df830d10b43b01388c599e04 F ext/wasm/batch-runner.js 6f5b86e0b5519a9a941d9f17ee9c5ecdc63f452f157602fe7fdf87f6275a2b49 F ext/wasm/common/SqliteTestUtil.js 529161a624265ba84271a52db58da022649832fa1c71309fb1e02cc037327a2b @@ -505,7 +504,7 @@ F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d695 F ext/wasm/fiddle/fiddle-worker.js bccf46045be8824752876f3eec01c223be0616ccac184bffd0024cfe7a3262b8 F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 F ext/wasm/fiddle/fiddle.js 4ffcfc9a235beebaddec689a549e9e0dfad6dca5c1f0b41f03468d7e76480686 -F ext/wasm/index.html de69bc78273fbe024a21dae0f378a87b37ad4cd585ed0cd0a0ed2c5d5afe9fbc +F ext/wasm/index.html 4b74ea334f0bba9923a61246d807a1873784fcef4d3fb342b698613d0a29fd9c F ext/wasm/jaccwabyt/jaccwabyt.js 0d7f32817456a0f3937fcfd934afeb32154ca33580ab264dab6c285e6dbbd215 F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356afacbdbf312b2588106 F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f @@ -2026,8 +2025,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b534831f3efb8910a17e29956e3e87cc80055ea66e15dbef992b6a556ff042f8 -R 28e069ac7c71e059841784989b21e62c +P 5bf235bbe035e4ace7a54851e190742528af6b4266328a1b8bbb9fb3dd7f2118 +R 41b4a0b0d5548be3b8c342b39c8cce92 U stephan -Z 483126717a4194dd1997ada5cd19f259 +Z 524d6845dd279f2ce03cb802d7ae16c8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 44047c5b0f..a0a90cad10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5bf235bbe035e4ace7a54851e190742528af6b4266328a1b8bbb9fb3dd7f2118 \ No newline at end of file +1e43855498b4329f733bd6e0731841dc884e885de7d2d338d402d05d54309427 \ No newline at end of file