paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge opfs-proxy-atomics branch into the fiddle-opfs branch.

Browse Source 
FossilOrigin-Name: 25a36920d44544547a84161681cd41e292b4a70df60ac3630791873a79237d98
This commit is contained in:
stephan
2022-09-20 14:39:54 +00:00
parent 138647a552 0e0aafc698
commit fea03bc0ca
8 changed files with 522 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ext/wasm/GNUmakefile
+11 -9
View File
@@ -65,7 +65,7 @@ SQLITE_OPT = \
.PHONY: release
release:
$(MAKE) 'emcc_opt=-Os -g3 -flto'
$(MAKE) "emcc_opt=-Os -g3 -flto" fiddle_opt=-Os
# ^^^^^ target-specific vars, e.g.:
# release: emcc_opt=...
# apparently only work for file targets, not PHONY targets?
@@ -311,8 +311,7 @@ dir.sql := sql
speedtest1 := ../../speedtest1
speedtest1.c := ../../test/speedtest1.c
speedtest1.sql := $(dir.sql)/speedtest1.sql
speedtest1.cliflags := --size 1 --big-transactions
# --------------------^^^^^^^^ small size is needed for batch-runner-kvvfs.html
speedtest1.cliflags := --size 100 --big-transactions
$(speedtest1):
$(MAKE) -C ../.. speedtest1
$(speedtest1.sql): $(speedtest1) $(MAKEFILE)
@@ -432,19 +431,22 @@ push-fiddle: $(fiddle_files)
# experimentation shows -O2 to be the clear winner in terms of speed.
# Note that build times with anything higher than -O0 are somewhat
# painful.
.PHONY: o0 o1 o2 o3 os oz
o-xtra :=
#o-xtra += -flto
o0:
$(MAKE) clean; $(MAKE) -e emcc_opt=-O0
$(MAKE) clean; $(MAKE) -e "emcc_opt=-O0 $(o-xtra)" fiddle_opt=-O0
o1:
$(MAKE) clean; $(MAKE) -e emcc_opt=-O1
$(MAKE) clean; $(MAKE) -e "emcc_opt=-O1 $(o-xtra)" fiddle_opt=-O1
o2:
$(MAKE) clean; $(MAKE) -e emcc_opt=-O2
$(MAKE) clean; $(MAKE) -e "emcc_opt=-O2 $(o-xtra)" fiddle_opt=-O2
o3:
$(MAKE) clean; $(MAKE) -e emcc_opt=-O3
$(MAKE) clean; $(MAKE) -e "emcc_opt=-O3 $(o-xtra)" fiddle_opt=-O3
os:
$(MAKE) clean; $(MAKE) -e emcc_opt=-Os
$(MAKE) clean; $(MAKE) -e "emcc_opt=-Os $(o-xtra)" fiddle_opt=-Os
oz:
$(MAKE) clean; $(MAKE) -e emcc_opt=-Oz
$(MAKE) clean; $(MAKE) -e "emcc_opt=-Oz $(o-xtra)" fiddle_opt=-Oz
########################################################################
# Sub-makes...
ext/wasm/api/sqlite3-api-opfs.js
+266 -184
View File
@@ -128,7 +128,6 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
// failure is, e.g., that the remote script is 404.
promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons."));
};
const wMsg = (type,args)=>W.postMessage({type,args});
/**
Generic utilities for working with OPFS. This will get filled out
by the Promise setup and, on success, installed as sqlite3.opfs.
@@ -153,6 +152,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
"metrics for",self.location.href,":",metrics,
"\nTotal of",n,"op(s) for",t,
"ms (incl. "+w+" ms of waiting on the async side)");
console.log("Serialization metrics:",JSON.stringify(metrics.s11n,0,2));
},
reset: function(){
let k;
@@ -160,177 +160,17 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
for(k in state.opIds){
r(metrics[k] = Object.create(null));
}
let s = metrics.s11n = Object.create(null);
s = s.serialize = Object.create(null);
s.count = s.time = 0;
s = metrics.s11n.deserialize = Object.create(null);
s.count = s.time = 0;
//[ // timed routines which are not in state.opIds
// 'xFileControl'
//].forEach((k)=>r(metrics[k] = Object.create(null)));
}
}/*metrics*/;
/**
State which we send to the async-api Worker or share with it.
This object must initially contain only cloneable or sharable
objects. After the worker's "inited" message arrives, other types
of data may be added to it.
For purposes of Atomics.wait() and Atomics.notify(), we use a
SharedArrayBuffer with one slot reserved for each of the API
proxy's methods. The sync side of the API uses Atomics.wait()
on the corresponding slot and the async side uses
Atomics.notify() on that slot.
The approach of using a single SAB to serialize comms for all
instances might(?) lead to deadlock situations in multi-db
cases. We should probably have one SAB here with a single slot
for locking a per-file initialization step and then allocate a
separate SAB like the above one for each file. That will
require a bit of acrobatics but should be feasible.
*/
const state = Object.create(null);
state.littleEndian = true;
state.verbose = options.verbose;
state.fileBufferSize =
1024 * 64 /* size of aFileHandle.sab. 64k = max sqlite3 page
size. */;
state.sabS11nOffset = state.fileBufferSize;
state.sabS11nSize = 2048;
state.sabIO = new SharedArrayBuffer(
state.fileBufferSize
+ state.sabS11nSize/* arg/result serialization block */
);
state.opIds = Object.create(null);
state.rcIds = Object.create(null);
const metrics = Object.create(null);
{
let i = 0;
state.opIds.whichOp = i++;
state.opIds.nothing = i++;
state.opIds.xAccess = i++;
state.rcIds.xAccess = i++;
state.opIds.xClose = i++;
state.rcIds.xClose = i++;
state.opIds.xDelete = i++;
state.rcIds.xDelete = i++;
state.opIds.xDeleteNoWait = i++;
state.rcIds.xDeleteNoWait = i++;
state.opIds.xFileSize = i++;
state.rcIds.xFileSize = i++;
state.opIds.xOpen = i++;
state.rcIds.xOpen = i++;
state.opIds.xRead = i++;
state.rcIds.xRead = i++;
state.opIds.xSleep = i++;
state.rcIds.xSleep = i++;
state.opIds.xSync = i++;
state.rcIds.xSync = i++;
state.opIds.xTruncate = i++;
state.rcIds.xTruncate = i++;
state.opIds.xWrite = i++;
state.rcIds.xWrite = i++;
state.opIds.mkdir = i++;
state.rcIds.mkdir = i++;
state.sabOP = new SharedArrayBuffer(i * 4/*sizeof int32*/);
state.opIds.xFileControl = state.opIds.xSync /* special case */;
opfsUtil.metrics.reset();
}
state.sq3Codes = Object.create(null);
state.sq3Codes._reverse = Object.create(null);
[ // SQLITE_xxx constants to export to the async worker counterpart...
'SQLITE_ERROR', 'SQLITE_IOERR',
'SQLITE_NOTFOUND', 'SQLITE_MISUSE',
'SQLITE_IOERR_READ', 'SQLITE_IOERR_SHORT_READ',
'SQLITE_IOERR_WRITE', 'SQLITE_IOERR_FSYNC',
'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE',
'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE',
'SQLITE_IOERR_DELETE',
'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE',
'SQLITE_OPEN_READONLY'
].forEach(function(k){
state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k);
state.sq3Codes._reverse[capi[k]] = k;
});
const isWorkerErrCode = (n)=>!!state.sq3Codes._reverse[n];
/**
Runs the given operation in the async worker counterpart, waits
for its response, and returns the result which the async worker
writes to the given op's index in state.sabOPView. The 2nd argument
must be a single object or primitive value, depending on the
given operation's signature in the async API counterpart.
*/
const opRun = (op,...args)=>{
const t = performance.now();
Atomics.store(state.sabOPView, state.opIds[op], -1);
wMsg(op, args);
Atomics.wait(state.sabOPView, state.opIds[op], -1);
metrics[op].wait += performance.now() - t;
return Atomics.load(state.sabOPView, state.opIds[op]);
};
const initS11n = ()=>{
// Achtung: this code is 100% duplicated in the other half of this proxy!
if(state.s11n) return state.s11n;
const jsonDecoder = new TextDecoder(),
jsonEncoder = new TextEncoder('utf-8'),
viewSz = new DataView(state.sabIO, state.sabS11nOffset, 4),
viewJson = new Uint8Array(state.sabIO, state.sabS11nOffset+4, state.sabS11nSize-4);
state.s11n = Object.create(null);
/**
Returns an array of the state serialized by the most recent
serialize() operation (here or in the counterpart thread), or
null if the serialization buffer is empty.
*/
state.s11n.deserialize = function(){
const sz = viewSz.getInt32(0, state.littleEndian);
const json = sz ? jsonDecoder.decode(
viewJson.slice(0, sz)
/* slice() (copy) needed, instead of subarray() (reference),
because TextDecoder throws if asked to decode from an
SAB. */
) : null;
return JSON.parse(json);
}
/**
Serializes all arguments to the shared buffer for consumption
by the counterpart thread. This impl currently uses JSON for
serialization for simplicy of implementation, but if that
proves imperformant then a lower-level approach will be
created.
*/
state.s11n.serialize = function(...args){
const json = jsonEncoder.encode(JSON.stringify(args));
viewSz.setInt32(0, json.byteLength, state.littleEndian);
viewJson.set(json);
};
return state.s11n;
};
/**
Generates a random ASCII string len characters long, intended for
use as a temporary file name.
*/
const randomFilename = function f(len=16){
if(!f._chars){
f._chars = "abcdefghijklmnopqrstuvwxyz"+
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"+
"012346789";
f._n = f._chars.length;
}
const a = [];
let i = 0;
for( ; i < len; ++i){
const ndx = Math.random() * (f._n * 64) % f._n | 0;
a[i] = f._chars[ndx];
}
return a.join('');
};
/**
Map of sqlite3_file pointers to objects constructed by xOpen().
*/
const __openFiles = Object.create(null);
const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/;
const dVfs = pDVfs
? new sqlite3_vfs(pDVfs)
@@ -363,6 +203,258 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
gets called at all in a wasm build, which is undefined).
*/
/**
State which we send to the async-api Worker or share with it.
This object must initially contain only cloneable or sharable
objects. After the worker's "inited" message arrives, other types
of data may be added to it.
For purposes of Atomics.wait() and Atomics.notify(), we use a
SharedArrayBuffer with one slot reserved for each of the API
proxy's methods. The sync side of the API uses Atomics.wait()
on the corresponding slot and the async side uses
Atomics.notify() on that slot.
The approach of using a single SAB to serialize comms for all
instances might(?) lead to deadlock situations in multi-db
cases. We should probably have one SAB here with a single slot
for locking a per-file initialization step and then allocate a
separate SAB like the above one for each file. That will
require a bit of acrobatics but should be feasible.
*/
const state = Object.create(null);
state.littleEndian = true;
state.verbose = options.verbose;
/* Size of file I/O buffer block. 64k = max sqlite3 page size. */
state.fileBufferSize =
1024 * 64;
state.sabS11nOffset = state.fileBufferSize;
/**
The size of the block in our SAB for serializing arguments and
result values. Need to be large enough to hold serialized
values of any of the proxied APIs. Filenames are the largest
part but are limited to opfsVfs.$mxPathname bytes.
*/
state.sabS11nSize = opfsVfs.$mxPathname * 2;
/**
The SAB used for all data I/O (files and arg/result s11n).
*/
state.sabIO = new SharedArrayBuffer(
state.fileBufferSize/* file i/o block */
+ state.sabS11nSize/* argument/result serialization block */
);
state.opIds = Object.create(null);
const metrics = Object.create(null);
{
/* Indexes for use in our SharedArrayBuffer... */
let i = 0;
/* SAB slot used to communicate which operation is desired
between both workers. This worker writes to it and the other
listens for changes. */
state.opIds.whichOp = i++;
/* Slot for storing return values. This work listens to that
slot and the other worker writes to it. */
state.opIds.rc = i++;
/* Each function gets an ID which this worker writes to
the whichOp slot. The async-api worker uses Atomic.wait()
on the whichOp slot to figure out which operation to run
next. */
state.opIds.xAccess = i++;
state.opIds.xClose = i++;
state.opIds.xDelete = i++;
state.opIds.xDeleteNoWait = i++;
state.opIds.xFileSize = i++;
state.opIds.xOpen = i++;
state.opIds.xRead = i++;
state.opIds.xSleep = i++;
state.opIds.xSync = i++;
state.opIds.xTruncate = i++;
state.opIds.xWrite = i++;
state.opIds.mkdir = i++;
state.opIds.xFileControl = i++;
state.sabOP = new SharedArrayBuffer(i * 4/*sizeof int32*/);
opfsUtil.metrics.reset();
}
/**
SQLITE_xxx constants to export to the async worker
counterpart...
*/
state.sq3Codes = Object.create(null);
state.sq3Codes._reverse = Object.create(null);
[
'SQLITE_ERROR', 'SQLITE_IOERR',
'SQLITE_NOTFOUND', 'SQLITE_MISUSE',
'SQLITE_IOERR_READ', 'SQLITE_IOERR_SHORT_READ',
'SQLITE_IOERR_WRITE', 'SQLITE_IOERR_FSYNC',
'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE',
'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE',
'SQLITE_IOERR_DELETE',
'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE',
'SQLITE_OPEN_READONLY'
].forEach(function(k){
state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k);
state.sq3Codes._reverse[capi[k]] = k;
});
const isWorkerErrCode = (n)=>!!state.sq3Codes._reverse[n];
/**
Runs the given operation (by name) in the async worker
counterpart, waits for its response, and returns the result
which the async worker writes to SAB[state.opIds.rc]. The
2nd and subsequent arguments must be the aruguments for the
async op.
*/
const opRun = (op,...args)=>{
const opNdx = state.opIds[op] || toss("Invalid op ID:",op);
state.s11n.serialize(...args);
Atomics.store(state.sabOPView, state.opIds.rc, -1);
Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx);
Atomics.notify(state.sabOPView, state.opIds.whichOp) /* async thread will take over here */;
const t = performance.now();
Atomics.wait(state.sabOPView, state.opIds.rc, -1);
const rc = Atomics.load(state.sabOPView, state.opIds.rc);
metrics[op].wait += performance.now() - t;
return rc;
};
const initS11n = ()=>{
/**
ACHTUNG: this code is 100% duplicated in the other half of
this proxy!
Historical note: this impl was initially about 5% this size by using
using JSON.stringify/parse(), but using fit-to-purpose serialization
saves considerable runtime.
*/
if(state.s11n) return state.s11n;
const textDecoder = new TextDecoder(),
textEncoder = new TextEncoder('utf-8'),
viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize),
viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize);
state.s11n = Object.create(null);
const TypeIds = Object.create(null);
TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' };
TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' };
TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' };
TypeIds.string = { id: 4 };
const getTypeId = (v)=>{
return TypeIds[typeof v] || toss("This value type cannot be serialized.",v);
};
const getTypeIdById = (tid)=>{
switch(tid){
case TypeIds.number.id: return TypeIds.number;
case TypeIds.bigint.id: return TypeIds.bigint;
case TypeIds.boolean.id: return TypeIds.boolean;
case TypeIds.string.id: return TypeIds.string;
default: toss("Invalid type ID:",tid);
}
};
/**
Returns an array of the state serialized by the most recent
serialize() operation (here or in the counterpart thread), or
null if the serialization buffer is empty.
*/
state.s11n.deserialize = function(){
++metrics.s11n.deserialize.count;
const t = performance.now();
let rc = null;
const argc = viewU8[0];
if(argc){
rc = [];
let offset = 1, i, n, v, typeIds = [];
for(i = 0; i < argc; ++i, ++offset){
typeIds.push(getTypeIdById(viewU8[offset]));
}
for(i = 0; i < argc; ++i){
const t = typeIds[i];
if(t.getter){
v = viewDV[t.getter](offset, state.littleEndian);
offset += t.size;
}else{
n = viewDV.getInt32(offset, state.littleEndian);
offset += 4;
v = textDecoder.decode(viewU8.slice(offset, offset+n));
offset += n;
}
rc.push(v);
}
}
//log("deserialize:",argc, rc);
metrics.s11n.deserialize.time += performance.now() - t;
return rc;
};
/**
Serializes all arguments to the shared buffer for consumption
by the counterpart thread.
This routine is only intended for serializing OPFS VFS
arguments and (in at least one special case) result values,
and the buffer is sized to be able to comfortably handle
those.
If passed no arguments then it zeroes out the serialization
state.
*/
state.s11n.serialize = function(...args){
++metrics.s11n.serialize.count;
const t = performance.now();
if(args.length){
//log("serialize():",args);
let i = 0, offset = 1, typeIds = [];
viewU8[0] = args.length & 0xff;
for(; i < args.length; ++i, ++offset){
typeIds.push(getTypeId(args[i]));
viewU8[offset] = typeIds[i].id;
}
for(i = 0; i < args.length; ++i) {
const t = typeIds[i];
if(t.setter){
viewDV[t.setter](offset, args[i], state.littleEndian);
offset += t.size;
}else{
const s = textEncoder.encode(args[i]);
viewDV.setInt32(offset, s.byteLength, state.littleEndian);
offset += 4;
viewU8.set(s, offset);
offset += s.byteLength;
}
}
//log("serialize() result:",viewU8.slice(0,offset));
}else{
viewU8[0] = 0;
}
metrics.s11n.serialize.time += performance.now() - t;
};
return state.s11n;
}/*initS11n()*/;
/**
Generates a random ASCII string len characters long, intended for
use as a temporary file name.
*/
const randomFilename = function f(len=16){
if(!f._chars){
f._chars = "abcdefghijklmnopqrstuvwxyz"+
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"+
"012346789";
f._n = f._chars.length;
}
const a = [];
let i = 0;
for( ; i < len; ++i){
const ndx = Math.random() * (f._n * 64) % f._n | 0;
a[i] = f._chars[ndx];
}
return a.join('');
};
/**
Map of sqlite3_file pointers to objects constructed by xOpen().
*/
const __openFiles = Object.create(null);
/**
Installs a StructBinder-bound function pointer member of the
given name and function in the given StructType target object.
@@ -416,7 +508,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
}
const memKey = tgt.memberKey(name);
//log("installMethod",tgt, name, sigN);
const fProxy = 1
const fProxy = 0
// We can remove this proxy middle-man once the VFS is working
? callee.argcProxy(func, sigN)
: func;
@@ -552,9 +644,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
const vfsSyncWrappers = {
xAccess: function(pVfs,zName,flags,pOut){
mTimeStart('xAccess');
wasm.setMemValue(
pOut, (opRun('xAccess', wasm.cstringToJs(zName)) ? 0 : 1), 'i32'
);
const rc = opRun('xAccess', wasm.cstringToJs(zName));
wasm.setMemValue( pOut, (rc ? 0 : 1), 'i32' );
mTimeEnd();
return 0;
},
@@ -686,22 +777,12 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
opfsUtil.deleteEntry = function(fsEntryName,recursive=false){
return 0===opRun('xDelete', fsEntryName, 0, recursive);
};
/**
Exactly like deleteEntry() but runs asynchronously. This is a
"fire and forget" operation: it does not return a promise
because the counterpart operation happens in another thread and
waiting on that result in a Promise would block the OPFS VFS
from acting until it completed.
*/
opfsUtil.deleteEntryAsync = function(fsEntryName,recursive=false){
wMsg('xDeleteNoWait', [fsEntryName, 0, recursive]);
};
/**
Synchronously creates the given directory name, recursively, in
the OPFS filesystem. Returns true if it succeeds or the
directory already exists, else false.
*/
opfsUtil.mkdir = async function(absDirName){
opfsUtil.mkdir = function(absDirName){
return 0===opRun('mkdir', absDirName);
};
/**
@@ -736,7 +817,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
features like getting a directory listing.
*/
const sanityCheck = async function(){
const sanityCheck = function(){
const scope = wasm.scopedAllocPush();
const sq3File = new sqlite3_file();
try{
@@ -746,7 +827,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
//| capi.SQLITE_OPEN_DELETEONCLOSE
| capi.SQLITE_OPEN_MAIN_DB;
const pOut = wasm.scopedAlloc(8);
const dbFile = "/sanity/check/file";
const dbFile = "/sanity/check/file"+randomFilename(8);
const zDbFile = wasm.scopedAllocCString(dbFile);
let rc;
vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut);
@@ -791,6 +872,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut);
rc = wasm.getMemValue(pOut,'i32');
if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete().");
warn("End of OPFS sanity checks.");
}finally{
sq3File.dispose();
wasm.scopedAllocPop(scope);
@@ -803,7 +885,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
switch(data.type){
case 'opfs-async-loaded':
/*Pass our config and shared state on to the async worker.*/
wMsg('opfs-async-init',state);
W.postMessage({type: 'opfs-async-init',args: state});
break;
case 'opfs-async-inited':{
/*Indicates that the async partner has received the 'init',
ext/wasm/batch-runner-kvvfs.html
-65
View File
@@ -1,65 +0,0 @@
<!doctype html>
<html lang="en-us">
<head>
<meta charset="utf-8">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon">
<link rel="stylesheet" href="common/emscripten.css"/>
<link rel="stylesheet" href="common/testing.css"/>
<title>sqlite3-api kvvfs batch SQL runner</title>
</head>
<body>
<header id='titlebar'><span>sqlite3-api kvvfs batch SQL runner</span></header>
<!-- emscripten bits -->
<figure id="module-spinner">
<div class="spinner"></div>
<div class='center'><strong>Initializing app...</strong></div>
<div class='center'>
On a slow internet connection this may take a moment. If this
message displays for "a long time", intialization may have
failed and the JavaScript console may contain clues as to why.
</div>
</figure>
<div class="emscripten" id="module-status">Downloading...</div>
<div class="emscripten">
<progress value="0" max="100" id="module-progress" hidden='1'></progress>
</div><!-- /emscripten bits -->
<p>
This page is for batch-running extracts from the output
of <tt>speedtest1 --script</tt>, as well as other standalone SQL
scripts. <span class='warning'>This variant runs with KVVFS and
has tiny size limits.</span>
</p>
<p id='warn-list' class='warning'>ACHTUNG: this file requires a generated input list
file. Run "make batch" from this directory to generate it.
</p>
<p id='warn-opfs' class='warning'>WARNING: if the WASMFS/OPFS layer crashes, this page may
become completely unresponsive and need to be closed and
reloaded to recover.
</p>
<hr>
<div>
<select class='disable-during-eval' id='sql-select'>
<option disabled selected>Populated via script code</option>
</select>
<button class='disable-during-eval' id='sql-run'>Run selected SQL</button>
<button class='disable-during-eval' id='sql-run-next'>Run next...</button>
<button class='disable-during-eval' id='sql-run-remaining'>Run all remaining...</button>
<button class='disable-during-eval' id='export-metrics'>Export metrics (WIP)</button>
<button class='disable-during-eval' id='db-reset'>Reset db</button>
<button id='output-clear'>Clear output</button>
<span class='input-wrapper'>
<input type='checkbox' class='disable-during-eval' id='cb-reverse-log-order' checked></input>
<label for='cb-reverse-log-order'>Reverse log order</label>
</span>
</div>
<hr>
<div id='reverse-log-notice' class='hidden'>(Log output is in reverse order, newest first!)</div>
<div id='test-output'></div>
<!-- batch-runner.js "should" work with sqlite3-kvvfs so long as
its data sets don't violate the the storage limits. -->
<script src="sqlite3-kvvfs.js"></script>
<script src="common/SqliteTestUtil.js"></script>
<script src="batch-runner.js"></script>
</body>
</html>
ext/wasm/index.html
-1
View File
@@ -47,7 +47,6 @@
<li><a href='testing-worker1-promiser.html'>testing-worker1-promiser</a>:
tests for the Promise-based wrapper of the Worker-based API.</li>
<li><a href='batch-runner.html'>batch-runner</a>: runs batches of SQL exported from speedtest1.</li>
<li><a href='batch-runner-kvvfs.html'>batch-runner-kvvfs</a>: KVVFS-specific variant of batch-runner.html.</li>
<li><a href='speedtest1.html'>speedtest1</a>: a main-thread WASM build of speedtest1.</li>
<li><a href='speedtest1-worker.html'>speedtest1-worker</a>: an interactive Worker-thread variant of speedtest1.</li>
<li><a href='speedtest1-wasmfs.html'>speedtest1-wasmfs</a>: a variant of speedtest1 built solely for the wasmfs/opfs feature.</li>
ext/wasm/speedtest1-worker.html
+51 -50
View File
@@ -212,70 +212,71 @@
/* TODO? Flags which require values need custom UI
controls and some of them make little sense here
(e.g. --script FILE). */
flags["autovacuum"] = "Enable AUTOVACUUM mode";
flags["big-transactions"] = "Important for tests 410 and 510!";
//flags["cachesize"] = "N Set the cache size to N";
flags["checkpoint"] = "Run PRAGMA wal_checkpoint after each test case";
flags["exclusive"] = "Enable locking_mode=EXCLUSIVE";
flags["explain"] = "Like --sqlonly but with added EXPLAIN keywords";
//flags["heap"] = "SZ MIN Memory allocator uses SZ bytes & min allocation MIN";
flags["incrvacuum"] = "Enable incremenatal vacuum mode";
//flags["journal"] = "M Set the journal_mode to M";
//flags["key"] = "KEY Set the encryption key to KEY";
//flags["lookaside"] = "N SZ Configure lookaside for N slots of SZ bytes each";
flags["memdb"] = "Use an in-memory database";
//flags["mmap"] = "SZ MMAP the first SZ bytes of the database file";
flags["multithread"] = "Set multithreaded mode";
flags["nomemstat"] = "Disable memory statistics";
flags["nomutex"] = "Open db with SQLITE_OPEN_NOMUTEX";
flags["nosync"] = "Set PRAGMA synchronous=OFF";
flags["notnull"] = "Add NOT NULL constraints to table columns";
//flags["output"] = "FILE Store SQL output in FILE";
//flags["pagesize"] = "N Set the page size to N";
//flags["pcache"] = "N SZ Configure N pages of pagecache each of size SZ bytes";
//flags["primarykey"] = "Use PRIMARY KEY instead of UNIQUE where appropriate";
//flags["repeat"] = "N Repeat each SELECT N times (default: 1)";
flags["reprepare"] = "Reprepare each statement upon every invocation";
//flags["reserve"] = "N Reserve N bytes on each database page";
//flags["script"] = "FILE Write an SQL script for the test into FILE";
flags["serialized"] = "Set serialized threading mode";
flags["singlethread"] = "Set single-threaded mode - disables all mutexing";
flags["sqlonly"] = "No-op. Only show the SQL that would have been run.";
flags["shrink"] = "memory Invoke sqlite3_db_release_memory() frequently.";
//flags["size"] = "N Relative test size. Default=100";
flags["strict"] = "Use STRICT table where appropriate";
flags["stats"] = "Show statistics at the end";
//flags["temp"] = "N N from 0 to 9. 0: no temp table. 9: all temp tables";
//flags["testset"] = "T Run test-set T (main, cte, rtree, orm, fp, debug)";
flags["trace"] = "Turn on SQL tracing";
//flags["threads"] = "N Use up to N threads for sorting";
flags["--autovacuum"] = "Enable AUTOVACUUM mode";
flags["--big-transactions"] = "Important for tests 410 and 510!";
//flags["--cachesize"] = "N Set the cache size to N";
flags["--checkpoint"] = "Run PRAGMA wal_checkpoint after each test case";
flags["--exclusive"] = "Enable locking_mode=EXCLUSIVE";
flags["--explain"] = "Like --sqlonly but with added EXPLAIN keywords";
//flags["--heap"] = "SZ MIN Memory allocator uses SZ bytes & min allocation MIN";
flags["--incrvacuum"] = "Enable incremenatal vacuum mode";
//flags["--journal"] = "M Set the journal_mode to M";
//flags["--key"] = "KEY Set the encryption key to KEY";
//flags["--lookaside"] = "N SZ Configure lookaside for N slots of SZ bytes each";
flags["--memdb"] = "Use an in-memory database";
//flags["--mmap"] = "SZ MMAP the first SZ bytes of the database file";
flags["--multithread"] = "Set multithreaded mode";
flags["--nomemstat"] = "Disable memory statistics";
flags["--nomutex"] = "Open db with SQLITE_OPEN_NOMUTEX";
flags["--nosync"] = "Set PRAGMA synchronous=OFF";
flags["--notnull"] = "Add NOT NULL constraints to table columns";
//flags["--output"] = "FILE Store SQL output in FILE";
//flags["--pagesize"] = "N Set the page size to N";
//flags["--pcache"] = "N SZ Configure N pages of pagecache each of size SZ bytes";
//flags["--primarykey"] = "Use PRIMARY KEY instead of UNIQUE where appropriate";
//flags["--repeat"] = "N Repeat each SELECT N times (default: 1)";
flags["--reprepare"] = "Reprepare each statement upon every invocation";
//flags["--reserve"] = "N Reserve N bytes on each database page";
//flags["--script"] = "FILE Write an SQL script for the test into FILE";
flags["--serialized"] = "Set serialized threading mode";
flags["--singlethread"] = "Set single-threaded mode - disables all mutexing";
flags["--sqlonly"] = "No-op. Only show the SQL that would have been run.";
flags["--shrink"] = "memory Invoke sqlite3_db_release_memory() frequently.";
//flags["--size"] = "N Relative test size. Default=100";
flags["--strict"] = "Use STRICT table where appropriate";
flags["--stats"] = "Show statistics at the end";
//flags["--temp"] = "N N from 0 to 9. 0: no temp table. 9: all temp tables";
//flags["--testset"] = "T Run test-set T (main, cte, rtree, orm, fp, debug)";
flags["--trace"] = "Turn on SQL tracing";
//flags["--threads"] = "N Use up to N threads for sorting";
/*
The core API's WASM build does not support UTF16, but in
this app it's not an issue because the data are not crossing
JS/WASM boundaries.
*/
flags["utf16be"] = "Set text encoding to UTF-16BE";
flags["utf16le"] = "Set text encoding to UTF-16LE";
flags["verify"] = "Run additional verification steps.";
flags["without"] = "rowid Use WITHOUT ROWID where appropriate";
flags["--utf16be"] = "Set text encoding to UTF-16BE";
flags["--utf16le"] = "Set text encoding to UTF-16LE";
flags["--verify"] = "Run additional verification steps.";
flags["--without"] = "rowid Use WITHOUT ROWID where appropriate";
const preselectedFlags = [
'big-transactions',
'singlethread'
'--big-transactions',
'--singlethread'
];
if('opfs'!==urlParams.get('vfs')){
preselectedFlags.push('memdb');
if(urlParams.has('flags')){
preselectedFlags.push(...urlParams.get('flags').split(','));
}
if('opfs'!==urlParams.get('vfs')){
preselectedFlags.push('--memdb');
}
Object.keys(flags).sort().forEach(function(f){
const opt = document.createElement('option');
eFlags.appendChild(opt);
const lbl = nbspPad('--'+f)+flags[f];
const lbl = nbspPad(f)+flags[f];
//opt.innerText = lbl;
opt.innerHTML = lbl;
opt.value = '--'+f;
opt.value = f;
if(preselectedFlags.indexOf(f) >= 0) opt.selected = true;
});
});
const cbReverseLog = E('#cb-reverse-log-order');
const lblReverseLog = E('#lbl-reverse-log-order');
if(cbReverseLog.checked){
ext/wasm/sqlite3-opfs-async-proxy.js
+182 -70
View File
@@ -63,10 +63,15 @@ const error = (...args)=>logImpl(0, ...args);
const metrics = Object.create(null);
metrics.reset = ()=>{
let k;
const r = (m)=>(m.count = m.time = 0);
const r = (m)=>(m.count = m.time = m.wait = 0);
for(k in state.opIds){
r(metrics[k] = Object.create(null));
}
let s = metrics.s11n = Object.create(null);
s = s.serialize = Object.create(null);
s.count = s.time = 0;
s = metrics.s11n.deserialize = Object.create(null);
s.count = s.time = 0;
};
metrics.dump = ()=>{
let k, n = 0, t = 0, w = 0;
@@ -74,11 +79,16 @@ metrics.dump = ()=>{
const m = metrics[k];
n += m.count;
t += m.time;
w += m.wait;
m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0;
}
console.log(self.location.href,
"metrics for",self.location.href,":",metrics,
"\nTotal of",n,"op(s) for",t,"ms");
"metrics for",self.location.href,":\n",
JSON.stringify(metrics,0,2)
/*dev console can't expand this object!*/,
"\nTotal of",n,"op(s) for",t,"ms",
"approx",w,"ms spent waiting on OPFS APIs.");
console.log("Serialization metrics:",JSON.stringify(metrics.s11n,0,2));
};
warn("This file is very much experimental and under construction.",
@@ -130,9 +140,9 @@ const getDirForPath = async function f(absFilename, createDirs = false){
and then Atomics.notify()'s it.
*/
const storeAndNotify = (opName, value)=>{
log(opName+"() is notify()ing w/ value:",value);
Atomics.store(state.sabOPView, state.opIds[opName], value);
Atomics.notify(state.sabOPView, state.opIds[opName]);
log(opName+"() => notify(",state.opIds.rc,",",value,")");
Atomics.store(state.sabOPView, state.opIds.rc, value);
Atomics.notify(state.sabOPView, state.opIds.rc);
};
/**
@@ -155,6 +165,17 @@ const mTimeStart = (op)=>{
const mTimeEnd = ()=>(
metrics[opTimer.op].time += performance.now() - opTimer.start
);
const waitTimer = Object.create(null);
waitTimer.op = undefined;
waitTimer.start = undefined;
const wTimeStart = (op)=>{
waitTimer.start = performance.now();
waitTimer.op = op;
//metrics[op] || toss("Maintenance required: missing metrics for",op);
};
const wTimeEnd = ()=>(
metrics[waitTimer.op].wait += performance.now() - waitTimer.start
);
/**
Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods
@@ -163,17 +184,20 @@ const mTimeEnd = ()=>(
*/
const vfsAsyncImpls = {
mkdir: async function(dirname){
mTimeStart('mkdir');
let rc = 0;
wTimeStart('mkdir');
try {
await getDirForPath(dirname+"/filepart", true);
}catch(e){
//error("mkdir failed",filename, e.message);
rc = state.sq3Codes.SQLITE_IOERR;
}
wTimeEnd();
storeAndNotify('mkdir', rc);
mTimeEnd();
},
xAccess: async function(filename){
log("xAccess(",arguments[0],")");
mTimeStart('xAccess');
/* OPFS cannot support the full range of xAccess() queries sqlite3
calls for. We can essentially just tell if the file is
@@ -187,20 +211,23 @@ const vfsAsyncImpls = {
accessible, non-0 means not accessible.
*/
let rc = 0;
wTimeStart('xAccess');
try{
const [dh, fn] = await getDirForPath(filename);
await dh.getFileHandle(fn);
}catch(e){
rc = state.sq3Codes.SQLITE_IOERR;
}
wTimeEnd();
storeAndNotify('xAccess', rc);
mTimeEnd();
},
xClose: async function(fid){
const opName = 'xClose';
mTimeStart(opName);
log(opName+"(",arguments[0],")");
const fh = __openFiles[fid];
let rc = 0;
wTimeStart('xClose');
if(fh){
delete __openFiles[fid];
if(fh.accessHandle) await fh.accessHandle.close();
@@ -208,10 +235,11 @@ const vfsAsyncImpls = {
try{ await fh.dirHandle.removeEntry(fh.filenamePart) }
catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) }
}
storeAndNotify(opName, 0);
}else{
storeAndNotify(opName, state.sq3Codes.SQLITE_NOFOUND);
rc = state.sq3Codes.SQLITE_NOTFOUND;
}
wTimeEnd();
storeAndNotify(opName, rc);
mTimeEnd();
},
xDelete: async function(...args){
@@ -233,12 +261,11 @@ const vfsAsyncImpls = {
presumably it will fail if the dir is not empty and that flag
is false.
*/
log("xDelete(",arguments[0],")");
let rc = 0;
wTimeStart('xDelete');
try {
while(filename){
const [hDir, filenamePart] = await getDirForPath(filename, false);
//log("Removing:",hDir, filenamePart);
if(!filenamePart) break;
await hDir.removeEntry(filenamePart, {recursive});
if(0x1234 !== syncDir) break;
@@ -252,13 +279,14 @@ const vfsAsyncImpls = {
//error("Delete failed",filename, e.message);
rc = state.sq3Codes.SQLITE_IOERR_DELETE;
}
wTimeEnd();
return rc;
},
xFileSize: async function(fid){
mTimeStart('xFileSize');
log("xFileSize(",arguments,")");
const fh = __openFiles[fid];
let sz;
wTimeStart('xFileSize');
try{
sz = await fh.accessHandle.getSize();
state.s11n.serialize(Number(sz));
@@ -267,15 +295,16 @@ const vfsAsyncImpls = {
error("xFileSize():",e, fh);
sz = state.sq3Codes.SQLITE_IOERR;
}
wTimeEnd();
storeAndNotify('xFileSize', sz);
mTimeEnd();
},
xOpen: async function(fid/*sqlite3_file pointer*/, filename, flags){
const opName = 'xOpen';
mTimeStart(opName);
log(opName+"(",arguments[0],")");
const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags);
const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags);
wTimeStart('xOpen');
try{
let hDir, filenamePart;
try {
@@ -283,6 +312,7 @@ const vfsAsyncImpls = {
}catch(e){
storeAndNotify(opName, state.sql3Codes.SQLITE_NOTFOUND);
mTimeEnd();
wTimeEnd();
return;
}
const hFile = await hDir.getFileHandle(filenamePart, {create});
@@ -294,6 +324,7 @@ const vfsAsyncImpls = {
places that limitation on it.
*/
fobj.accessHandle = await hFile.createSyncAccessHandle();
wTimeEnd();
__openFiles[fid] = fobj;
fobj.filenameAbs = filename;
fobj.filenamePart = filenamePart;
@@ -304,6 +335,7 @@ const vfsAsyncImpls = {
fobj.deleteOnClose = deleteOnClose;
storeAndNotify(opName, 0);
}catch(e){
wTimeEnd();
error(opName,e);
storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR);
}
@@ -311,14 +343,15 @@ const vfsAsyncImpls = {
},
xRead: async function(fid,n,offset){
mTimeStart('xRead');
log("xRead(",arguments[0],")");
let rc = 0;
try{
const fh = __openFiles[fid];
wTimeStart('xRead');
const nRead = fh.accessHandle.read(
fh.sabView.subarray(0, n),
{at: Number(offset)}
);
wTimeEnd();
if(nRead < n){/* Zero-fill remaining bytes */
fh.sabView.fill(0, nRead, n);
rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ;
@@ -332,17 +365,20 @@ const vfsAsyncImpls = {
},
xSync: async function(fid,flags/*ignored*/){
mTimeStart('xSync');
log("xSync(",arguments[0],")");
const fh = __openFiles[fid];
if(!fh.readOnly && fh.accessHandle) await fh.accessHandle.flush();
if(!fh.readOnly && fh.accessHandle){
wTimeStart('xSync');
await fh.accessHandle.flush();
wTimeEnd();
}
storeAndNotify('xSync',0);
mTimeEnd();
},
xTruncate: async function(fid,size){
mTimeStart('xTruncate');
log("xTruncate(",arguments[0],")");
let rc = 0;
const fh = __openFiles[fid];
wTimeStart('xTruncate');
try{
affirmNotRO('xTruncate', fh);
await fh.accessHandle.truncate(size);
@@ -350,13 +386,14 @@ const vfsAsyncImpls = {
error("xTruncate():",e,fh);
rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE;
}
wTimeEnd();
storeAndNotify('xTruncate',rc);
mTimeEnd();
},
xWrite: async function(fid,n,offset){
mTimeStart('xWrite');
log("xWrite(",arguments[0],")");
let rc;
wTimeStart('xWrite');
try{
const fh = __openFiles[fid];
affirmNotRO('xWrite', fh);
@@ -367,82 +404,171 @@ const vfsAsyncImpls = {
}catch(e){
error("xWrite():",e,fh);
rc = state.sq3Codes.SQLITE_IOERR_WRITE;
}finally{
wTimeEnd();
}
storeAndNotify('xWrite',rc);
mTimeEnd();
}
};
const initS11n = ()=>{
// Achtung: this code is 100% duplicated in the other half of this proxy!
/**
Historical note: this impl was initially about 1% this size by using
using JSON.stringify/parse(), but using fit-to-purpose serialization
saves considerable runtime.
*/
if(state.s11n) return state.s11n;
const jsonDecoder = new TextDecoder(),
jsonEncoder = new TextEncoder('utf-8'),
viewSz = new DataView(state.sabIO, state.sabS11nOffset, 4),
viewJson = new Uint8Array(state.sabIO, state.sabS11nOffset+4, state.sabS11nSize-4);
const textDecoder = new TextDecoder(),
textEncoder = new TextEncoder('utf-8'),
viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize),
viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize);
state.s11n = Object.create(null);
const TypeIds = Object.create(null);
TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' };
TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' };
TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' };
TypeIds.string = { id: 4 };
const getTypeId = (v)=>{
return TypeIds[typeof v] || toss("This value type cannot be serialized.",v);
};
const getTypeIdById = (tid)=>{
switch(tid){
case TypeIds.number.id: return TypeIds.number;
case TypeIds.bigint.id: return TypeIds.bigint;
case TypeIds.boolean.id: return TypeIds.boolean;
case TypeIds.string.id: return TypeIds.string;
default: toss("Invalid type ID:",tid);
}
};
/**
Returns an array of the state serialized by the most recent
serialize() operation (here or in the counterpart thread), or
null if the serialization buffer is empty.
*/
state.s11n.deserialize = function(){
const sz = viewSz.getInt32(0, state.littleEndian);
const json = sz ? jsonDecoder.decode(
viewJson.slice(0, sz)
/* slice() (copy) needed, instead of subarray() (reference),
because TextDecoder throws if asked to decode from an
SAB. */
) : null;
return JSON.parse(json);
}
++metrics.s11n.deserialize.count;
const t = performance.now();
let rc = null;
const argc = viewU8[0];
if(argc){
rc = [];
let offset = 1, i, n, v, typeIds = [];
for(i = 0; i < argc; ++i, ++offset){
typeIds.push(getTypeIdById(viewU8[offset]));
}
for(i = 0; i < argc; ++i){
const t = typeIds[i];
if(t.getter){
v = viewDV[t.getter](offset, state.littleEndian);
offset += t.size;
}else{
n = viewDV.getInt32(offset, state.littleEndian);
offset += 4;
v = textDecoder.decode(viewU8.slice(offset, offset+n));
offset += n;
}
rc.push(v);
}
}
//log("deserialize:",argc, rc);
metrics.s11n.deserialize.time += performance.now() - t;
return rc;
};
/**
Serializes all arguments to the shared buffer for consumption
by the counterpart thread. This impl currently uses JSON for
serialization for simplicy of implementation, but if that
proves imperformant then a lower-level approach will be
created.
by the counterpart thread.
This routine is only intended for serializing OPFS VFS
arguments and (in at least one special case) result values,
and the buffer is sized to be able to comfortably handle
those.
If passed no arguments then it zeroes out the serialization
state.
*/
state.s11n.serialize = function(...args){
const json = jsonEncoder.encode(JSON.stringify(args));
viewSz.setInt32(0, json.byteLength, state.littleEndian);
viewJson.set(json);
++metrics.s11n.serialize.count;
const t = performance.now();
if(args.length){
//log("serialize():",args);
let i = 0, offset = 1, typeIds = [];
viewU8[0] = args.length & 0xff;
for(; i < args.length; ++i, ++offset){
typeIds.push(getTypeId(args[i]));
viewU8[offset] = typeIds[i].id;
}
for(i = 0; i < args.length; ++i) {
const t = typeIds[i];
if(t.setter){
viewDV[t.setter](offset, args[i], state.littleEndian);
offset += t.size;
}else{
const s = textEncoder.encode(args[i]);
viewDV.setInt32(offset, s.byteLength, state.littleEndian);
offset += 4;
viewU8.set(s, offset);
offset += s.byteLength;
}
}
//log("serialize() result:",viewU8.slice(0,offset));
}else{
viewU8[0] = 0;
}
metrics.s11n.serialize.time += performance.now() - t;
};
return state.s11n;
};
}/*initS11n()*/;
const waitLoop = function(){
const waitLoop = async function f(){
const opHandlers = Object.create(null);
for(let k of Object.keys(state.opIds)){
const vi = vfsAsyncImpls[k];
if(!vi) continue;
const o = Object.create(null);
opHandlers[state.opIds[k]] = o;
o.key = k;
o.f = vi;// || toss("No vfsAsyncImpls[",k,"]");
}
const sabOP = state.sabOP;
for(;;){
let metricsTimer = self.location.port>=1024 ? performance.now() : 0;
// ^^^ in dev environment, dump out these metrics one time after a delay.
while(true){
try {
Atomics.store(sabOP, state.opIds.whichOp, 0);
Atomic.wait(sabOP, state.opIds.whichOp);
const opId = Atomics.load(sabOP, state.opIds.whichOp);
if('timed-out'===Atomics.wait(state.sabOPView, state.opIds.whichOp, 0, 500)){
continue;
}
const opId = Atomics.load(state.sabOPView, state.opIds.whichOp);
Atomics.store(state.sabOPView, state.opIds.whichOp, 0);
const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId);
const args = state.s11n.deserialize();
log("whichOp =",opId,hnd,args);
const rc = 0/*TODO: run op*/;
Atomics.store(sabOP, state.rcIds[hnd.key], rc);
Atomics.notify(sabOP, state.rcIds[hnd.key]);
//warn("waitLoop() whichOp =",opId, hnd, args);
if(hnd.f) await hnd.f(...args);
else error("Missing callback for opId",opId);
}catch(e){
error('in waitLoop():',e.message);
}finally{
// We can't call metrics.dump() from the dev console because this
// thread is continually tied up in Atomics.wait(), so let's
// do, for dev purposes only, a dump one time after 60 seconds.
if(metricsTimer && (performance.now() > metricsTimer + 60000)){
metrics.dump();
metricsTimer = 0;
}
}
}
};
};
navigator.storage.getDirectory().then(function(d){
const wMsg = (type)=>postMessage({type});
state.rootDir = d;
log("state.rootDir =",state.rootDir);
self.onmessage = async function({data}){
log("self.onmessage()",data);
self.onmessage = function({data}){
switch(data.type){
case 'opfs-async-init':{
/* Receive shared state from synchronous partner */
@@ -458,7 +584,6 @@ navigator.storage.getDirectory().then(function(d){
state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize);
state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize);
state.opIds = opt.opIds;
state.rcIds = opt.rcIds;
state.sq3Codes = opt.sq3Codes;
Object.keys(vfsAsyncImpls).forEach((k)=>{
if(!Number.isFinite(state.opIds[k])){
@@ -469,20 +594,7 @@ navigator.storage.getDirectory().then(function(d){
metrics.reset();
log("init state",state);
wMsg('opfs-async-inited');
break;
}
default:{
let err;
const m = vfsAsyncImpls[data.type] || toss("Unknown message type:",data.type);
try {
await m(...data.args).catch((e)=>err=e);
}catch(e){
err = e;
}
if(err){
error("Error handling",data.type+"():",e);
storeAndNotify(data.type, state.sq3Codes.SQLITE_ERROR);
}
waitLoop();
break;
}
}
manifest
+11 -11
View File
@@ -1,5 +1,5 @@
C OPFS\sVFS:\sfurther\sinternal\srefactoring\stowards\sexperimenting\swith\sa\snew\scomms\smodel.
D 2022-09-20T03:31:02.139
C Merge\sopfs-proxy-atomics\sbranch\sinto\sthe\sfiddle-opfs\sbranch.
D 2022-09-20T14:39:54.222
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -474,7 +474,7 @@ F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb
F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3
F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02
F ext/wasm/GNUmakefile b6a5b642e8b3e587d3edcfeb6b6275acbe4730293f4ad46c4997cd932d57aec5
F ext/wasm/GNUmakefile e6359a72f044a877ac6ca8f2b04fa643664157f84d81a6d9ad09f384a65160d2
F ext/wasm/README.md e1ee1e7c321c6a250bf78a84ca6f5882890a237a450ba5a0649c7a8399194c52
F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 150a793a47205b8009ac934f3b6d6ebf67b965c072339aaa25ce808a19e116cc
F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287
@@ -484,12 +484,11 @@ F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a
F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77
F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30
F ext/wasm/api/sqlite3-api-oo1.js 2d13dddf0d2b4168a9249f124134d37924331e5b55e05dba18b6d661fbeefe48
F ext/wasm/api/sqlite3-api-opfs.js 351459d571166ff4cebaccd6b8aad2b0fe5eac54a8c777ba52c31c931a3eb2e2
F ext/wasm/api/sqlite3-api-opfs.js ce75aba0cbfb600cf839362012d17b7b2984aeac5189586c9a5a8f37a573a929
F ext/wasm/api/sqlite3-api-prologue.js 0d2639387b94c30f492d4aea6e44fb7b16720808678464559458fd2ae3759655
F ext/wasm/api/sqlite3-api-worker1.js ee4cf149cbacb63d06b536674f822aa5088b7e022cdffc69f1f36cebe2f9fea0
F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
F ext/wasm/api/sqlite3-wasm.c 4130e2df9587f4e4c3afc04c3549d682c8a5c0cfe5b22819a0a86edb7f01b9bd
F ext/wasm/batch-runner-kvvfs.html ef3b2f553abad4f17a2a29ce6526023793a88e8597b7a24b8c7855a030b90a16
F ext/wasm/batch-runner.html 2857a6db7292ac83d1581af865d643fd34235db2df830d10b43b01388c599e04
F ext/wasm/batch-runner.js 6f5b86e0b5519a9a941d9f17ee9c5ecdc63f452f157602fe7fdf87f6275a2b49
F ext/wasm/common/SqliteTestUtil.js 529161a624265ba84271a52db58da022649832fa1c71309fb1e02cc037327a2b
@@ -505,7 +504,7 @@ F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d695
F ext/wasm/fiddle/fiddle-worker.js bccf46045be8824752876f3eec01c223be0616ccac184bffd0024cfe7a3262b8
F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08
F ext/wasm/fiddle/fiddle.js 4ffcfc9a235beebaddec689a549e9e0dfad6dca5c1f0b41f03468d7e76480686
F ext/wasm/index.html de69bc78273fbe024a21dae0f378a87b37ad4cd585ed0cd0a0ed2c5d5afe9fbc
F ext/wasm/index.html 4b74ea334f0bba9923a61246d807a1873784fcef4d3fb342b698613d0a29fd9c
F ext/wasm/jaccwabyt/jaccwabyt.js 0d7f32817456a0f3937fcfd934afeb32154ca33580ab264dab6c285e6dbbd215
F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356afacbdbf312b2588106
F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f
@@ -513,13 +512,13 @@ F ext/wasm/jaccwabyt/jaccwabyt_test.exports 5ff001ef975c426ffe88d7d8a6e96ec725e5
F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06
F ext/wasm/scratchpad-wasmfs-main.js f0836e3576df7a89390d777bb53e142e559e8a79becfb2a5a976490b05a1c4fa
F ext/wasm/speedtest1-wasmfs.html 9d8cd19eab8854d17f7129aa11607cae6f6d9857c505a4aef13000588583d93e
F ext/wasm/speedtest1-worker.html 7bd3ede33e08067f59e8d6e3dc25016fc930222b059d66b2554203bf262aba02
F ext/wasm/speedtest1-worker.html ede59f2c1884bf72e3d650064604b48703c81848250b19b8063d260aa3a2201d
F ext/wasm/speedtest1-worker.js 11e7f68cedd2a83b0e638f94c1d2f58406ba672a7e88b66bff5d4f4284e8ba16
F ext/wasm/speedtest1.html 512addeb3c27c94901178b7bcbde83a6f95c093f9ebe16a2959a0aa0d828cf1d
F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5
F ext/wasm/sqlite3-opfs-async-proxy.js 462081970a6a46d9b2c386474aacad2d81e6629bb554d6cad5c58515f08c8a38
F ext/wasm/sqlite3-opfs-async-proxy.js 9305d92f32d02983c4528b9c801096cfd8295ca7d24e357d90de9bbcb201d035
F ext/wasm/sqlite3-worker1-promiser.js 4fd0465688a28a75f1d4ee4406540ba494f49844e3cad0670d0437a001943365
F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e
F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5
@@ -2026,8 +2025,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P d4d63e4580ad8d497310608175308c03c517e051d7865cb66aa0b10356612d7d
R 7fef73d3d6085edd0d87db08b80a36c1
P 5ca412ced24b4e3af5f467e710a597ed440badf7b8335346aade11d3cad3d1a1 1e43855498b4329f733bd6e0731841dc884e885de7d2d338d402d05d54309427
R 41b4a0b0d5548be3b8c342b39c8cce92
T +closed 1e43855498b4329f733bd6e0731841dc884e885de7d2d338d402d05d54309427 Closed\sby\sintegrate-merge.
U stephan
Z d9dc6ab88cc2471cfc1f140af8fec063
Z 919304ebee6398ab381c09fc060e29a5
# Remove this line to create a well-formed Fossil manifest.
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
5ca412ced24b4e3af5f467e710a597ed440badf7b8335346aade11d3cad3d1a1
25a36920d44544547a84161681cd41e292b4a70df60ac3630791873a79237d98