Add initial bits of an experimental async-impl-via-synchronous-interface proxy intended to marshal OPFS via sqlite3_vfs API.

FossilOrigin-Name: 38da059b472415da52f57de7332fbeb8a91e3add1f4be3ff9c1924b52672f77c
This commit is contained in:
stephan
2022-09-17 15:08:22 +00:00
parent ce260f4e43
commit 132a87baa3
7 changed files with 469 additions and 12 deletions
+4 -3
View File
@@ -34,11 +34,12 @@
self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
const warn = console.warn.bind(console),
error = console.error.bind(console);
if(!self.importScripts || !self.FileSystemFileHandle){
//|| !self.FileSystemFileHandle.prototype.createSyncAccessHandle){
// ^^^ sync API is not required with WASMFS/OPFS backend.
if(self.window===self || !self.importScripts || !self.FileSystemFileHandle
|| !self.FileSystemFileHandle.prototype.createSyncAccessHandle){
warn("OPFS is not available in this environment.");
return;
}else if(!navigator.storage.getDirectory){
warn("The OPFS VFS requires navigator.storage.getDirectory.");
}else if(!sqlite3.capi.wasm.bigIntEnabled){
error("OPFS requires BigInt support but sqlite3.capi.wasm.bigIntEnabled is false.");
return;
+7 -1
View File
@@ -49,7 +49,13 @@
reminder: we cannot currently (2022-09-15) load WASMFS in a
worker due to an Emscripten limitation.</li>
<li><a href='scratchpad-opfs-worker.html'>scratchpad-opfs-worker</a>:
experimenting with OPFS from a Worker thread (without WASMFS).
experimenting with OPFS from a Worker thread (without WASMFS).</li>
<li><a href='x-sync-async.html'>x-sync-async</a> is an
experiment in implementing a syncronous sqlite3 VFS proxy
for a fully synchronous backend interface (namely OPFS), using SharedArrayBuffer
and the Atomics APIs to regulate communication between the synchronous
interface and the async impl.
</li>
<!--li><a href='x.html'></a></li-->
</ul>
</div>
+286
View File
@@ -0,0 +1,286 @@
/*
2022-09-16
The author disclaims copyright to this source code. In place of a
legal notice, here is a blessing:
* May you do good and not evil.
* May you find forgiveness for yourself and forgive others.
* May you share freely, never taking more than you give.
***********************************************************************
A EXTREMELY INCOMPLETE and UNDER CONSTRUCTION experiment for OPFS: a
Worker which manages asynchronous OPFS handles on behalf of a
synchronous API which controls it via a combination of Worker
messages, SharedArrayBuffer, and Atomics.
Highly indebted to:
https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/OriginPrivateFileSystemVFS.js
for demonstrating how to use the OPFS APIs.
*/
'use strict';
(function(){
const toss = function(...args){throw new Error(args.join(' '))};
if(self.window === self){
toss("This code cannot run from the main thread.",
"Load it as a Worker from a separate Worker.");
}else if(!navigator.storage.getDirectory){
toss("This API requires navigator.storage.getDirectory.");
}
const logPrefix = "OPFS worker:";
const log = (...args)=>{
console.log(logPrefix,...args);
};
const warn = (...args)=>{
console.warn(logPrefix,...args);
};
const error = (...args)=>{
console.error(logPrefix,...args);
};
warn("This file is very much experimental and under construction.",self.location.pathname);
const wMsg = (type,payload)=>postMessage({type,payload});
const state = Object.create(null);
/*state.opSab;
state.sabIO;
state.opBuf;
state.opIds;
state.rootDir;*/
/**
Map of sqlite3_file pointers (integers) to metadata related to a
given OPFS file handles. The pointers are, in this side of the
interface, opaque file handle IDs provided by the synchronous
part of this constellation. Each value is an object with a structure
demonstrated in the xOpen() impl.
*/
state.openFiles = Object.create(null);
/**
Map of dir names to FileSystemDirectoryHandle objects.
*/
state.dirCache = new Map;
const __splitPath = (absFilename)=>{
const a = absFilename.split('/').filter((v)=>!!v);
return [a, a.pop()];
};
/**
Takes the absolute path to a filesystem element. Returns an array
of [handleOfContainingDir, filename]. If the 2nd argument is
truthy then each directory element leading to the file is created
along the way. Throws if any creation or resolution fails.
*/
const getDirForPath = async function f(absFilename, createDirs = false){
const url = new URL(
absFilename, 'file://xyz'
) /* use URL to resolve path pieces such as a/../b */;
const [path, filename] = __splitPath(url.pathname);
const allDirs = path.join('/');
let dh = state.dirCache.get(allDirs);
if(!dh){
dh = state.rootDir;
for(const dirName of path){
if(dirName){
dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs});
}
}
state.dirCache.set(allDirs, dh);
}
return [dh, filename];
};
/**
Generates a random ASCII string len characters long, intended for
use as a temporary file name.
*/
const randomFilename = function f(len=16){
if(!f._chars){
f._chars = "abcdefghijklmnopqrstuvwxyz"+
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"+
"012346789";
f._n = f._chars.length;
}
const a = [];
let i = 0;
for( ; i < len; ++i){
const ndx = Math.random() * (f._n * 64) % f._n | 0;
a[i] = f._chars[ndx];
}
return a.join('');
};
const storeAndNotify = (opName, value)=>{
log(opName+"() is notify()ing w/ value:",value);
Atomics.store(state.opBuf, state.opIds[opName], value);
Atomics.notify(state.opBuf, state.opIds[opName]);
};
const isInt32 = function(n){
return ('bigint'!==typeof n /*TypeError: can't convert BigInt to number*/)
&& !!(n===(n|0) && n<=2147483647 && n>=-2147483648);
};
const affirm32Bits = function(n){
return isInt32(n) || toss("Number is too large (>31 bits):",n);
};
const ioMethods = {
xAccess: async function({filename, exists, readWrite}){
log("xAccess(",arguments,")");
const rc = 1;
storeAndNotify('xAccess', rc);
},
xClose: async function(fid){
const opName = 'xClose';
log(opName+"(",arguments[0],")");
log("state.openFiles",state.openFiles);
const fh = state.openFiles[fid];
if(fh){
delete state.openFiles[fid];
//await fh.close();
if(fh.accessHandle) await fh.accessHandle.close();
if(fh.deleteOnClose){
try{
await fh.dirHandle.removeEntry(fh.filenamePart);
}
catch(e){
warn("Ignoring dirHandle.removeEntry() failure of",fh);
}
}
log("state.openFiles",state.openFiles);
storeAndNotify(opName, 0);
}else{
storeAndNotify(opName, state.errCodes.NotFound);
}
},
xDelete: async function(filename){
log("xDelete(",arguments,")");
storeAndNotify('xClose', 0);
},
xFileSize: async function(fid){
log("xFileSize(",arguments,")");
const fh = state.openFiles[fid];
const sz = await fh.getSize();
affirm32Bits(sz);
storeAndNotify('xFileSize', sz | 0);
},
xOpen: async function({
fid/*sqlite3_file pointer*/, sab/*file-specific SharedArrayBuffer*/,
filename,
fileType = undefined /*mainDb, mainJournal, etc.*/,
create = false, readOnly = false, deleteOnClose = false,
}){
const opName = 'xOpen';
try{
if(create) readOnly = false;
log(opName+"(",arguments[0],")");
let hDir, filenamePart, hFile;
try {
[hDir, filenamePart] = await getDirForPath(filename, !!create);
}catch(e){
storeAndNotify(opName, state.errCodes.NotFound);
return;
}
hFile = await hDir.getFileHandle(filenamePart, {create: !!create});
log(opName,"filenamePart =",filenamePart, 'hDir =',hDir);
const fobj = state.openFiles[fid] = Object.create(null);
fobj.filenameAbs = filename;
fobj.filenamePart = filenamePart;
fobj.dirHandle = hDir;
fobj.fileHandle = hFile;
fobj.accessHandle = undefined;
fobj.fileType = fileType;
fobj.sab = sab;
fobj.create = !!create;
fobj.readOnly = !!readOnly;
fobj.deleteOnClose = !!deleteOnClose;
/**
wa-sqlite, at this point, grabs a SyncAccessHandle and
assigns it to the accessHandle prop of the file state
object, but it's unclear why it does that.
*/
storeAndNotify(opName, 0);
}catch(e){
error(opName,e);
storeAndNotify(opName, state.errCodes.IO);
}
},
xRead: async function({fid,n,offset}){
log("xRead(",arguments,")");
affirm32Bits(n + offset);
const fh = state.openFiles[fid];
storeAndNotify('xRead',fid);
},
xSleep: async function f({ms}){
log("xSleep(",arguments[0],")");
await new Promise((resolve)=>{
setTimeout(()=>resolve(), ms);
}).finally(()=>storeAndNotify('xSleep',0));
},
xSync: async function({fid}){
log("xSync(",arguments,")");
const fh = state.openFiles[fid];
await fh.flush();
storeAndNotify('xSync',fid);
},
xTruncate: async function({fid,size}){
log("xTruncate(",arguments,")");
affirm32Bits(size);
const fh = state.openFiles[fid];
fh.truncate(size);
storeAndNotify('xTruncate',fid);
},
xWrite: async function({fid,src,n,offset}){
log("xWrite(",arguments,")");
const fh = state.openFiles[fid];
storeAndNotify('xWrite',fid);
}
};
const onReady = function(){
self.onmessage = async function({data}){
log("self.onmessage",data);
switch(data.type){
case 'init':{
const opt = data.payload;
state.opSab = opt.opSab;
state.opBuf = new Int32Array(state.opSab);
state.opIds = opt.opIds;
state.errCodes = opt.errCodes;
state.sq3Codes = opt.sq3Codes;
Object.keys(ioMethods).forEach((k)=>{
if(!state.opIds[k]){
toss("Maintenance required: missing state.opIds[",k,"]");
}
});
log("init state",state);
break;
}
default:{
const m = ioMethods[data.type] || toss("Unknown message type:",data.type);
try {
await m(data.payload);
}catch(e){
error("Error handling",data.type+"():",e);
storeAndNotify(data.type, -99);
}
break;
}
}
};
wMsg('ready');
};
navigator.storage.getDirectory().then(function(d){
state.rootDir = d;
log("state.rootDir =",state.rootDir);
onReady();
});
})();
+28
View File
@@ -0,0 +1,28 @@
<!doctype html>
<html lang="en-us">
<head>
<meta charset="utf-8">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon">
<link rel="stylesheet" href="common/emscripten.css"/>
<link rel="stylesheet" href="common/testing.css"/>
<title>Async-behind-Sync experiment</title>
</head>
<body>
<header id='titlebar'><span>Async-behind-Sync Experiment</span></header>
<div>This is an experiment in wrapping the
asynchronous OPFS APIs behind a fully synchronous proxy. It is
very much incomplete, under construction, and experimental.
See the dev console for all output.
</div>
<div id='test-output'>
</div>
<!--script src="common/whwasmutil.js"></script-->
<!--script src="common/SqliteTestUtil.js"></script-->
<script>
(function(){
new Worker("x-sync-async.js");
})();
</script>
</body>
</html>
+133
View File
@@ -0,0 +1,133 @@
'use strict';
const doAtomicsStuff = function(sqlite3){
const logPrefix = "OPFS syncer:";
const log = (...args)=>{
console.log(logPrefix,...args);
};
const warn = (...args)=>{
console.warn(logPrefix,...args);
};
const error = (...args)=>{
console.error(logPrefix,...args);
};
const W = new Worker("sqlite3-opfs-async-proxy.js");
const wMsg = (type,payload)=>W.postMessage({type,payload});
warn("This file is very much experimental and under construction.",self.location.pathname);
/**
State which we send to the async-api Worker or share with it.
This object must initially contain only cloneable or sharable
objects. After the worker's "ready" message arrives, other types
of data may be added to it.
*/
const state = Object.create(null);
state.opIds = Object.create(null);
state.opIds.xAccess = 1;
state.opIds.xClose = 2;
state.opIds.xDelete = 3;
state.opIds.xFileSize = 4;
state.opIds.xOpen = 5;
state.opIds.xRead = 6;
state.opIds.xSync = 7;
state.opIds.xTruncate = 8;
state.opIds.xWrite = 9;
state.opIds.xSleep = 10;
state.opIds.xBlock = 99 /* to block worker while this code is still handling something */;
state.opSab = new SharedArrayBuffer(64);
state.fileBufferSize = 1024 * 65 /* 64k = max sqlite3 page size */;
/* TODO: use SQLITE_xxx err codes. */
state.errCodes = Object.create(null);
state.errCodes.Error = -100;
state.errCodes.IO = -101;
state.errCodes.NotFound = -102;
state.errCodes.Misuse = -103;
// TODO: add any SQLITE_xxx symbols we need here.
state.sq3Codes = Object.create(null);
const isWorkerErrCode = (n)=>(n<=state.errCodes.Error);
const opStore = (op,val=-1)=>Atomics.store(state.opBuf, state.opIds[op], val);
const opWait = (op,val=-1)=>Atomics.wait(state.opBuf, state.opIds[op], val);
const opRun = (op,args)=>{
opStore(op);
wMsg(op, args);
opWait(op);
return Atomics.load(state.opBuf, state.opIds[op]);
};
const wait = (ms,value)=>{
return new Promise((resolve)=>{
setTimeout(()=>resolve(value), ms);
});
};
const vfsSyncWrappers = {
xOpen: function f(pFile, name, flags, outFlags = {}){
if(!f._){
f._ = {
// TODO: map openFlags to args.fileType names.
};
}
const args = Object.create(null);
args.fid = pFile;
args.filename = name;
args.sab = new SharedArrayBuffer(state.fileBufferSize);
args.fileType = undefined /*TODO: populate based on SQLITE_OPEN_xxx */;
// TODO: populate args object based on flags:
// args.create, args.readOnly, args.deleteOnClose
args.create = true;
args.deleteOnClose = true;
const rc = opRun('xOpen', args);
if(!rc){
outFlags.readOnly = args.readOnly;
args.ba = new Uint8Array(args.sab);
state.openFiles[pFile] = args;
}
return rc;
},
xClose: function(pFile){
let rc = 0;
if(state.openFiles[pFile]){
delete state.openFiles[pFile];
rc = opRun('xClose', pFile);
}
return rc;
}
};
const doSomething = function(){
//state.ioBuf = new Uint8Array(state.sabIo);
const fid = 37;
let rc = vfsSyncWrappers.xOpen(fid, "/foo/bar/baz.sqlite3",0, {});
log("open rc =",rc,"state.opBuf[xOpen] =",state.opBuf[state.opIds.xOpen]);
if(isWorkerErrCode(rc)){
error("open failed with code",rc);
return;
}
log("xSleep()ing before close()ing...");
opRun('xSleep',{ms: 1500});
log("wait()ing before close()ing...");
wait(1500).then(function(){
rc = vfsSyncWrappers.xClose(fid);
log("xClose rc =",rc,"opBuf =",state.opBuf);
});
};
W.onmessage = function({data}){
log("Worker.onmessage:",data);
switch(data.type){
case 'ready':
wMsg('init',state);
state.opBuf = new Int32Array(state.opSab);
state.openFiles = Object.create(null);
doSomething();
break;
}
};
}/*doAtomicsStuff*/
importScripts('sqlite3.js');
self.sqlite3InitModule().then((EmscriptenModule)=>doAtomicsStuff(EmscriptenModule.sqlite3));
+10 -7
View File
@@ -1,5 +1,5 @@
C Merge\skv-vfs\sbranch\sinto\sfiddle-opfs\sbranch.\sAdjust\sspeedtest1\s--size\sflags\sto\saccount\sfor\snew\ssize\slimit.
D 2022-09-16T20:16:50.240
C Add\sinitial\sbits\sof\san\sexperimental\sasync-impl-via-synchronous-interface\sproxy\sintended\sto\smarshal\sOPFS\svia\ssqlite3_vfs\sAPI.
D 2022-09-17T15:08:22.642
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -484,7 +484,7 @@ F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a
F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77
F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30
F ext/wasm/api/sqlite3-api-oo1.js d7526517f7ad3f6bda16ad66d373bbb71b43168deef7af60eda5c9fe873d1387
F ext/wasm/api/sqlite3-api-opfs.js 011799db398157cbd254264b6ebae00d7234b93d0e9e810345f213a5774993c0
F ext/wasm/api/sqlite3-api-opfs.js 130f60cc8f5835f9d77d4f12308bf4c8fb6d9c315009fc7239c5d67ff2bc8c67
F ext/wasm/api/sqlite3-api-prologue.js 48ebca4ae340b0242d4f39bbded01bd0588393c8023628be1c454b4db6f7bd6e
F ext/wasm/api/sqlite3-api-worker1.js d33062afa045fd4be01ba4abc266801807472558b862b30056211b00c9c347b4
F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
@@ -502,7 +502,7 @@ F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d695
F ext/wasm/fiddle/fiddle-worker.js bccf46045be8824752876f3eec01c223be0616ccac184bffd0024cfe7a3262b8
F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08
F ext/wasm/fiddle/fiddle.js 4ffcfc9a235beebaddec689a549e9e0dfad6dca5c1f0b41f03468d7e76480686
F ext/wasm/index.html 095b9a8cee9aac2654c23686ead22f3452b89d581fb41d34d47b6548546b5365
F ext/wasm/index.html 8365e47e2aff1829923f0481292948487b27a0755fde9c0d3ad15f7ea0118992
F ext/wasm/jaccwabyt/jaccwabyt.js 0d7f32817456a0f3937fcfd934afeb32154ca33580ab264dab6c285e6dbbd215
F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356afacbdbf312b2588106
F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f
@@ -523,6 +523,7 @@ F ext/wasm/speedtest1.html fbb8e4d1639028443f3687a683be660beca6927920545cf6b1fdf
F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5
F ext/wasm/sqlite3-opfs-async-proxy.js c42a097dfbb96abef08554b173a47788f5bc1f58c266f859ba01c1fa3ff8327d
F ext/wasm/sqlite3-worker1-promiser.js 92b8da5f38439ffec459a8215775d30fa498bc0f1ab929ff341fc3dd479660b9
F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e
F ext/wasm/testing-worker1-promiser.html 6eaec6e04a56cf24cf4fa8ef49d78ce8905dde1354235c9125dca6885f7ce893
@@ -532,6 +533,8 @@ F ext/wasm/testing1.js 7cd8ab255c238b030d928755ae8e91e7d90a12f2ae601b1b8f7827aaa
F ext/wasm/testing2.html a66951c38137ff1d687df79466351f3c734fa9c6d9cce71d3cf97c291b2167e3
F ext/wasm/testing2.js 25584bcc30f19673ce13a6f301f89f8820a59dfe044e0c4f2913941f4097fe3c
F ext/wasm/wasmfs.make 21a5cf297954a689e0dc2a95299ae158f681cae5e90c10b99d986097815fd42d
F ext/wasm/x-sync-async.html 283539e4fcca8c60fea18dbf1f1c0df168340145a19123f8fd5b70f41291b36f
F ext/wasm/x-sync-async.js 42da502ea0b89bfa226c7ac7555c0c87d4ab8a10221ea6fadb4f7877c26a5137
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
@@ -2027,8 +2030,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 52d1b185b9f6cee1eb3dec436f47e0f52e4621a127abfad8c27f92fd78147889 ef54961ce69fddb4cfeeff0860288de2858a6f7a5aa396691e8e99933eb9af54
R e35abbf6d1d9b2b828d30426a9a9f018
P afb79050e635f3c698e51f06c346cbf23b096cfda7d0f1d8e68514ea0c25b7b7
R dbdee404fc63f84ef61ddb4fd79c0ad1
U stephan
Z 0ddb259c4784fc46ece386ccc85309e5
Z 67d0abd2dfaa993776fbe8c511ee53f5
# Remove this line to create a well-formed Fossil manifest.
+1 -1
View File
@@ -1 +1 @@
afb79050e635f3c698e51f06c346cbf23b096cfda7d0f1d8e68514ea0c25b7b7
38da059b472415da52f57de7332fbeb8a91e3add1f4be3ff9c1924b52672f77c