paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

The Worker-specific variants of the most significant DB-class JS bindings are implemented, most notably various uses of DB.exec().

Browse Source 
FossilOrigin-Name: 371d6f7497016ca9bf82c8524b4d701ddf1e614b3fb4e69ff63935da0d51ed05
This commit is contained in:
stephan
2022-06-01 11:20:07 +00:00
parent 645682a7c7
commit ea7a4e4fb3
6 changed files with 315 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ext/fiddle/fiddle-worker.js
+3 -2
View File
@@ -234,8 +234,9 @@
fiddleModule.FS.createDataFile("/", fn, buffer, true, true);
const oldName = Sqlite3Shell.dbFilename();
Sqlite3Shell.exec('.open "/'+fn+'"');
if(oldName !== fn){
fiddleModule.FS.unlink(oldName);
if(oldName && oldName !== fn){
try{fiddleModule.FS.unlink(oldName);}
catch(e){/*ignored*/}
}
stdout("Replaced DB with",fn+".");
return;
ext/fiddle/sqlite3-api.js
+202 -48
View File
@@ -278,7 +278,8 @@ Module.postRun.push(function(namespace/*the module object, the target for
image.
If the filename is provided, only the last component of the
path is used - any path prefix is stripped. If no name is
path is used - any path prefix is stripped and certain
"special" characters are replaced with `_`. If no name is
provided, a random name is generated. The resulting filename is
the one used for accessing the db file within root directory of
the emscripten-supplied virtual filesystem, and is set (with no
@@ -300,7 +301,7 @@ Module.postRun.push(function(namespace/*the module object, the target for
arg = undefined;
}else if(arguments.length){ /*(filename[,buffer])*/
if('string'===typeof arg){
const p = arg.split('/').pop().replace(':','');
const p = arg.split('/').pop().replace(':','_');
if(p) fn = p;
if(arguments.length>1){
buffer = arguments[1];
@@ -406,7 +407,7 @@ Module.postRun.push(function(namespace/*the module object, the target for
if('string'!==typeof out.sql) toss("Missing SQL argument.");
if(out.opt.callback || out.opt.resultRows){
switch((undefined===out.opt.rowMode)
? 'stmt' : out.opt.rowMode) {
? 'stmt' : out.opt.rowMode) {
case 'object': out.cbArg = (stmt)=>stmt.get({}); break;
case 'array': out.cbArg = (stmt)=>stmt.get([]); break;
case 'stmt': out.cbArg = (stmt)=>stmt; break;
@@ -440,9 +441,12 @@ Module.postRun.push(function(namespace/*the module object, the target for
/**
Finalizes all open statements and closes this database
connection. This is a no-op if the db has already been
closed.
closed. If the db is open and alsoUnlink is truthy then the
this.filename entry in the pseudo-filesystem will also be
removed (and any error in that attempt is silently
ignored).
*/
close: function(){
close: function(alsoUnlink){
if(this._pDb){
let s;
const that = this;
@@ -453,9 +457,15 @@ Module.postRun.push(function(namespace/*the module object, the target for
Object.values(this._udfs).forEach(SQM.removeFunction);
delete this._udfs;
delete this._statements;
delete this.filename;
api.sqlite3_close_v2(this._pDb);
delete this._pDb;
if(this.filename){
if(alsoUnlink){
try{SQM.FS.unlink(this.filename);}
catch(e){/*ignored*/}
}
delete this.filename;
}
}
},
/**
@@ -491,7 +501,7 @@ Module.postRun.push(function(namespace/*the module object, the target for
one.
This function supports the following additional options not
used by execMulti():
supported by execMulti():
- .multi: if true, this function acts as a proxy for
execMulti() and behaves identically to that function.
@@ -499,10 +509,11 @@ Module.postRun.push(function(namespace/*the module object, the target for
- .resultRows: if this is an array, each row of the result
set (if any) is appended to it in the format specified
for the `rowMode` property, with the exception that the
`rowMode` property _must_ be one of 'array' or 'object'
if this is set (else an exception is throws). It is legal
only legal values for `rowMode` in this case are 'array'
or 'object', neither of which is the default. It is legal
to use both `resultRows` and `callback`, but `resultRows`
is likely much simpler to use for small data sets.
is likely much simpler to use for small data sets and can
be used over a WebWorker-style message interface.
- .columnNames: if this is an array and the query has
result columns, the array is passed to
@@ -528,8 +539,8 @@ Module.postRun.push(function(namespace/*the module object, the target for
try {
if(Array.isArray(opt.resultRows)){
if(opt.rowMode!=='array' && opt.rowMode!=='object'){
throw new Error("Invalid rowMode for resultRows array: must "+
"be one of 'array' or 'object'.");
toss("Invalid rowMode for resultRows array: must",
"be one of 'array' or 'object'.");
}
rowTarget = opt.resultRows;
}
@@ -1316,6 +1327,13 @@ Module.postRun.push(function(namespace/*the module object, the target for
const ptr = api.sqlite3_column_blob(this._pStmt, ndx);
const rc = new Uint8Array(n);
for(let i = 0; i < n; ++i) rc[i] = HEAP8[ptr + i];
if(n && this.db._blobXfer instanceof Array){
/* This is an optimization soley for the
Worker-based API. These values will be
transfered to the main thread directly
instead of being copied. */
this.db._blobXfer.push(rc.buffer);
}
return rc;
}
default: toss("Don't know how to translate",
@@ -1462,13 +1480,9 @@ Module.postRun.push(function(namespace/*the module object, the target for
SQLite3
};
const postApiLoaded = function(){
setTimeout(()=>postMessage({type:'sqlite3-api',data:'loaded'}), 0);
};
if(self === self.window){
/* This is running in the main window thread, so we're done. */
postApiLoaded();
setTimeout(()=>postMessage({type:'sqlite3-api',data:'loaded'}), 0);
return;
}
/******************************************************************
@@ -1506,18 +1520,170 @@ Module.postRun.push(function(namespace/*the module object, the target for
engine's memory.
*/
/**
Helper for managing Worker-level state.
*/
const wState = {
db: undefined,
open: function(arg){
if(!arg && this.db) return this.db;
else if(this.db) this.db.close();
return this.db = (Array.isArray(arg) ? new DB(...arg) : new DB(arg));
},
close: function(){
if(this.db){
this.db.close();
this.db = undefined;
}
},
affirmOpen: function(){
return this.db || toss("DB is not opened.");
},
post: function(type,data,xferList){
if(xferList){
self.postMessage({type, data},xferList);
xferList.length = 0;
}else{
self.postMessage({type, data});
}
}
};
const wMsg = (type,data)=>self.postMessage({type, data});
/**
UNDER CONSTRUCTION:
A level of "organizational abstraction" for the Worker
API. Each method in this object must map directly to a Worker
message type key. The onmessage() dispatcher attempts to
dispatch all inbound messages to a method of this object,
passing it the event.data part of the inbound event object. All
methods must return a plain Object containing any response
state, which the dispatcher may amend. All methods must throw
on error.
*/
const wMsgHandler = {
xfer: [/*Temp holder for "transferable" postMessage() state.*/],
/**
Proxy for DB.exec() which expects a single argument of type
string (SQL to execute) or an options object in the form
expected by exec(). The notable differences from exec()
include:
- The default value for options.rowMode is 'array' because
the normal default cannot cross the window/Worker boundary.
- A function-type options.callback property cannot cross
the window/Worker boundary, so is not useful here. If
options.callback is a string then it is assumed to be a
message type key, in which case a callback function will be
applied which posts each row result via:
postMessage({type: thatKeyType, data: theRow})
And, at the end of the result set (whether or not any
result rows were produced), it will post an identical
message with data:null to alert the caller than the result
set is completed.
The callback proxy must not recurse into this interface, or
results are undefined. (It hypothetically cannot recurse
because an exec() call will be tying up the Worker thread,
causing any recursion attempt to wait until the first
exec() is completed.)
The response is the input options object (or a synthesized
one if passed only a string), noting that
options.resultRows and options.columnNames may be populated
by the call to exec().
This opens/creates the Worker's db if needed.
*/
exec: function(ev){
const opt = (
'string'===typeof ev.data
) ? {sql: ev.data} : (ev.data || {});
if(!opt.rowMode){
/* Since the default rowMode of 'stmt' is not useful
for the Worker interface, we'll default to
something else. */
opt.rowMode = 'array';
}else if('stmt'===opt.rowMode){
toss("Invalid rowMode for exec(): stmt mode",
"does not work in the Worker API.");
}
const db = wState.open();
if(opt.callback || opt.resultRows instanceof Array){
// Part of a copy-avoidance optimization for blobs
db._blobXfer = this.xfer;
}
const callbackMsgType = opt.callback;
if('string' === typeof callbackMsgType){
const that = this;
opt.callback =
(row)=>wState.post(callbackMsgType,row,this.xfer);
}
try {
db.exec(opt);
if(opt.callback instanceof Function){
opt.callback = callbackMsgType;
wState.post(callbackMsgType, null);
}
}finally{
delete db._blobXfer;
if('string'===typeof callbackMsgType){
opt.callback = callbackMsgType;
}
}
return opt;
}/*exec()*/,
/**
Proxy for DB.exportBinaryImage(). Throws if the db has not
been opened. Response is an object:
{
buffer: Uint8Array (db file contents),
filename: the current db filename,
mimetype: string
}
*/
export: function(ev){
const db = wState.affirmOpen();
const response = {
buffer: db.exportBinaryImage(),
filename: db.filename,
mimetype: 'application/x-sqlite3'
};
this.xfer.push(response.buffer.buffer);
return response;
}/*export()*/,
/**
Proxy for the DB constructor. Expects to be passed a single
object or a falsy value to use defaults. The object may
have a filename property to name the db file (see the DB
constructor for peculiarities and transformations) and/or a
buffer property (a Uint8Array holding a complete database
file's contents). The response is an object:
{
filename: db filename (possibly differing from the input)
}
If the Worker's db is currently opened, this call closes it
before proceeding.
*/
open: function(ev){
wState.close(/*true???*/);
const args = [], data = (ev.data || {});
if(data.filename) args.push(data.filename);
if(data.buffer){
args.push(data.buffer);
this.xfer.push(data.buffer.buffer);
}
const db = wState.open(args);
return {filename: db.filename};
}
}/*wMsgHandler*/;
/**
UNDER CONSTRUCTION!
A subset of the DB API is accessible via Worker messages in the form:
@@ -1534,51 +1700,39 @@ Module.postRun.push(function(namespace/*the module object, the target for
during processing result in an `error`-type event with a
payload in the form:
{message: error string,
errorClass: class name of the error type,
[, messageId: if set in the inbound message]}
{
message: error string,
errorClass: class name of the error type,
input: ev.data,
[messageId: if set in the inbound message]
}
The individual APIs will be documented as they are fleshed out.
The individual APIs are documented in the wMsgHandler object.
*/
self.onmessage = function(ev){
ev = ev.data;
let response = {}, evType = ev.type;
let response, evType = ev.type;
try {
switch(evType){
case 'open': {
const args = [], data = (ev.data || {});
if(data.filename) args.push(data.filename);
if(data.buffer) args.push(data.buffer);
const d = wState.open(args);
response.filename = d.filename;
response.messageId = data.messageId;
break;
}
case 'exec': {
const opt = (
'string'===typeof ev.data
) ? {sql: ev.data} : (ev.data || {});
if(!opt.rowMode) opt.rowMode = 'array';
wState.open().exec(opt);
response = opt;
break;
}
default:
throw new Error("Unknown db worker message type: "+ev.type);
if(wMsgHandler.hasOwnProperty(evType) &&
wMsgHandler[evType] instanceof Function){
response = wMsgHandler[evType](ev);
}else{
toss("Unknown db worker message type:",ev.type);
}
}catch(err){
evType = 'error';
response = {
message: err.message,
errorClass: err.name
errorClass: err.name,
input: ev
};
}
if(!response.messageId && ev.data
&& 'object'===typeof ev.data && ev.data.messageId){
response.messageId = ev.data.messageId;
}
wMsg(evType, response);
wState.post(evType, response, wMsgHandler.xfer);
};
postApiLoaded();
setTimeout(()=>postMessage({type:'sqlite3-api',data:'loaded'}), 0);
});
ext/fiddle/sqlite3-worker.js
+29 -20
View File
@@ -10,26 +10,35 @@
***********************************************************************
UNDER CONSTRUCTION
This is a JS Worker file for the main sqlite3 api. It loads
sqlite3.js and offers access to the db via the Worker
message-passing interface.
*/
sqlite3.js, initializes the module, and postMessage()'s a message
after the module is initialized:
{type: 'sqlite3-api', data: 'ready'}
This seemingly superfluous level of indirection is necessary when
loading sqlite3.js via a Worker. Loading sqlite3.js from the main
window thread elides the Worker-specific API. Instantiating a worker
with new Worker("sqlite.js") will not (cannot) call
initSqlite3Module() to initialize the module due to a
timing/order-of-operations conflict (and that symbol is not exported
in a way that a Worker loading it that way can see it). Thus JS
code wanting to load the sqlite3 Worker-specific API needs to pass
_this_ file (or equivalent) to the Worker constructor and then
listen for an event in the form shown above in order to know when
the module has completed initialization. sqlite3.js will fire a
similar event, with data:'loaded' as the final step in its loading
process. Whether or not we _really_ need both 'loaded' and 'ready'
events is unclear, but they are currently separate events primarily
for the sake of clarity in the timing of when it's okay to use the
loaded module. At the time the 'loaded' event is fired, it's
possible (but unknown and unknowable) that the emscripten-generated
module-setup infrastructure still has work to do. Thus it is
hypothesized that client code is better off waiting for the 'ready'
even before using the API.
*/
"use strict";
(function(){
/** Posts a worker message as {type:type, data:data}. */
const wMsg = (type,data)=>self.postMessage({type, data});
self.onmessage = function(ev){
/*ev = ev.data;
switch(ev.type){
default: break;
};*/
console.warn("Unknown sqlite3-worker message type:",ev);
};
importScripts('sqlite3.js');
initSqlite3Module().then(function(){
wMsg('sqlite3-api','ready');
});
})();
importScripts('sqlite3.js');
initSqlite3Module().then(function(){
setTimeout(()=>self.postMessage({type:'sqlite3-api',data:'ready'}), 0);
});
ext/fiddle/testing2.js
+70 -25
View File
@@ -40,17 +40,45 @@
}
};
const testCount = ()=>log("Total test count:",T.counter);
const runOneTest = function(eventType, eventData, callback){
T.assert(eventData && 'object'===typeof eventData);
/* ^^^ that is for the testing and messageId-related code, not
a hard requirement of all of the Worker-exposed APIs. */
eventData.messageId = MsgHandlerQueue.push(eventType,function(ev){
log("runOneTest",eventType,"result",ev.data);
callback(ev);
if(callback instanceof Function){
callback(ev);
testCount();
}
});
wMsg(eventType, eventData);
};
const testCount = ()=>log("Total test count:",T.counter);
/** Methods which map directly to onmessage() event.type keys.
They get passed the inbound event. */
const dbMsgHandler = {
open: function(ev){
log("open result",ev.data);
},
exec: function(ev){
log("exec result",ev.data);
},
export: function(ev){
log("exec result",ev.data);
},
error: function(ev){
error("ERROR from the worker:",ev.data);
},
resultRowTest1: function f(ev){
if(undefined === f.counter) f.counter = 0;
if(ev.data) ++f.counter;
//log("exec() result row:",ev.data);
T.assert(null===ev.data || 'number' === typeof ev.data.b);
}
};
const runTests = function(){
/**
"The problem" now is that the test results are async. We
@@ -66,24 +94,21 @@
//log("open result",ev);
T.assert('testing2.sqlite3'===ev.data.filename)
.assert(ev.data.messageId);
testCount();
});
runOneTest('exec',{
sql: ["create table t(a,b)",
"insert into t(a,b) values(1,2),(3,4),(5,6)"
].join(';'),
multi: true,
resultRows: [],
columnNames: []
resultRows: [], columnNames: []
}, function(ev){
ev = ev.data;
T.assert(0===ev.resultRows.length)
.assert(0===ev.columnNames.length);
testCount();
});
runOneTest('exec',{
sql: 'select a a, b b from t order by a',
resultRows: [], columnNames: []
resultRows: [], columnNames: [],
}, function(ev){
ev = ev.data;
T.assert(3===ev.resultRows.length)
@@ -91,34 +116,54 @@
.assert(6===ev.resultRows[2][1])
.assert(2===ev.columnNames.length)
.assert('b'===ev.columnNames[1]);
testCount();
});
runOneTest('exec',{sql:'select 1 from intentional_error'}, function(){
runOneTest('exec',{
sql: 'select a a, b b from t order by a',
resultRows: [], columnNames: [],
rowMode: 'object'
}, function(ev){
ev = ev.data;
T.assert(3===ev.resultRows.length)
.assert(1===ev.resultRows[0].a)
.assert(6===ev.resultRows[2].b)
});
runOneTest('exec',{sql:'intentional_error'}, function(){
throw new Error("This is not supposed to be reached.");
});
// Ensure that the message-handler queue survives ^^^ that error...
runOneTest('exec',{
sql:'select 1',
resultRows: [],
rowMode: 'array',
//rowMode: 'array', // array is the default in the Worker interface
}, function(ev){
ev = ev.data;
T.assert(1 === ev.resultRows.length)
.assert(1 === ev.resultRows[0][0]);
testCount();
});
};
const dbMsgHandler = {
open: function(ev){
log("open result",ev.data);
},
exec: function(ev){
log("exec result",ev.data);
},
error: function(ev){
error("ERROR from the worker:",ev.data);
}
runOneTest('exec',{
sql: 'select a a, b b from t order by a',
callback: 'resultRowTest1',
rowMode: 'object'
}, function(ev){
T.assert(3===dbMsgHandler.resultRowTest1.counter);
dbMsgHandler.resultRowTest1.counter = 0;
});
runOneTest('exec',{sql: 'delete from t where a>3'});
runOneTest('exec',{
sql: 'select count(a) from t',
resultRows: []
},function(ev){
ev = ev.data;
T.assert(1===ev.resultRows.length)
.assert(2===ev.resultRows[0][0]);
});
runOneTest('export',{}, function(ev){
ev = ev.data;
T.assert('string' === typeof ev.filename)
.assert(ev.buffer instanceof Uint8Array)
.assert(ev.buffer.length > 1024)
.assert('application/x-sqlite3' === ev.mimetype);
});
};
SW.onmessage = function(ev){
@@ -166,5 +211,5 @@
}
};
log("Init complete, but async bits may still be running.");
log("Init complete, but async init bits may still be running.");
})();
manifest
+10 -10
View File
@@ -1,5 +1,5 @@
C Move\sthe\ssqlite_offset()\sfunction\simplementation\sto\sbe\san\sin-line\sfunction,\nthereby\savoiding\sspecial\scase\scode\sand\sfreeing\sup\sa\sbit\sin\sthe\nFuncDef.flags\sfield.
D 2022-06-01T11:05:59.088
C The\sWorker-specific\svariants\sof\sthe\smost\ssignificant\sDB-class\sJS\sbindings\sare\simplemented,\smost\snotably\svarious\suses\sof\sDB.exec().
D 2022-06-01T11:20:07.553
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -61,17 +61,17 @@ F ext/fiddle/EXPORTED_RUNTIME_METHODS b831017ba67ba993b34a27400cef2f6095bd6789c0
F ext/fiddle/Makefile e25d34a0e1324f771d64c09c592601b97219282011587e6ce410fa8acdedb913
F ext/fiddle/SqliteTestUtil.js 559731c3e8e0de330ec7d292e6c1846566408caee6637acc8a119ac338a8781c
F ext/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f
F ext/fiddle/fiddle-worker.js 3a19253dc026d1ad9064ee853f3c4da3385223ce4434dab1838837525d817371
F ext/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae
F ext/fiddle/fiddle.html 724f1cd4126616bc87f5871f78d3f7aaaf41e45c9728724627baab87e6af35f0
F ext/fiddle/fiddle.js 5b456ed7085830cda2fc75a0801476174a978521949335f24bc4154d076dcd4d
F ext/fiddle/index.md d9c1c308d8074341bc3b11d1d39073cd77754cb3ca9aeb949f23fdd8323d81cf
F ext/fiddle/sqlite3-api.js a2c0fa1a30e564a16650e3224a23a60cc8636325028223183933630669ebec8d
F ext/fiddle/sqlite3-worker.js c137daed6529b5f527ed61eb358cb0d23f90e04784442479cd15ac684eccdf7a
F ext/fiddle/sqlite3-api.js 2d7b0e63affa1dcad97de6c5bfef2eb1fb2ecdde44b8991299cd01b988ce9994
F ext/fiddle/sqlite3-worker.js a9c2b614beca187dbdd8c053ec2770cc61ec1ac9c0ec6398ceb49a79f705a421
F ext/fiddle/testing.css 750572dded671d2cf142bbcb27af5542522ac08db128245d0b9fe410aa1d7f2a
F ext/fiddle/testing1.html ea1f3be727f78e420007f823912c1a03b337ecbb8e79449abc2244ad4fe15d9a
F ext/fiddle/testing1.js b5bf7e33b35f02f4208e4d68eaa41e5ed42eaefd57e0a1131e87cba96d4808dc
F ext/fiddle/testing2.html 9063b2430ade2fe9da4e711addd1b51a2741cf0c7ebf6926472a5e5dd63c0bc4
F ext/fiddle/testing2.js 15e53ded82e78a5360daa4af109124c81b52eba79be2de241bef6558697931b7
F ext/fiddle/testing2.js afb3f79c5731b50148201797150a64e210a16730a34a72965599f6985a4f83c6
F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
F ext/fts1/ft_hash.h 06df7bba40dadd19597aa400a875dbc2fed705ea
@@ -1974,8 +1974,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P d9efe3e92d1c95aee6f5ae37a8ba28d8cf4891d746744ce4aa2464f766821a0b
R 52cfcb84f807d4bfd299e794f93f33ab
U drh
Z 74e345d13b420413b02c8b365815c584
P 1c9812c458bd229c862efe5df1b64fae333da9871c8756b5ae4605a81bcda4b5
R b028ebbd2fb0859e58130b8e01609e9c
U stephan
Z b5b92e7ff660a052d41c589f495d31e9
# Remove this line to create a well-formed Fossil manifest.
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
1c9812c458bd229c862efe5df1b64fae333da9871c8756b5ae4605a81bcda4b5
371d6f7497016ca9bf82c8524b4d701ddf1e614b3fb4e69ff63935da0d51ed05