paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Docs, cleanups, and dead code removal.

Browse Source 
FossilOrigin-Name: 49db59aa9c74e49d878adc8671b0d32db8f1f898bde29d046ce0e368d8987868
This commit is contained in:
stephan
2025-11-23 20:05:21 +00:00
parent da0d872270
commit 606b1bd153
3 changed files with 131 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js
+124 -25
View File
@@ -19,6 +19,64 @@
Documentation home page: https://sqlite.org/wasm
*/
/**
kvvfs - the Key/Value VFS - is an SQLite3 VFS which delegates
storage of its pages and metadata to a key-value store.
It was conceived in order to support JS's localStorage and
sessionStorage objects. Its native implementation uses files as
key/value storage (one file per record) but the JS implementation
replaces a few methods so that it can use the aforementioned
objects as storage.
It uses a bespoke ASCII encoding to store each db page as a
separate record and stores some metadata, like the db's encoded
size and its journal, as individual records.
kvvfs is significantly less efficient than a plain in-memory db but
it also, as a side effect of its design, offers a JSON-friendly
interchange format for exporting and importing databases.
kvvfs is _not_ designed for heavy db loads. It is relatively
malloc()-heavy, having to de/allocate frequently, and it
spends much of its time converting the raw db pages into and out of
an ASCII encoding.
But it _does_ work and is "performant enough" for db work of the
scale of a db which will fit within sessionStorage or localStorage
(just 2-3mb).
"Version 2" extends it to support using Storage-like objects as
backing storage, Storage being the JS class which localStorage and
sessionStorage both derive from. This essentially moves the backing
store from whatever localStorage and sessionStorage use to an
in-memory object.
This effort is primarily a stepping stone towards eliminating, if
it proves possible, the POSIX I/O API dependencies in SQLite's WASM
builds. That is: if this VFS works properly, it can be set as the
default VFS and we can eliminate the "unix" VFS from the JS/WASM
builds (as opposed to server-wise/WASI builds). That still, as of
2025-11-23, a ways away, but it's the main driver for version 2 of
kvvfs.
Version 2 remains compatible with version 1 databases and always
writes localStorage/sessionStorage metadata in the v1 format, so
such dbs can be manipulated freely by either version. For transient
storage objects (new in version 2), the format of its record keys
is simpified, requiring less space than v1 keys by eliding
redundant (in this context) info from the keys.
Another benefit of v2 is its ability to export dbs into a
JSON-friendly (but not human-friendly) format.
A potential, as-yet-unproven, benefit, would be the ability to plug
arbitrary Storage-compatible objects in so that clients could,
e.g. asynchronously post updates to db pages to some back-end for
backups.
*/
globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
'use strict';
/* We unregister the kvvfs VFS from Worker threads later on. */
@@ -57,12 +115,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
Implementation of JS's Storage interface for use as backing store
of the kvvfs. Storage is a native class and its constructor
cannot be legally called from JS, making it impossible to
directly subclass Storage.
directly subclass Storage. This class implements the Storage
interface, however, to make it a drop-in replacement for
localStorage/sessionStorage.
This impl simply proxies a plain, prototype-less Object, suitable
for JSON-ing.
*/
class TransientStorage {
class KVVfsStorage {
#map;
#keys;
#getKeys(){return this.#keys ??= Object.keys(this.#map);}
@@ -101,7 +161,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
get length() {
return this.#getKeys().length;
}
}/*TransientStorage*/;
}/*KVVfsStorage*/;
/**
Map of JS-stringified KVVfsFile::zClass names to
@@ -115,7 +175,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
/* Start off with mappings for well-known names. */
localThread: {
refc: 3/*never reaches 0*/,
s: new TransientStorage,
s: new KVVfsStorage,
files: [/*KVVfsFile instances currently using this storage*/]
}
});
@@ -207,6 +267,30 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
return rc;
};
//#if nope
// fileForDb() works but we don't have a current need for it.
/**
Expects an (sqlite3*). Uses sqlite3_file_control() to extract its
(sqlite3_file*). On success it returns a new KVVfsFile instance
wrapping that pointer, which the caller must eventual call
dispose() on (which won't free the underlying pointer, just the
wrapper).
*/
const fileForDb = function(pDb){
const stack = pstack.pointer;
try{
const pOut = pstack.allocPtr();
return wasm.exports.sqlite3_file_control(
pDb, wasm.ptr.null, capi.SQLITE_FCNTL_FILE_POINTER, pOut
)
? null
: new KVVfsFile(wasm.peekPtr(pOut));
}finally{
pstack.restore(stack);
}
};
//#endif nope
/**
Clears all storage used by the kvvfs DB backend, deleting any
DB(s) stored there.
@@ -492,25 +576,18 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
if( rc ) return rc;
const f = new KVVfsFile(pProtoFile);
util.assert(f.$zClass, "Missing f.$zClass");
const jzClass = wasm.cstrToJs(zName);//f.$zClass);
const jzClass = wasm.cstrToJs(zName);
let s = cache.jzClassToStorage[jzClass];
//debug("xOpen", jzClass, s);
if( s ){
++s.refc;
s.files.push(f);
if( false && !s.keyPrefix ){
/* this messes up the recordHandler methods. They have only
the key, not the sqlite3_file object, so cannot map
a prefixless key to a storage object. */
f.$zClass = wasm.ptr.null;
}
}else{
/* TODO: a url flag which tells it to keep the storage
around forever so that future xOpen()s get the same
Storage-ish objects. We can accomplish that by
simply increasing the refcount once more. */
util.assert( !f.$isJournal, "Opening a journal before its db? "+jzClass );
//breaks stuff f.$zClass = wasm.ptr.null /* causes the "kvvfs-" prefix to be elided from keys */;
const other = f.$isJournal
? jzClass.replace(cache.rxJournalSuffix,'')
: jzClass + '-journal';
@@ -522,7 +599,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
will follow soon enough and bump the
refcount. If we start at 2 here, that
pending open will increment it again. */,
s: new TransientStorage,
s: new KVVfsStorage,
keyPrefix: '',
files: [f]
});
@@ -619,12 +696,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
// these impls work but there's currently no pressing need _not_ use
// the native impls.
xCurrentTime: function(pVfs,pOut){
wasm.poke64f(pOut, 2440587.5 + (new Date().getTime()/86400000));
wasm.poke64f(pOut, 2440587.5 + (Date.now()/86400000));
return 0;
},
xCurrentTimeInt64: function(pVfs,pOut){
wasm.poke64(pOut, (2440587.5 * 86400000) + new Date().getTime());
wasm.poke64(pOut, (2440587.5 * 86400000) + Date.now());
return 0;
}
//#endif
@@ -857,7 +934,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
const s = store.s;
const rc = Object.assign(Object.create(null),{
name: this.filename,
timestamp: (new Date()).valueOf(),
timestamp: Date.now(),
pages: []
});
const pages = Object.create(null);
@@ -904,16 +981,27 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
invoking this while the db is in active use invokes undefined
behavior.
Throws on error. Returns this object on success.
Returns this object on success. Throws on error. Error
conditions include:
FIXMEs:
- This db is closed.
- We need the page size in the export so that we can reset it,
if needed, on the import.
- Other handles to the same storage object are opened.
Performing this page-by-page import would invoke undefined
behavior on them.
- We need to ensure that the native-size KVVfsFile::szDb and
KVVfsFile::szPage get set to -1 for all open instances so that
they re-read the db size.
- A transaction is active.
Those are the error case it can easily cover. The room for
undefined behavior in wiping a db's storage out from under it
is a whole other potential minefield.
If it throws after starting the input then it clears the
storage before returning, to avoid leaving the db in an
undefined state. It has no inherent error conditions during the
input phase beyond out-of-memory but it may throw for any of
the above-listed conditions before reaching that step, in which
case the db is not modified.
*/
jdb.prototype.importFromObject = function(exp){
this.affirmOpen();
@@ -923,12 +1011,23 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
|| !Array.isArray(exp.pages) ){
util.toss3(capi.SQLITE_MISUSE, "Malformed export object.");
}
if( s.files.length>1 ){
util.toss3(capi.SQLITE_IOERR_ACCESS,
"Cannot import a db when multiple handles to it",
"are opened.");
}else if( capi.sqlite3_txn_state(this.pointer, null)>0 ){
util.toss3(capi.SQLITE_MISUSE,
"Cannot import the db while a transaction is active.");
}
warn("importFromObject() is incomplete");
this.clearStorage();
const store = kvvfsWhich(this.filename);
util.assert(store?.s, "Somehow missing a storage object for",this.filename);
util.assert(store?.s, "Somehow missing a storage object for", this.filename);
const keyPrefix = kvvfsKeyPrefix(this.filename);
this.clearStorage();
try{
s.files.forEach((f)=>f.$szPage = $f.$szDb = -1)
/* Force the native KVVfsFile instances to re-read the db
and page size. */;
s.setItem(keyPrefix+'sz', exp.size);
if( exp.journal ) s.setItem(keyPrefix+'jrnl', exp.journal);
exp.pages.forEach((v,ndx)=>s.setItem(keyPrefix+(ndx+1)));
manifest
+6 -6
View File
@@ -1,5 +1,5 @@
C Get\sthe\skvvfs\sv2\sworking\sin\s64-bit\swasm\sbuilds\s(a\s0\svs\snull\svs\sBigInt\sissue).
D 2025-11-23T16:54:16.202
C Docs,\scleanups,\sand\sdead\scode\sremoval.
D 2025-11-23T20:05:21.758
F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -600,7 +600,7 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667
F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966
F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d
F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 6886aa3a355ea7332190877fac6e5d569ee0c158cf467b45b0fc876e5d305d33
F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 1aaea0e80bbc992a365088b2f0c7ddb8fe9ec61d539fd75bc457db879e7a7eaa
F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 26cb41d5a62f46a106b6371eb00fef02de3cdbfaa51338ba087a45f53028e0d0
F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3
F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 9097074724172e31e56ce20ccd7482259cf72a76124213cbc9469d757676da86
@@ -2178,8 +2178,8 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd
F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P f63014f70febf82976a99f9f1ce6d793e2ca8d1dd2f72622152b64b2d65f8adc
R 9d104150c6613ed7753bd1fb4ac1b002
P 4811742688bd8ae847c0a13ca9b395b2d2edd24ce4e7760a5d1b0adf1c7f332d
R e3580b09ff0203465b33f3cc14e105b3
U stephan
Z cb3a60effd1de39bc1ff05e3b6dc575c
Z b07b7b7559951d4d97e0ab23e9e648e7
# Remove this line to create a well-formed Fossil manifest.
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
4811742688bd8ae847c0a13ca9b395b2d2edd24ce4e7760a5d1b0adf1c7f332d
49db59aa9c74e49d878adc8671b0d32db8f1f898bde29d046ce0e368d8987868