Cherrypick [ea370b9b05f7ed7eaa] and part of [92b500da] into kv-vfs branch.

FossilOrigin-Name: 812638039e1b8621a00dcf583ae3fb8a751f9aedbb18223b1fbffaee38cc0e15

Makefile.in

@@ -184,7 +184,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
          main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
          memdb.lo memjournal.lo \
          mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
-         notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
+         notify.lo opcodes.lo os.lo os_kv.lo os_unix.lo os_win.lo \
          pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
          random.lo resolve.lo rowset.lo rtree.lo \
          sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
@@ -257,6 +257,7 @@ SRC = \
   $(TOP)/src/os.h \
   $(TOP)/src/os_common.h \
   $(TOP)/src/os_setup.h \
+  $(TOP)/src/os_kv.c \
   $(TOP)/src/os_unix.c \
   $(TOP)/src/os_win.c \
   $(TOP)/src/os_win.h \
@@ -492,6 +493,7 @@ TESTSRC2 = \
   $(TOP)/src/main.c \
   $(TOP)/src/mem5.c \
   $(TOP)/src/os.c \
+  $(TOP)/src/os_kv.c \
   $(TOP)/src/os_unix.c \
   $(TOP)/src/os_win.c \
   $(TOP)/src/pager.c \
@@ -633,6 +635,7 @@ FUZZCHECK_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 FUZZCHECK_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
 FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c $(TOP)/test/fuzzinvariants.c
 DBFUZZ_OPT =
+ST_OPT = -DSQLITE_OS_KV_OPTIONAL
 
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
@@ -938,6 +941,9 @@ pcache1.lo:	$(TOP)/src/pcache1.c $(HDR) $(TOP)/src/pcache.h
 os.lo:	$(TOP)/src/os.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os.c
 
+os_kv.lo:	$(TOP)/src/os_kv.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os_kv.c
+
 os_unix.lo:	$(TOP)/src/os_unix.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os_unix.c
 
@@ -1387,7 +1393,7 @@ LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
 wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS)
 
-speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.c
+speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.c Makefile
 	$(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS)
 
 startup$(TEXE):	$(TOP)/test/startup.c sqlite3.c

ext/wasm/GNUmakefile

@@ -262,6 +262,7 @@ all: $(sqlite3.js)
 # End main Emscripten-based module build
 ########################################################################
 
+include kvvfs.make
 
 ########################################################################
 # fiddle_remote is the remote destination for the fiddle app. It

ext/wasm/kvvfs.make

@@ -0,0 +1,82 @@
+#!/usr/bin/make
+#^^^^ help emacs select makefile mode
+#
+# This is a sub-make for building a standalone kvvfs-based
+# sqlite3.wasm.  It is intended to be "include"d from the main
+# GNUMakefile.
+#
+# Notable potential TODOs:
+#
+# - Trim down a custom sqlite3-api.js for this build. We can elimate
+#   the jaccwabyt dependency, for example, because this build won't
+#   make use of the VFS bits. Similarly, we can eliminate or replace
+#   parts of the OO1 API, or provide a related API which manages
+#   singletons of the localStorage/sessionStorage instances.
+#
+########################################################################
+MAKEFILE.kvvfs := $(lastword $(MAKEFILE_LIST))
+
+kvvfs.js     := sqlite3-kvvfs.js
+kvvfs.wasm   := sqlite3-kvvfs.wasm
+kvvfs.wasm.c := $(dir.api)/sqlite3-wasm.c
+
+CLEAN_FILES += $(kvvfs.js) $(kvvfs.wasm)
+
+########################################################################
+# emcc flags for .c/.o/.wasm.
+kvvfs.flags =
+#kvvfs.flags += -v # _very_ loud but also informative about what it's doing
+
+########################################################################
+# emcc flags for .c/.o.
+kvvfs.cflags :=
+kvvfs.cflags += -std=c99 -fPIC
+kvvfs.cflags += -I. -I$(dir.top)
+kvvfs.cflags += -DSQLITE_OS_KV=1 $(SQLITE_OPT)
+
+########################################################################
+# emcc flags specific to building the final .js/.wasm file...
+kvvfs.jsflags := -fPIC
+kvvfs.jsflags += --no-entry
+kvvfs.jsflags += -sENVIRONMENT=web
+kvvfs.jsflags += -sMODULARIZE
+kvvfs.jsflags += -sSTRICT_JS
+kvvfs.jsflags += -sDYNAMIC_EXECUTION=0
+kvvfs.jsflags += -sNO_POLYFILL
+kvvfs.jsflags += -sEXPORTED_FUNCTIONS=@$(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api
+kvvfs.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory,allocateUTF8OnStack
+                                            # wasmMemory ==> for -sIMPORTED_MEMORY
+                                            # allocateUTF8OnStack ==> kvvfs internals
+kvvfs.jsflags += -sUSE_CLOSURE_COMPILER=0
+kvvfs.jsflags += -sIMPORTED_MEMORY
+#kvvfs.jsflags += -sINITIAL_MEMORY=13107200
+#kvvfs.jsflags += -sTOTAL_STACK=4194304
+kvvfs.jsflags += -sEXPORT_NAME=sqlite3InitModule
+kvvfs.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr.
+kvvfs.jsflags += --post-js=$(post-js.js)
+#kvvfs.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API
+#                                Perhaps the kvvfs build doesn't?
+#kvvfs.jsflags += -sABORTING_MALLOC
+kvvfs.jsflags += -sALLOW_MEMORY_GROWTH
+kvvfs.jsflags += -sALLOW_TABLE_GROWTH
+kvvfs.jsflags += -Wno-limited-postlink-optimizations
+# ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag.
+kvvfs.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=0
+kvvfs.jsflags += -sLLD_REPORT_UNDEFINED
+#kvvfs.jsflags += --import-undefined
+kvvfs.jsflags += -sMEMORY64=0
+ifneq (0,$(enable_bigint))
+kvvfs.jsflags += -sWASM_BIGINT
+endif
+
+$(kvvfs.js): $(MAKEFILE) $(MAKEFILE.kvvfs) $(kvvfs.wasm.c) \
+    EXPORTED_FUNCTIONS.api \
+    $(post-js.js)
+	$(emcc.bin) -o $@ $(emcc_opt) $(emcc.flags) $(kvvfs.cflags) $(kvvfs.jsflags) $(kvvfs.wasm.c)
+	chmod -x $(kvvfs.wasm)
+ifneq (,$(wasm-strip))
+	$(wasm-strip) $(kvvfs.wasm)
+endif
+	@ls -la $@ $(kvvfs.wasm)
+
+kvvfs: $(kvvfs.js)

ext/wasm/kvvfs1.html

@@ -0,0 +1,34 @@
+<!doctype html>
+<html lang="en-us">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon">
+    <link rel="stylesheet" href="common/emscripten.css"/>
+    <link rel="stylesheet" href="common/testing.css"/>
+    <title>sqlite3-kvvfs.js tests</title>
+  </head>
+  <body>
+    <header id='titlebar'><span>sqlite3-kvvfs.js tests</span></header>
+    <!-- emscripten bits -->
+    <figure id="module-spinner">
+      <div class="spinner"></div>
+      <div class='center'><strong>Initializing app...</strong></div>
+      <div class='center'>
+        On a slow internet connection this may take a moment.  If this
+        message displays for "a long time", intialization may have
+        failed and the JavaScript console may contain clues as to why.
+      </div>
+    </figure>
+    <div class="emscripten" id="module-status">Downloading...</div>
+    <div class="emscripten">
+      <progress value="0" max="100" id="module-progress" hidden='1'></progress>  
+    </div><!-- /emscripten bits -->
+    <div>Everything on this page happens in the dev console.</div>
+    <hr>
+    <div id='test-output'></div>
+    <script src="sqlite3-kvvfs.js"></script>
+    <script src="common/SqliteTestUtil.js"></script>
+    <script src="kvvfs1.js"></script>
+  </body>
+</html>

ext/wasm/kvvfs1.js

@@ -0,0 +1,84 @@
+/*
+  2022-09-12
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  A basic test script for sqlite3-kvvfs.wasm. This file must be run in
+  main JS thread and sqlite3-kvvfs.js must have been loaded before it.
+*/
+'use strict';
+(function(){
+  const T = self.SqliteTestUtil;
+  const toss = function(...args){throw new Error(args.join(' '))};
+  const debug = console.debug.bind(console);
+  const eOutput = document.querySelector('#test-output');
+  const log = console.log.bind(console)
+  const logHtml = function(...args){
+    log.apply(this, args);
+    const ln = document.createElement('div');
+    ln.append(document.createTextNode(args.join(' ')));
+    eOutput.append(ln);
+  };
+
+  const runTests = function(Module){
+    //log("Module",Module);
+    const sqlite3 = Module.sqlite3,
+          capi = sqlite3.capi,
+          oo = sqlite3.oo1,
+          wasm = capi.wasm;
+    log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid());
+    log("Build options:",wasm.compileOptionUsed());
+    self.S = sqlite3;
+    T.assert(0 === capi.sqlite3_vfs_find(null));
+    S.capi.sqlite3_initialize();
+    T.assert( Number.isFinite( capi.sqlite3_vfs_find(null) ) );
+    const stores = {
+      local: localStorage,
+      session: sessionStorage
+    };
+    const cleanupStore = function(n){
+      const s = stores[n];
+      const isKv = (key)=>key.startsWith('kvvfs-'+n);
+      let i, k, toRemove = [];
+      for( i = 0; (k = s.key(i)); ++i) {
+        if(isKv(k)) toRemove.push(k);
+      }
+      toRemove.forEach((k)=>s.removeItem(k));
+    };
+    const dbStorage = 1 ? 'session' : 'local';
+    const db = new oo.DB(dbStorage);
+    try {
+      db.exec("create table if not exists t(a)");
+      if(undefined===db.selectValue("select a from t limit 1")){
+        log("New db. Populating..");
+        db.exec("insert into t(a) values(1),(2),(3)");
+      }else{
+        log("Found existing table data:");
+        db.exec({
+          sql: "select * from t order by a",
+          rowMode: 0,
+          callback: function(v){log(v)}
+        });
+      }
+    }finally{
+      const n = db.filename;
+      db.close();
+      //cleanupStore(n);
+    }
+    
+    log("Init done. Proceed from the dev console.");
+  };
+
+  sqlite3InitModule(self.sqlite3TestModule).then(function(theModule){
+    console.warn("Installing Emscripten module as global EM for dev console access.");
+    self.EM = theModule;
+    runTests(theModule);
+  });
+})();

main.mk

@@ -68,7 +68,7 @@ LIBOBJ+= vdbe.o parse.o \
          main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \
          memdb.o memjournal.o \
          mutex.o mutex_noop.o mutex_unix.o mutex_w32.o \
-         notify.o opcodes.o os.o os_unix.o os_win.o \
+         notify.o opcodes.o os.o os_kv.o os_unix.o os_win.o \
          pager.o pcache.o pcache1.o pragma.o prepare.o printf.o \
          random.o resolve.o rowset.o rtree.o \
          select.o sqlite3rbu.o status.o stmt.o \
@@ -134,6 +134,7 @@ SRC = \
   $(TOP)/src/os.h \
   $(TOP)/src/os_common.h \
   $(TOP)/src/os_setup.h \
+  $(TOP)/src/os_kv.c \
   $(TOP)/src/os_unix.c \
   $(TOP)/src/os_win.c \
   $(TOP)/src/os_win.h \
@@ -411,6 +412,7 @@ TESTSRC2 = \
   $(TOP)/src/main.c \
   $(TOP)/src/mem5.c \
   $(TOP)/src/os.c \
+  $(TOP)/src/os_kv.c \
   $(TOP)/src/os_unix.c \
   $(TOP)/src/os_win.c \
   $(TOP)/src/pager.c \

manifest

@@ -1,9 +1,9 @@
-C Copy\sall\sof\sthe\sfiddle-opfs\schanges\sto\sspeedtest1\sinto\strunk.
-D 2022-10-03T13:56:52.781
+C Cherrypick\s[ea370b9b05f7ed7eaa]\sand\spart\sof\s[92b500da]\sinto\skv-vfs\sbranch.
+D 2022-10-09T13:33:34.070
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
-F Makefile.in 525bccb89e36a927f9312a231f054a2dc029f6af75901c7fc1a781d51b260323
+F Makefile.in c80d05a84c16fce844db1dafdfcd2cdd116177027202e09c4a54681ecd72c6ec
 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241
 F Makefile.msc d547a2fdba38a1c6cd1954977d0b0cc017f5f8fbfbc65287bf8d335808938016
 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e
@@ -474,7 +474,7 @@ F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb
 F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3
 F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02
-F ext/wasm/GNUmakefile 5359a37fc13b68fad2259228590450339a0c59687744edd0db7bb93d3b1ae2b1
+F ext/wasm/GNUmakefile 12a672ab9125dc860457c2853f7651b98517e424d7a0e9714c89b28c5ff73800
 F ext/wasm/README.md 4b00ae7c7d93c4591251245f0996a319e2651361013c98d2efb0b026771b7331
 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33
 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287
@@ -502,6 +502,9 @@ F ext/wasm/jaccwabyt/jaccwabyt.js 99b424b4d467d4544e82615b58e2fe07532a898540bf9d
 F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356afacbdbf312b2588106
 F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f
 F ext/wasm/jaccwabyt/jaccwabyt_test.exports 5ff001ef975c426ffe88d7d8a6e96ec725e568d2c2307c416902059339c06f19
+F ext/wasm/kvvfs.make 7cc9cf10e744c3ba523c3eaf5c4af47028f3a5bb76db304ea8044a9b2a9d496f
+F ext/wasm/kvvfs1.html 2acb241a6110a4ec581adbf07a23d5fc2ef9c7142aa9d60856732a102abc5016
+F ext/wasm/kvvfs1.js 46afaf4faba041bf938355627bc529854295e561f49db3a240c914e75a529338
 F ext/wasm/testing1.html 0bf3ff224628c1f1e3ed22a2dc1837c6c73722ad8c0ad9c8e6fb9e6047667231
 F ext/wasm/testing1.js cba7134901a965743fa9289d82447ab71de4690b1ee5d06f6cb83e8b569d7943
 F ext/wasm/testing2.html 73e5048e666fd6fb28b6e635677a9810e1e139c599ddcf28d687c982134b92b8
@@ -509,7 +512,7 @@ F ext/wasm/testing2.js d37433c601f88ed275712c1cfc92d3fb36c7c22e1ed8c7396fb2359e4
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 20801eed419dc58936ff9449b04041edbbbc0488a9fc683e72471dded050e0bb
+F main.mk b1b02927b2e0b73d025f93e388f2f7fba07614e972453983bddc65cd2bf33bb1
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -552,7 +555,7 @@ F src/insert.c aea5361767817f917b0f0f647a1f0b1621bd858938ae6ae545c3b6b9814b798f
 F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 853385cc7a604157e137585097949252d5d0c731768e16b044608e5c95c3614b
-F src/main.c 8983b4a316d7e09946dd731913aa41712f02e2b55cb5c6c92126ccfe2473244a
+F src/main.c e11267cdd380be68d95d4292666636a7f1dff5f17a395f3d55be910d7e9350fb
 F src/malloc.c dfddca1e163496c0a10250cedeafaf56dff47673e0f15888fb0925340a8e3f90
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -571,8 +574,9 @@ F src/notify.c 89a97dc854c3aa62ad5f384ef50c5a4a11d70fcc69f86de3e991573421130ed6
 F src/os.c 0eb831ba3575af5277e47f4edd14fdfc90025c67eb25ce5cda634518d308d4e9
 F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
-F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 102f7e5c5b59c18ea3dbc929dc3be8acb3afc0e0b6ad572e032335c9c27f44f1
+F src/os_kv.c fe6d52ab2bcb00b855a4237f0320b6f1789bfb1ac805bd5b52559451e8017041
+F src/os_setup.h 0711dbc4678f3ac52d7fe736951b6384a0615387c4ba5135a4764e4e31f4b6a6
+F src/os_unix.c 287aa5f5691a2b356780c63e83abaa33549add84227b8313395f04088486d79c
 F src/os_win.c 8d129ae3e59e0fa900e20d0ad789e96f2e08177f0b00b53cdda65c40331e0902
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 6176d9752eb580419e8fef4592dc417a6b00ddfd43ee22f818819bf8840ceee8
@@ -593,7 +597,7 @@ F src/shell.c.in e7e7c2c69ae86c5ee9e8ad66227203d46ff6dce8700a1b1dababff01c71d33d
 F src/sqlite.h.in 3f4695457fcb41ea8a44c6d09fc2a591bbeda44b13bda86618f3ce0071bbca7c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d
-F src/sqliteInt.h 7571703e6d11cd0e0283be68e9b6baceefedf10e3bbac5869d12183e031c5254
+F src/sqliteInt.h 13e1876c4694958683eac18974ebfc71ac0793f643c0956fb0ff6b8a15337dc0
 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -1937,7 +1941,7 @@ F tool/mkshellc.tcl df5d249617f9cc94d5c48eb0401673eb3f31f383ecbc54e8a13ca3dd97e8
 F tool/mksourceid.c 36aa8020014aed0836fd13c51d6dc9219b0df1761d6b5f58ff5b616211b079b9
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl 4f7cfef5152b0c91920355cbfc1d608a4ad242cb819f1aea07f6d0274f584a7f
-F tool/mksqlite3c.tcl eee7e9d9c58abb1045f6ed74ad95ad26e8d22be29fdc431deea5267fb3fa049c
+F tool/mksqlite3c.tcl 4fc26a9bfa0c4505b203d7ca0cf086e75ebcd4d63eb719c82f37e3fecdf23d37
 F tool/mksqlite3h.tcl 1f5e4a1dbbbc43c83cc6e74fe32c6c620502240b66c7c0f33a51378e78fc4edf
 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b
 F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5
@@ -2000,8 +2004,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P aeb884029ac1c68f0d7361dda0a545cff19ed265b608c1e329a1926397c5c8a3
-R 9ef7b22777f981a0530dc8b380258dc2
-U drh
-Z 4bdd14d37dfc0c4e0295fe6f5fd60fb3
+P e9411c74fcce5e109fd02e7db55c0bf113ae8e02f46dfccc0b950dd06e5bd39b
+Q +92b500da70a3dc64e910c232a2cac7620b6609162c2a5058b269d3b83d763c02
+Q +ea370b9b05f7ed7eaa154ba58019f6642217eabc18517e721567adf948d93980
+R cb72d36beff7f7f7a1bade2b759b933e
+U stephan
+Z 6daf03a792b7f21d2196b1f5d1530641
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-f3d31c9982cab517a3907a28c571225cee614ee78375c70eb2efb9a2b6b4b042
+812638039e1b8621a00dcf583ae3fb8a751f9aedbb18223b1fbffaee38cc0e15

src/main.c

@@ -4781,8 +4781,8 @@ int sqlite3_snapshot_open(
 */
 int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb){
   int rc = SQLITE_ERROR;
-  int iDb;
 #ifndef SQLITE_OMIT_WAL
+  int iDb;
 
 #ifdef SQLITE_ENABLE_API_ARMOR
   if( !sqlite3SafetyCheckOk(db) ){

src/os_kv.c

@@ -0,0 +1,972 @@
+/*
+** 2022-09-06
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains an experimental VFS layer that operates on a
+** Key/Value storage engine where both keys and values must be pure
+** text.
+*/
+#include <sqliteInt.h>
+#if SQLITE_OS_KV || (SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL))
+
+/*****************************************************************************
+** Debugging logic
+*/
+
+/* SQLITE_KV_TRACE() is used for tracing calls to kvstorage routines. */
+#if 0
+#define SQLITE_KV_TRACE(X)  printf X
+#else
+#define SQLITE_KV_TRACE(X)
+#endif
+
+/* SQLITE_KV_LOG() is used for tracing calls to the VFS interface */
+#if 0
+#define SQLITE_KV_LOG(X)  printf X
+#else
+#define SQLITE_KV_LOG(X)
+#endif
+
+
+/*
+** Forward declaration of objects used by this VFS implementation
+*/
+typedef struct KVVfsFile KVVfsFile;
+
+/* A single open file.  There are only two files represented by this
+** VFS - the database and the rollback journal.
+*/
+struct KVVfsFile {
+  sqlite3_file base;              /* IO methods */
+  const char *zClass;             /* Storage class */
+  int isJournal;                  /* True if this is a journal file */
+  unsigned int nJrnl;             /* Space allocated for aJrnl[] */
+  char *aJrnl;                    /* Journal content */
+  int szPage;                     /* Last known page size */
+  sqlite3_int64 szDb;             /* Database file size.  -1 means unknown */
+};
+
+/*
+** Methods for KVVfsFile
+*/
+static int kvvfsClose(sqlite3_file*);
+static int kvvfsReadDb(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
+static int kvvfsReadJrnl(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
+static int kvvfsWriteDb(sqlite3_file*,const void*,int iAmt, sqlite3_int64);
+static int kvvfsWriteJrnl(sqlite3_file*,const void*,int iAmt, sqlite3_int64);
+static int kvvfsTruncateDb(sqlite3_file*, sqlite3_int64 size);
+static int kvvfsTruncateJrnl(sqlite3_file*, sqlite3_int64 size);
+static int kvvfsSyncDb(sqlite3_file*, int flags);
+static int kvvfsSyncJrnl(sqlite3_file*, int flags);
+static int kvvfsFileSizeDb(sqlite3_file*, sqlite3_int64 *pSize);
+static int kvvfsFileSizeJrnl(sqlite3_file*, sqlite3_int64 *pSize);
+static int kvvfsLock(sqlite3_file*, int);
+static int kvvfsUnlock(sqlite3_file*, int);
+static int kvvfsCheckReservedLock(sqlite3_file*, int *pResOut);
+static int kvvfsFileControlDb(sqlite3_file*, int op, void *pArg);
+static int kvvfsFileControlJrnl(sqlite3_file*, int op, void *pArg);
+static int kvvfsSectorSize(sqlite3_file*);
+static int kvvfsDeviceCharacteristics(sqlite3_file*);
+
+/*
+** Methods for sqlite3_vfs
+*/
+static int kvvfsOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
+static int kvvfsDelete(sqlite3_vfs*, const char *zName, int syncDir);
+static int kvvfsAccess(sqlite3_vfs*, const char *zName, int flags, int *);
+static int kvvfsFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut);
+static void *kvvfsDlOpen(sqlite3_vfs*, const char *zFilename);
+static int kvvfsRandomness(sqlite3_vfs*, int nByte, char *zOut);
+static int kvvfsSleep(sqlite3_vfs*, int microseconds);
+static int kvvfsCurrentTime(sqlite3_vfs*, double*);
+static int kvvfsCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
+
+static sqlite3_vfs sqlite3OsKvvfsObject = {
+  1,                              /* iVersion */
+  sizeof(KVVfsFile),              /* szOsFile */
+  1024,                           /* mxPathname */
+  0,                              /* pNext */
+  "kvvfs",                        /* zName */
+  0,                              /* pAppData */
+  kvvfsOpen,                      /* xOpen */
+  kvvfsDelete,                    /* xDelete */
+  kvvfsAccess,                    /* xAccess */
+  kvvfsFullPathname,              /* xFullPathname */
+  kvvfsDlOpen,                    /* xDlOpen */
+  0,                              /* xDlError */
+  0,                              /* xDlSym */
+  0,                              /* xDlClose */
+  kvvfsRandomness,                /* xRandomness */
+  kvvfsSleep,                     /* xSleep */
+  kvvfsCurrentTime,               /* xCurrentTime */
+  0,                              /* xGetLastError */
+  kvvfsCurrentTimeInt64           /* xCurrentTimeInt64 */
+};
+
+/* Methods for sqlite3_file objects referencing a database file
+*/
+static sqlite3_io_methods kvvfs_db_io_methods = {
+  1,                              /* iVersion */
+  kvvfsClose,                     /* xClose */
+  kvvfsReadDb,                    /* xRead */
+  kvvfsWriteDb,                   /* xWrite */
+  kvvfsTruncateDb,                /* xTruncate */
+  kvvfsSyncDb,                    /* xSync */
+  kvvfsFileSizeDb,                /* xFileSize */
+  kvvfsLock,                      /* xLock */
+  kvvfsUnlock,                    /* xUnlock */
+  kvvfsCheckReservedLock,         /* xCheckReservedLock */
+  kvvfsFileControlDb,             /* xFileControl */
+  kvvfsSectorSize,                /* xSectorSize */
+  kvvfsDeviceCharacteristics,     /* xDeviceCharacteristics */
+  0,                              /* xShmMap */
+  0,                              /* xShmLock */
+  0,                              /* xShmBarrier */
+  0,                              /* xShmUnmap */
+  0,                              /* xFetch */
+  0                               /* xUnfetch */
+};
+
+/* Methods for sqlite3_file objects referencing a rollback journal
+*/
+static sqlite3_io_methods kvvfs_jrnl_io_methods = {
+  1,                              /* iVersion */
+  kvvfsClose,                     /* xClose */
+  kvvfsReadJrnl,                  /* xRead */
+  kvvfsWriteJrnl,                 /* xWrite */
+  kvvfsTruncateJrnl,              /* xTruncate */
+  kvvfsSyncJrnl,                  /* xSync */
+  kvvfsFileSizeJrnl,              /* xFileSize */
+  kvvfsLock,                      /* xLock */
+  kvvfsUnlock,                    /* xUnlock */
+  kvvfsCheckReservedLock,         /* xCheckReservedLock */
+  kvvfsFileControlJrnl,           /* xFileControl */
+  kvvfsSectorSize,                /* xSectorSize */
+  kvvfsDeviceCharacteristics,     /* xDeviceCharacteristics */
+  0,                              /* xShmMap */
+  0,                              /* xShmLock */
+  0,                              /* xShmBarrier */
+  0,                              /* xShmUnmap */
+  0,                              /* xFetch */
+  0                               /* xUnfetch */
+};
+
+/****** Storage subsystem **************************************************/
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+/* Forward declarations for the low-level storage engine
+*/
+static int kvstorageWrite(const char*, const char *zKey, const char *zData);
+static int kvstorageDelete(const char*, const char *zKey);
+static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf);
+#define KVSTORAGE_KEY_SZ  32
+
+/* Expand the key name with an appropriate prefix and put the result
+** zKeyOut[].  The zKeyOut[] buffer is assumed to hold at least
+** KVSTORAGE_KEY_SZ bytes.
+*/
+static void kvstorageMakeKey(
+  const char *zClass,
+  const char *zKeyIn,
+  char *zKeyOut
+){
+  sqlite3_snprintf(KVSTORAGE_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn);
+}
+
+/* Write content into a key.  zClass is the particular namespace of the
+** underlying key/value store to use - either "local" or "session".
+**
+** Both zKey and zData are zero-terminated pure text strings.
+**
+** Return the number of errors.
+*/
+static int kvstorageWrite(
+  const char *zClass,
+  const char *zKey,
+  const char *zData
+){
+  FILE *fd;
+  char zXKey[KVSTORAGE_KEY_SZ];
+  kvstorageMakeKey(zClass, zKey, zXKey);
+  fd = fopen(zXKey, "wb");
+  if( fd ){
+    SQLITE_KV_TRACE(("KVVFS-WRITE  %-15s (%d) %.50s%s\n", zXKey,
+                 (int)strlen(zData), zData,
+                 strlen(zData)>50 ? "..." : ""));
+    fputs(zData, fd);
+    fclose(fd);
+    return 0;
+  }else{
+    return 1;
+  }
+}
+
+/* Delete a key (with its corresponding data) from the key/value
+** namespace given by zClass.  If the key does not previously exist,
+** this routine is a no-op.
+*/
+static int kvstorageDelete(const char *zClass, const char *zKey){
+  char zXKey[KVSTORAGE_KEY_SZ];
+  kvstorageMakeKey(zClass, zKey, zXKey);
+  unlink(zXKey);
+  SQLITE_KV_TRACE(("KVVFS-DELETE %-15s\n", zXKey));
+  return 0;
+}
+
+/* Read the value associated with a zKey from the key/value namespace given
+** by zClass and put the text data associated with that key in the first
+** nBuf bytes of zBuf[].  The value might be truncated if zBuf is not large
+** enough to hold it all.  The value put into zBuf must always be zero
+** terminated, even if it gets truncated because nBuf is not large enough.
+**
+** Return the total number of bytes in the data, without truncation, and
+** not counting the final zero terminator.   Return -1 if the key does
+** not exist.
+**
+** If nBuf<=0 then this routine simply returns the size of the data without
+** actually reading it.
+*/
+static int kvstorageRead(
+  const char *zClass,
+  const char *zKey,
+  char *zBuf,
+  int nBuf
+){
+  FILE *fd;
+  struct stat buf;
+  char zXKey[KVSTORAGE_KEY_SZ];
+  kvstorageMakeKey(zClass, zKey, zXKey);
+  if( access(zXKey, R_OK)!=0
+   || stat(zXKey, &buf)!=0
+   || !S_ISREG(buf.st_mode)
+  ){
+    SQLITE_KV_TRACE(("KVVFS-READ   %-15s (-1)\n", zXKey));
+    return -1;
+  }
+  if( nBuf<=0 ){
+    return (int)buf.st_size;
+  }else if( nBuf==1 ){
+    zBuf[0] = 0;
+    SQLITE_KV_TRACE(("KVVFS-READ   %-15s (%d)\n", zXKey,
+                 (int)buf.st_size));
+    return (int)buf.st_size;
+  }
+  if( nBuf > buf.st_size + 1 ){
+    nBuf = buf.st_size + 1;
+  }
+  fd = fopen(zXKey, "rb");
+  if( fd==0 ){
+    SQLITE_KV_TRACE(("KVVFS-READ   %-15s (-1)\n", zXKey));
+    return -1;
+  }else{
+    sqlite3_int64 n = fread(zBuf, 1, nBuf-1, fd);
+    fclose(fd);
+    zBuf[n] = 0;
+    SQLITE_KV_TRACE(("KVVFS-READ   %-15s (%lld) %.50s%s\n", zXKey,
+                 n, zBuf, n>50 ? "..." : ""));
+    return (int)n;
+  }
+}
+
+/*
+** An internal level of indirection which enables us to replace the
+** kvvfs i/o methods with JavaScript implementations in WASM builds.
+** Maintenance reminder: if this struct changes in any way, the JSON
+** rendering of its structure must be updated in
+** sqlite3_wasm_enum_json(). There are no binary compatibility
+** concerns, so it does not need an iVersion member. This file is
+** necessarily always compiled together with sqlite3_wasm_enum_json(),
+** and JS code dynamically creates the mapping of members based on
+** that JSON description.
+*/
+typedef struct sqlite3_kvvfs_methods sqlite3_kvvfs_methods;
+struct sqlite3_kvvfs_methods {
+  int (*xRead)(const char *zClass, const char *zKey, char *zBuf, int nBuf);
+  int (*xWrite)(const char *zClass, const char *zKey, const char *zData);
+  int (*xDelete)(const char *zClass, const char *zKey);
+  const int nKeySize;
+};
+
+/*
+** This object holds the kvvfs I/O methods which may be swapped out
+** for JavaScript-side implementations in WASM builds. In such builds
+** it cannot be const, but in native builds it should be so that
+** the compiler can hopefully optimize this level of indirection out.
+** That said, kvvfs is intended primarily for use in WASM builds.
+**
+** Note that this is not explicitly flagged as static because the
+** amalgamation build will tag it with SQLITE_PRIVATE.
+*/
+#ifndef SQLITE_WASM
+const
+#endif
+sqlite3_kvvfs_methods sqlite3KvvfsMethods = {
+kvstorageRead,
+kvstorageWrite,
+kvstorageDelete,
+KVSTORAGE_KEY_SZ
+};
+
+/****** Utility subroutines ************************************************/
+
+/*
+** Encode binary into the text encoded used to persist on disk.
+** The output text is stored in aOut[], which must be at least
+** nData+1 bytes in length.
+**
+** Return the actual length of the encoded text, not counting the
+** zero terminator at the end.
+**
+** Encoding format
+** ---------------
+**
+**   *  Non-zero bytes are encoded as upper-case hexadecimal
+**
+**   *  A sequence of one or more zero-bytes that are not at the
+**      beginning of the buffer are encoded as a little-endian
+**      base-26 number using a..z.  "a" means 0.  "b" means 1,
+**      "z" means 25.  "ab" means 26.  "ac" means 52.  And so forth.
+**
+**   *  Because there is no overlap between the encoding characters
+**      of hexadecimal and base-26 numbers, it is always clear where
+**      one stops and the next begins.
+*/
+static int kvvfsEncode(const char *aData, int nData, char *aOut){
+  int i, j;
+  const unsigned char *a = (const unsigned char*)aData;
+  for(i=j=0; i<nData; i++){
+    unsigned char c = a[i];
+    if( c!=0 ){
+      aOut[j++] = "0123456789ABCDEF"[c>>4];
+      aOut[j++] = "0123456789ABCDEF"[c&0xf];
+    }else{
+      /* A sequence of 1 or more zeros is stored as a little-endian
+      ** base-26 number using a..z as the digits. So one zero is "b".
+      ** Two zeros is "c". 25 zeros is "z", 26 zeros is "ab", 27 is "bb",
+      ** and so forth.
+      */
+      int k;
+      for(k=1; i+k<nData && a[i+k]==0; k++){}
+      i += k-1;
+      while( k>0 ){
+        aOut[j++] = 'a'+(k%26);
+        k /= 26;
+      }
+    }
+  }
+  aOut[j] = 0;
+  return j;
+}
+
+static const signed char kvvfsHexValue[256] = {
+  -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
+   0,  1,  2,  3,  4,  5,  6,  7,    8,  9, -1, -1, -1, -1, -1, -1,
+  -1, 10, 11, 12, 13, 14, 15, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
+
+  -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1
+};
+
+/*
+** Decode the text encoding back to binary.  The binary content is
+** written into pOut, which must be at least nOut bytes in length.
+**
+** The return value is the number of bytes actually written into aOut[].
+*/
+static int kvvfsDecode(const char *a, char *aOut, int nOut){
+  int i, j;
+  int c;
+  const unsigned char *aIn = (const unsigned char*)a;
+  i = 0;
+  j = 0;
+  while( 1 ){
+    c = kvvfsHexValue[aIn[i]];
+    if( c<0 ){
+      int n = 0;
+      int mult = 1;
+      c = aIn[i];
+      if( c==0 ) break;
+      while( c>='a' && c<='z' ){
+        n += (c - 'a')*mult;
+        mult *= 26;
+        c = aIn[++i];
+      }
+      if( j+n>nOut ) return -1;
+      memset(&aOut[j], 0, n);
+      j += n;
+      c = aIn[i];
+      if( c==0 ) break;
+    }else{
+      aOut[j] = c<<4;
+      c = kvvfsHexValue[aIn[++i]];
+      if( c<0 ) break;
+      aOut[j++] += c;
+      i++;
+    }
+  }
+  return j;
+}
+
+/*
+** Decode a complete journal file.  Allocate space in pFile->aJrnl
+** and store the decoding there.  Or leave pFile->aJrnl set to NULL
+** if an error is encountered.
+**
+** The first few characters of the text encoding will be a little-endian
+** base-26 number (digits a..z) that is the total number of bytes
+** in the decoded journal file image.  This base-26 number is followed
+** by a single space, then the encoding of the journal.  The space
+** separator is required to act as a terminator for the base-26 number.
+*/
+static void kvvfsDecodeJournal(
+  KVVfsFile *pFile,      /* Store decoding in pFile->aJrnl */
+  const char *zTxt,      /* Text encoding.  Zero-terminated */
+  int nTxt               /* Bytes in zTxt, excluding zero terminator */
+){
+  unsigned int n = 0;
+  int c, i, mult;
+  i = 0;
+  mult = 1;
+  while( (c = zTxt[i++])>='a' && c<='z' ){
+    n += (zTxt[i] - 'a')*mult;
+    mult *= 26;
+  }
+  sqlite3_free(pFile->aJrnl);
+  pFile->aJrnl = sqlite3_malloc64( n );
+  if( pFile->aJrnl==0 ){
+    pFile->nJrnl = 0;
+    return;
+  }
+  pFile->nJrnl = n;
+  n = kvvfsDecode(zTxt+i, pFile->aJrnl, pFile->nJrnl);
+  if( n<pFile->nJrnl ){
+    sqlite3_free(pFile->aJrnl);
+    pFile->aJrnl = 0;
+    pFile->nJrnl = 0;
+  }
+}
+
+/*
+** Read or write the "sz" element, containing the database file size.
+*/
+static sqlite3_int64 kvvfsReadFileSize(KVVfsFile *pFile){
+  char zData[50];
+  zData[0] = 0;
+  sqlite3KvvfsMethods.xRead(pFile->zClass, "sz", zData, sizeof(zData)-1);
+  return strtoll(zData, 0, 0);
+}
+static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){
+  char zData[50];
+  sqlite3_snprintf(sizeof(zData), zData, "%lld", sz);
+  return sqlite3KvvfsMethods.xWrite(pFile->zClass, "sz", zData);
+}
+
+/****** sqlite3_io_methods methods ******************************************/
+
+/*
+** Close an kvvfs-file.
+*/
+static int kvvfsClose(sqlite3_file *pProtoFile){
+  KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+
+  SQLITE_KV_LOG(("xClose %s %s\n", pFile->zClass, 
+             pFile->isJournal ? "journal" : "db"));
+  sqlite3_free(pFile->aJrnl);
+  return SQLITE_OK;
+}
+
+/*
+** Read from the -journal file.
+*/
+static int kvvfsReadJrnl(
+  sqlite3_file *pProtoFile,
+  void *zBuf, 
+  int iAmt, 
+  sqlite_int64 iOfst
+){
+  KVVfsFile *pFile = (KVVfsFile*)pProtoFile;
+  assert( pFile->isJournal );
+  SQLITE_KV_LOG(("xRead('%s-journal',%d,%lld)\n", pFile->zClass, iAmt, iOfst));
+  if( pFile->aJrnl==0 ){
+    int szTxt = kvstorageRead(pFile->zClass, "jrnl", 0, 0);
+    char *aTxt;
+    if( szTxt<=4 ){
+      return SQLITE_IOERR;
+    }
+    aTxt = sqlite3_malloc64( szTxt+1 );
+    if( aTxt==0 ) return SQLITE_NOMEM;
+    kvstorageRead(pFile->zClass, "jrnl", aTxt, szTxt+1);
+    kvvfsDecodeJournal(pFile, aTxt, szTxt);
+    sqlite3_free(aTxt);
+    if( pFile->aJrnl==0 ) return SQLITE_IOERR;
+  }
+  if( iOfst+iAmt>pFile->nJrnl ){
+    return SQLITE_IOERR_SHORT_READ;
+  }
+  memcpy(zBuf, pFile->aJrnl+iOfst, iAmt);
+  return SQLITE_OK;
+}
+
+/*
+** Read from the database file.
+*/
+static int kvvfsReadDb(
+  sqlite3_file *pProtoFile,
+  void *zBuf, 
+  int iAmt, 
+  sqlite_int64 iOfst
+){
+  KVVfsFile *pFile = (KVVfsFile*)pProtoFile;
+  unsigned int pgno;
+  int got, n;
+  char zKey[30];
+  char aData[133073];
+  assert( iOfst>=0 );
+  assert( iAmt>=0 );
+  SQLITE_KV_LOG(("xRead('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst));
+  if( iOfst+iAmt>=512 ){
+    if( (iOfst % iAmt)!=0 ){
+      return SQLITE_IOERR_READ;
+    }
+    if( (iAmt & (iAmt-1))!=0 || iAmt<512 || iAmt>65536 ){
+      return SQLITE_IOERR_READ;
+    }
+    pFile->szPage = iAmt;
+    pgno = 1 + iOfst/iAmt;
+  }else{
+    pgno = 1;
+  }
+  sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno);
+  got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, aData, sizeof(aData)-1);
+  if( got<0 ){
+    n = 0;
+  }else{
+    aData[got] = 0;
+    if( iOfst+iAmt<512 ){
+      int k = iOfst+iAmt;
+      aData[k*2] = 0;
+      n = kvvfsDecode(aData, &aData[2000], sizeof(aData)-2000);
+      if( n>=iOfst+iAmt ){
+        memcpy(zBuf, &aData[2000+iOfst], iAmt);
+        n = iAmt;
+      }else{
+        n = 0;
+      }
+    }else{
+      n = kvvfsDecode(aData, zBuf, iAmt);
+    }
+  }
+  if( n<iAmt ){
+    memset(zBuf+n, 0, iAmt-n);
+    return SQLITE_IOERR_SHORT_READ;
+  }
+  return SQLITE_OK;
+}
+
+
+/*
+** Write into the -journal file.
+*/
+static int kvvfsWriteJrnl(
+  sqlite3_file *pProtoFile,
+  const void *zBuf, 
+  int iAmt, 
+  sqlite_int64 iOfst
+){
+  KVVfsFile *pFile = (KVVfsFile*)pProtoFile;
+  sqlite3_int64 iEnd = iOfst+iAmt;
+  SQLITE_KV_LOG(("xWrite('%s-journal',%d,%lld)\n", pFile->zClass, iAmt, iOfst));
+  if( iEnd>=0x10000000 ) return SQLITE_FULL;
+  if( pFile->aJrnl==0 || pFile->nJrnl<iEnd ){
+    char *aNew = sqlite3_realloc(pFile->aJrnl, iEnd);
+    if( aNew==0 ){
+      return SQLITE_IOERR_NOMEM;
+    }
+    pFile->aJrnl = aNew;
+    if( pFile->nJrnl<iOfst ){
+      memset(pFile->aJrnl+pFile->nJrnl, 0, iOfst-pFile->nJrnl);
+    }
+    pFile->nJrnl = iEnd;
+  }
+  memcpy(pFile->aJrnl+iOfst, zBuf, iAmt);
+  return SQLITE_OK;
+}
+
+/*
+** Write into the database file.
+*/
+static int kvvfsWriteDb(
+  sqlite3_file *pProtoFile,
+  const void *zBuf, 
+  int iAmt, 
+  sqlite_int64 iOfst
+){
+  KVVfsFile *pFile = (KVVfsFile*)pProtoFile;
+  unsigned int pgno;
+  char zKey[30];
+  char aData[131073];
+  SQLITE_KV_LOG(("xWrite('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst));
+  assert( iAmt>=512 && iAmt<=65536 );
+  assert( (iAmt & (iAmt-1))==0 );
+  pgno = 1 + iOfst/iAmt;
+  sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno);
+  kvvfsEncode(zBuf, iAmt, aData);
+  if( sqlite3KvvfsMethods.xWrite(pFile->zClass, zKey, aData) ){
+    return SQLITE_IOERR;
+  }
+  if( iOfst+iAmt > pFile->szDb ){
+    pFile->szDb = iOfst + iAmt;
+  }
+  if( iAmt>=512 && (iAmt & (iAmt-1))==0 ){
+    pFile->szPage = iAmt;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Truncate an kvvfs-file.
+*/
+static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){
+  KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+  SQLITE_KV_LOG(("xTruncate('%s-journal',%lld)\n", pFile->zClass, size));
+  assert( size==0 );
+  sqlite3KvvfsMethods.xDelete(pFile->zClass, "jrnl");
+  sqlite3_free(pFile->aJrnl);
+  pFile->aJrnl = 0;
+  pFile->nJrnl = 0;
+  return SQLITE_OK;
+}
+static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){
+  KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+  if( pFile->szDb>size
+   && pFile->szPage>0 
+   && (size % pFile->szPage)==0
+  ){
+    char zKey[50];
+    unsigned int pgno, pgnoMax;
+    SQLITE_KV_LOG(("xTruncate('%s-db',%lld)\n", pFile->zClass, size));
+    pgno = 1 + size/pFile->szPage;
+    pgnoMax = 2 + pFile->szDb/pFile->szPage;
+    while( pgno<=pgnoMax ){
+      sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno);
+      sqlite3KvvfsMethods.xDelete(pFile->zClass, zKey);
+      pgno++;
+    }
+    pFile->szDb = size;
+    return kvvfsWriteFileSize(pFile, size) ? SQLITE_IOERR : SQLITE_OK;
+  }
+  return SQLITE_IOERR;
+}
+
+/*
+** Sync an kvvfs-file.
+*/
+static int kvvfsSyncJrnl(sqlite3_file *pProtoFile, int flags){
+  int i, n;
+  KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+  char *zOut;
+  SQLITE_KV_LOG(("xSync('%s-journal')\n", pFile->zClass));
+  if( pFile->nJrnl<=0 ){
+    return kvvfsTruncateJrnl(pProtoFile, 0);
+  }
+  zOut = sqlite3_malloc64( pFile->nJrnl*2 + 50 );
+  if( zOut==0 ){
+    return SQLITE_IOERR_NOMEM;
+  }
+  n = pFile->nJrnl;
+  i = 0;
+  do{
+    zOut[i++] = 'a' + (n%26);
+    n /= 26;
+  }while( n>0 );
+  zOut[i++] = ' ';
+  kvvfsEncode(pFile->aJrnl, pFile->nJrnl, &zOut[i]);
+  i = sqlite3KvvfsMethods.xWrite(pFile->zClass, "jrnl", zOut);
+  sqlite3_free(zOut);
+  return i ? SQLITE_IOERR : SQLITE_OK;
+}
+static int kvvfsSyncDb(sqlite3_file *pProtoFile, int flags){
+  return SQLITE_OK;
+}
+
+/*
+** Return the current file-size of an kvvfs-file.
+*/
+static int kvvfsFileSizeJrnl(sqlite3_file *pProtoFile, sqlite_int64 *pSize){
+  KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+  SQLITE_KV_LOG(("xFileSize('%s-journal')\n", pFile->zClass));
+  *pSize = pFile->nJrnl;
+  return SQLITE_OK;
+}
+static int kvvfsFileSizeDb(sqlite3_file *pProtoFile, sqlite_int64 *pSize){
+  KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+  SQLITE_KV_LOG(("xFileSize('%s-db')\n", pFile->zClass));
+  if( pFile->szDb>=0 ){
+    *pSize = pFile->szDb;
+  }else{
+    *pSize = kvvfsReadFileSize(pFile);
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Lock an kvvfs-file.
+*/
+static int kvvfsLock(sqlite3_file *pProtoFile, int eLock){
+  KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+  assert( !pFile->isJournal );
+  SQLITE_KV_LOG(("xLock(%s,%d)\n", pFile->zClass, eLock));
+
+  if( eLock!=SQLITE_LOCK_NONE ){
+    pFile->szDb = kvvfsReadFileSize(pFile);
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Unlock an kvvfs-file.
+*/
+static int kvvfsUnlock(sqlite3_file *pProtoFile, int eLock){
+  KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+  assert( !pFile->isJournal );
+  SQLITE_KV_LOG(("xUnlock(%s,%d)\n", pFile->zClass, eLock));
+  if( eLock==SQLITE_LOCK_NONE ){
+    pFile->szDb = -1;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Check if another file-handle holds a RESERVED lock on an kvvfs-file.
+*/
+static int kvvfsCheckReservedLock(sqlite3_file *pProtoFile, int *pResOut){
+  SQLITE_KV_LOG(("xCheckReservedLock\n"));
+  *pResOut = 0;
+  return SQLITE_OK;
+}
+
+/*
+** File control method. For custom operations on an kvvfs-file.
+*/
+static int kvvfsFileControlJrnl(sqlite3_file *pProtoFile, int op, void *pArg){
+  SQLITE_KV_LOG(("xFileControl(%d) on journal\n", op));
+  return SQLITE_NOTFOUND;
+}
+static int kvvfsFileControlDb(sqlite3_file *pProtoFile, int op, void *pArg){
+  SQLITE_KV_LOG(("xFileControl(%d) on database\n", op));
+  if( op==SQLITE_FCNTL_SYNC ){
+    KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+    int rc = SQLITE_OK;
+    SQLITE_KV_LOG(("xSync('%s-db')\n", pFile->zClass));
+    if( pFile->szDb>0 && 0!=kvvfsWriteFileSize(pFile, pFile->szDb) ){
+      rc = SQLITE_IOERR;
+    }
+    return rc;
+  }
+  return SQLITE_NOTFOUND;
+}
+
+/*
+** Return the sector-size in bytes for an kvvfs-file.
+*/
+static int kvvfsSectorSize(sqlite3_file *pFile){
+  return 512;
+}
+
+/*
+** Return the device characteristic flags supported by an kvvfs-file.
+*/
+static int kvvfsDeviceCharacteristics(sqlite3_file *pProtoFile){
+  return 0;
+}
+
+/****** sqlite3_vfs methods *************************************************/
+
+/*
+** Open an kvvfs file handle.
+*/
+static int kvvfsOpen(
+  sqlite3_vfs *pProtoVfs,
+  const char *zName,
+  sqlite3_file *pProtoFile,
+  int flags,
+  int *pOutFlags
+){
+  KVVfsFile *pFile = (KVVfsFile*)pProtoFile;
+  SQLITE_KV_LOG(("xOpen(\"%s\")\n", zName));
+  if( strcmp(zName, "local")==0
+   || strcmp(zName, "session")==0
+  ){
+    pFile->isJournal = 0;
+    pFile->base.pMethods = &kvvfs_db_io_methods;
+  }else
+  if( strcmp(zName, "local-journal")==0 
+   || strcmp(zName, "session-journal")==0
+  ){
+    pFile->isJournal = 1;
+    pFile->base.pMethods = &kvvfs_jrnl_io_methods;
+  }else{
+    return SQLITE_CANTOPEN;
+  }
+  if( zName[0]=='s' ){
+    pFile->zClass = "session";
+  }else{
+    pFile->zClass = "local";
+  }
+  pFile->aJrnl = 0;
+  pFile->nJrnl = 0;
+  pFile->szPage = -1;
+  pFile->szDb = -1;
+  return SQLITE_OK;
+}
+
+/*
+** Delete the file located at zPath. If the dirSync argument is true,
+** ensure the file-system modifications are synced to disk before
+** returning.
+*/
+static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
+  if( strcmp(zPath, "local-journal")==0 ){
+    sqlite3KvvfsMethods.xDelete("local", "jrnl");
+  }else
+  if( strcmp(zPath, "session-journal")==0 ){
+    sqlite3KvvfsMethods.xDelete("session", "jrnl");
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Test for access permissions. Return true if the requested permission
+** is available, or false otherwise.
+*/
+static int kvvfsAccess(
+  sqlite3_vfs *pProtoVfs, 
+  const char *zPath, 
+  int flags, 
+  int *pResOut
+){
+  SQLITE_KV_LOG(("xAccess(\"%s\")\n", zPath));
+  if( strcmp(zPath, "local-journal")==0 ){
+    *pResOut = sqlite3KvvfsMethods.xRead("local", "jrnl", 0, 0)>0;
+  }else
+  if( strcmp(zPath, "session-journal")==0 ){
+    *pResOut = sqlite3KvvfsMethods.xRead("session", "jrnl", 0, 0)>0;
+  }else
+  if( strcmp(zPath, "local")==0 ){
+    *pResOut = sqlite3KvvfsMethods.xRead("local", "sz", 0, 0)>0;
+  }else
+  if( strcmp(zPath, "session")==0 ){
+    *pResOut = sqlite3KvvfsMethods.xRead("session", "sz", 0, 0)>0;
+  }else
+  {
+    *pResOut = 0;
+  }
+  SQLITE_KV_LOG(("xAccess returns %d\n",*pResOut));
+  return SQLITE_OK;
+}
+
+/*
+** Populate buffer zOut with the full canonical pathname corresponding
+** to the pathname in zPath. zOut is guaranteed to point to a buffer
+** of at least (INST_MAX_PATHNAME+1) bytes.
+*/
+static int kvvfsFullPathname(
+  sqlite3_vfs *pVfs, 
+  const char *zPath, 
+  int nOut, 
+  char *zOut
+){
+  size_t nPath;
+#ifdef SQLITE_OS_KV_ALWAYS_LOCAL
+  zPath = "local";
+#endif
+  nPath = strlen(zPath);
+  SQLITE_KV_LOG(("xFullPathname(\"%s\")\n", zPath));
+  if( nOut<nPath+1 ) nPath = nOut - 1;
+  memcpy(zOut, zPath, nPath);
+  zOut[nPath] = 0;
+  return SQLITE_OK;
+}
+
+/*
+** Open the dynamic library located at zPath and return a handle.
+*/
+static void *kvvfsDlOpen(sqlite3_vfs *pVfs, const char *zPath){
+  return 0;
+}
+
+/*
+** Populate the buffer pointed to by zBufOut with nByte bytes of 
+** random data.
+*/
+static int kvvfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
+  memset(zBufOut, 0, nByte);
+  return nByte;
+}
+
+/*
+** Sleep for nMicro microseconds. Return the number of microseconds 
+** actually slept.
+*/
+static int kvvfsSleep(sqlite3_vfs *pVfs, int nMicro){
+  return SQLITE_OK;
+}
+
+/*
+** Return the current time as a Julian Day number in *pTimeOut.
+*/
+static int kvvfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
+  sqlite3_int64 i = 0;
+  int rc;
+  rc = kvvfsCurrentTimeInt64(0, &i);
+  *pTimeOut = i/86400000.0;
+  return rc;
+}
+#include <sys/time.h>
+static int kvvfsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){
+  static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000;
+  struct timeval sNow;
+  (void)gettimeofday(&sNow, 0);  /* Cannot fail given valid arguments */
+  *pTimeOut = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000;
+  return SQLITE_OK;
+}
+#endif /* SQLITE_OS_KV || SQLITE_OS_UNIX */
+
+#if SQLITE_OS_KV
+/* 
+** This routine is called initialize the KV-vfs as the default VFS.
+*/
+int sqlite3_os_init(void){
+  return sqlite3_vfs_register(&sqlite3OsKvvfsObject, 1);
+}
+int sqlite3_os_end(void){
+  return SQLITE_OK;
+}
+#endif /* SQLITE_OS_KV */
+
+#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL)
+int sqlite3KvvfsInit(void){
+  return sqlite3_vfs_register(&sqlite3OsKvvfsObject, 0);
+}
+#endif

src/os_setup.h

@@ -20,38 +20,72 @@
 ** Figure out if we are dealing with Unix, Windows, or some other operating
 ** system.
 **
-** After the following block of preprocess macros, all of SQLITE_OS_UNIX,
-** SQLITE_OS_WIN, and SQLITE_OS_OTHER will defined to either 1 or 0.  One of
-** the three will be 1.  The other two will be 0.
+** After the following block of preprocess macros, all of 
+**
+**    SQLITE_OS_KV
+**    SQLITE_OS_OTHER
+**    SQLITE_OS_UNIX
+**    SQLITE_OS_WIN
+**
+** will defined to either 1 or 0. One of them will be 1. The others will be 0.
+** If none of the macros are initially defined, then select either
+** SQLITE_OS_UNIX or SQLITE_OS_WIN depending on the target platform.
+**
+** If SQLITE_OS_OTHER=1 is specified at compile-time, then the application
+** must provide its own VFS implementation together with sqlite3_os_init()
+** and sqlite3_os_end() routines.
 */
-#if defined(SQLITE_OS_OTHER)
-#  if SQLITE_OS_OTHER==1
-#    undef SQLITE_OS_UNIX
+#if !defined(SQLITE_OS_KV) && !defined(SQLITE_OS_OTHER) && \
+       !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_WIN)
+#  if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \
+          defined(__MINGW32__) || defined(__BORLANDC__)
+#    define SQLITE_OS_WIN 1
 #    define SQLITE_OS_UNIX 0
-#    undef SQLITE_OS_WIN
-#    define SQLITE_OS_WIN 0
 #  else
-#    undef SQLITE_OS_OTHER
+#    define SQLITE_OS_WIN 0
+#    define SQLITE_OS_UNIX 1
 #  endif
 #endif
-#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER)
+#if SQLITE_OS_OTHER+1>1
+#  undef SQLITE_OS_KV
+#  define SQLITE_OS_KV 0
+#  undef SQLITE_OS_UNIX
+#  define SQLITE_OS_UNIX 0
+#  undef SQLITE_OS_WIN
+#  define SQLITE_OS_WIN 0
+#endif
+#if SQLITE_OS_KV+1>1
+#  undef SQLITE_OS_OTHER
 #  define SQLITE_OS_OTHER 0
-#  ifndef SQLITE_OS_WIN
-#    if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \
-        defined(__MINGW32__) || defined(__BORLANDC__)
-#      define SQLITE_OS_WIN 1
-#      define SQLITE_OS_UNIX 0
-#    else
-#      define SQLITE_OS_WIN 0
-#      define SQLITE_OS_UNIX 1
-#    endif
-#  else
-#    define SQLITE_OS_UNIX 0
-#  endif
-#else
-#  ifndef SQLITE_OS_WIN
-#    define SQLITE_OS_WIN 0
-#  endif
+#  undef SQLITE_OS_UNIX
+#  define SQLITE_OS_UNIX 0
+#  undef SQLITE_OS_WIN
+#  define SQLITE_OS_WIN 0
+#  define SQLITE_OMIT_LOAD_EXTENSION 1
+#  define SQLITE_OMIT_WAL 1
+#  define SQLITE_OMIT DEPRECATED 1
+#  undef SQLITE_TEMP_STORE
+#  define SQLITE_TEMP_STORE 3  /* Always use memory for temporary storage */
+#  define SQLITE_DQS 0
+#  define SQLITE_OMIT_SHARED_CACHE 1
+#  define SQLITE_OMIT_AUTOINIT 1
+#endif
+#if SQLITE_OS_UNIX+1>1
+#  undef SQLITE_OS_KV
+#  define SQLITE_OS_KV 0
+#  undef SQLITE_OS_OTHER
+#  define SQLITE_OS_OTHER 0
+#  undef SQLITE_OS_WIN
+#  define SQLITE_OS_WIN 0
+#endif
+#if SQLITE_OS_WIN+1>1
+#  undef SQLITE_OS_KV
+#  define SQLITE_OS_KV 0
+#  undef SQLITE_OS_OTHER
+#  define SQLITE_OS_OTHER 0
+#  undef SQLITE_OS_UNIX
+#  define SQLITE_OS_UNIX 0
 #endif
 
+
 #endif /* SQLITE_OS_SETUP_H */

src/os_unix.c

@@ -87,13 +87,13 @@
 /*
 ** standard include files.
 */
-#include <sys/types.h>
-#include <sys/stat.h>
+#include <sys/types.h>   /* amalgamator: keep */
+#include <sys/stat.h>    /* amalgamator: keep */
 #include <fcntl.h>
 #include <sys/ioctl.h>
-#include <unistd.h>
+#include <unistd.h>      /* amalgamator: keep */
 #include <time.h>
-#include <sys/time.h>
+#include <sys/time.h>    /* amalgamator: keep */
 #include <errno.h>
 #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0
 # include <sys/mman.h>
@@ -8065,6 +8065,9 @@ int sqlite3_os_init(void){
     sqlite3_vfs_register(&aVfs[i], i==0);
 #endif
   }
+#ifdef SQLITE_OS_KV_OPTIONAL
+  sqlite3KvvfsInit();
+#endif
   unixBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1);
 
 #ifndef SQLITE_OMIT_WAL

src/sqliteInt.h

@@ -1268,11 +1268,11 @@ typedef int VList;
 ** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque
 ** pointer types (i.e. FuncDef) defined above.
 */
+#include "os.h"
 #include "pager.h"
 #include "btree.h"
 #include "vdbe.h"
 #include "pcache.h"
-#include "os.h"
 #include "mutex.h"
 
 /* The SQLITE_EXTRA_DURABLE compile-time option used to set the default
@@ -5456,4 +5456,8 @@ void sqlite3VectorErrorMsg(Parse*, Expr*);
 const char **sqlite3CompileOptions(int *pnOpt);
 #endif
 
+#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL)
+int sqlite3KvvfsInit(void);
+#endif
+
 #endif /* SQLITEINT_H */

tool/mksqlite3c.tcl

@@ -355,6 +355,7 @@ foreach file {
    hash.c
    opcodes.c
 
+   os_kv.c
    os_unix.c
    os_win.c
    memdb.c