Add NEVER to two unreachable branches in JSON.

FossilOrigin-Name: c96ebb086feb89341565cc52b970ae7799ce1327fe1ad4fc790f1b0dcaa6e229
Improvements to JSONB validation - catch more cases where the input does
2023-12-12 17:13:10 +00:00 · 2023-12-12 14:33:52 +00:00 · 2023-12-12 02:31:12 +00:00 · 2023-12-11 21:00:55 +00:00 · 2023-12-11 20:44:21 +00:00 · 2023-12-11 20:19:10 +00:00
128 changed files with 12932 additions and 6328 deletions
@@ -710,6 +710,21 @@ fuzzcheck-asan$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
 fuzzcheck-ubsan$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
 	$(LTLINK) -o $@ -fsanitize=undefined $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)

+# Usage:    FUZZDB=filename make run-fuzzcheck
+#
+# Where filename is a fuzzcheck database, this target builds and runs
+# fuzzcheck, fuzzcheck-asan, and fuzzcheck-ubsan on that database.
+#
+# FUZZDB can be a glob pattern of two or more databases. Example:
+#
+#     FUZZDB=test/fuzzdata*.db make run-fuzzcheck
+#
+run-fuzzcheck:	fuzzcheck$(TEXE) fuzzcheck-asan$(TEXE) fuzzcheck-ubsan$(TEXE)
+	@if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi
+	./fuzzcheck$(TEXE) --spinner $(FUZZDB)
+	./fuzzcheck-asan$(TEXE) --spinner $(FUZZDB)
+	./fuzzcheck-ubsan$(TEXE) --spinner $(FUZZDB)
+
 ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
             $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
@@ -1135,21 +1150,21 @@ keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
 # Source files that go into making shell.c
 SHELL_SRC = \
 	$(TOP)/src/shell.c.in \
-        $(TOP)/ext/misc/appendvfs.c \
+	$(TOP)/ext/consio/console_io.c \
+	$(TOP)/ext/consio/console_io.h \
+	$(TOP)/ext/misc/appendvfs.c \
 	$(TOP)/ext/misc/completion.c \
-        $(TOP)/ext/consio/console_io.c \
-        $(TOP)/ext/consio/console_io.h \
-        $(TOP)/ext/misc/decimal.c \
-        $(TOP)/ext/misc/basexx.c \
-        $(TOP)/ext/misc/base64.c \
-        $(TOP)/ext/misc/base85.c \
+	$(TOP)/ext/misc/decimal.c \
+	$(TOP)/ext/misc/basexx.c \
+	$(TOP)/ext/misc/base64.c \
+	$(TOP)/ext/misc/base85.c \
 	$(TOP)/ext/misc/fileio.c \
-        $(TOP)/ext/misc/ieee754.c \
-        $(TOP)/ext/misc/regexp.c \
-        $(TOP)/ext/misc/series.c \
+	$(TOP)/ext/misc/ieee754.c \
+	$(TOP)/ext/misc/regexp.c \
+	$(TOP)/ext/misc/series.c \
 	$(TOP)/ext/misc/shathree.c \
 	$(TOP)/ext/misc/sqlar.c \
-        $(TOP)/ext/misc/uint.c \
+	$(TOP)/ext/misc/uint.c \
 	$(TOP)/ext/expert/sqlite3expert.c \
 	$(TOP)/ext/expert/sqlite3expert.h \
 	$(TOP)/ext/misc/zipfile.c \
@@ -1158,7 +1173,7 @@ SHELL_SRC = \
 	$(TOP)/ext/recover/dbdata.c \
 	$(TOP)/ext/recover/sqlite3recover.c \
 	$(TOP)/ext/recover/sqlite3recover.h \
-        $(TOP)/src/test_windirent.c
+	$(TOP)/src/test_windirent.c

 shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl has_tclsh84
 	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c
@@ -159,7 +159,7 @@ extension and only later escaped to the wild as an independent library.)

 Test scripts and programs are found in the **test/** subdirectory.
 Additional test code is found in other source repositories.
-See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for
+See [How SQLite Is Tested](https://www.sqlite.org/testing.html) for
 additional information.

 The **ext/** subdirectory contains code for extensions.  The
@@ -183,7 +183,7 @@ manually-edited files and automatically-generated files.

 The SQLite interface is defined by the **sqlite3.h** header file, which is
 generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION.  The
-[Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
+[Tcl script](https://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
 The manifest.uuid file contains the SHA3 hash of the particular check-in
 and is used to generate the SQLITE\_SOURCE\_ID macro.  The VERSION file
 contains the current SQLite version number.  The sqlite3.h header is really
@@ -250,14 +250,14 @@ individual source file exceeds 32K lines in length.
 ## How It All Fits Together

 SQLite is modular in design.
-See the [architectural description](http://www.sqlite.org/arch.html)
+See the [architectural description](https://www.sqlite.org/arch.html)
 for details. Other documents that are useful in
 (helping to understand how SQLite works include the
-[file format](http://www.sqlite.org/fileformat2.html) description,
-the [virtual machine](http://www.sqlite.org/opcode.html) that runs
+[file format](https://www.sqlite.org/fileformat2.html) description,
+the [virtual machine](https://www.sqlite.org/opcode.html) that runs
 prepared statements, the description of
-[how transactions work](http://www.sqlite.org/atomiccommit.html), and
-the [overview of the query planner](http://www.sqlite.org/optoverview.html).
+[how transactions work](https://www.sqlite.org/atomiccommit.html), and
+the [overview of the query planner](https://www.sqlite.org/optoverview.html).

 Years of effort have gone into optimizing SQLite, both
 for small size and high performance.  And optimizations tend to result in
@@ -353,7 +353,7 @@ hidden by also modifying the makefiles.

 ## Contacts

-The main SQLite website is [http:/sqlite.org/](http://sqlite.org/)
+The main SQLite website is [https://sqlite.org/](https://sqlite.org/)
 with geographically distributed backups at
-[http://www2.sqlite.org/](http://www2.sqlite.org) and
-[http://www3.sqlite.org/](http://www3.sqlite.org).
+[https://www2.sqlite.org/](https://www2.sqlite.org) and
+[https://www3.sqlite.org/](https://www3.sqlite.org).
@@ -1 +1 @@
-3.44.2
+3.45.0
@@ -19,7 +19,7 @@ dnl	to configure the system for the local environment.
 # so that we create the export library with the dll.
 #-----------------------------------------------------------------------

-AC_INIT([sqlite],[3.44.2])
+AC_INIT([sqlite],[3.45.0])

 #--------------------------------------------------------------------
 # Call TEA_INIT as the first TEA_ macro to set up initial vars.
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.44.2.
+# Generated by GNU Autoconf 2.69 for sqlite 3.45.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.44.2'
-PACKAGE_STRING='sqlite 3.44.2'
+PACKAGE_VERSION='3.45.0'
+PACKAGE_STRING='sqlite 3.45.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''

@@ -1472,7 +1472,7 @@ if test "$ac_init_help" = "long"; then
  # Omit some internal or obsolete options to make the list less imposing.
  # This message is too long to be a string in the A/UX 3.1 sh.
  cat <<_ACEOF
-\`configure' configures sqlite 3.44.2 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.45.0 to adapt to many kinds of systems.

 Usage: $0 [OPTION]... [VAR=VALUE]...

@@ -1537,7 +1537,7 @@ fi

 if test -n "$ac_init_help"; then
  case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.44.2:";;
+     short | recursive ) echo "Configuration of sqlite 3.45.0:";;
   esac
  cat <<\_ACEOF

@@ -1668,7 +1668,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
  cat <<\_ACEOF
-sqlite configure 3.44.2
+sqlite configure 3.45.0
 generated by GNU Autoconf 2.69

 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2087,7 +2087,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.

-It was created by sqlite $as_me 3.44.2, which was
+It was created by sqlite $as_me 3.45.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  $ $0 $@
@@ -12481,7 +12481,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.44.2, which was
+This file was extended by sqlite $as_me 3.45.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  CONFIG_FILES    = $CONFIG_FILES
@@ -12547,7 +12547,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.44.2
+sqlite config.status 3.45.0
 configured by $0, generated by GNU Autoconf 2.69,
  with options \\"\$ac_cs_config\\"

@@ -74,7 +74,7 @@
 # you don't need (for example BLT) by erasing or commenting out
 # the corresponding code.
 #
-AC_INIT([sqlite],[m4_esyscmd(cat VERSION | tr -d '\n')])
+AC_INIT([sqlite],m4_esyscmd(cat VERSION | tr -d '\n'))

 dnl Make sure the local VERSION file matches this configure script
 sqlite_version_sanity_check=`cat $srcdir/VERSION | tr -d '\n'`
@@ -1,7 +1,7 @@
 # Notes On Compiling SQLite On Windows 11

 Here are step-by-step instructions on how to build SQLite from
-canonical source on a new Windows 11 PC, as of 2023-08-16:
+canonical source on a new Windows 11 PC, as of 2023-11-01:

  1.  Install Microsoft Visual Studio. The free "community edition" 
      will work fine.  Do a standard install for C++ development.
@@ -84,6 +84,18 @@ following minor changes:
      <li>  `set PATH=c:\tcl32\bin;%PATH%`
      </ul>

+## Building a DLL
+
+The command the developers use for building the deliverable DLL on the 
+[download page](https://sqlite.org/download.html) is as follows:
+
+> ~~~~
+nmake /f Makefile.msc sqlite3.dll USE_NATIVE_LIBPATHS=1 "OPTS=-DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_FTS5=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1=1 -DSQLITE_ENABLE_GEOPOLY=1 -DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 -DSQLITE_ENABLE_SERIALIZE=1 -DSQLITE_ENABLE_MATH_FUNCTIONS=1"
+~~~~
+
+That command generates both the sqlite3.dll and sqlite3.def files.  The same
+command works for both 32-bit and 64-bit builds.
+
 ## Statically Linking The TCL Library

 Some utility programs associated with SQLite need to be linked
@@ -2,6 +2,26 @@

 # The testrunner.tcl Script

+<ul type=none>
+  <li> 1. <a href=#overview>Overview</a>
+  <li> 2. <a href=#binary_tests>Binary Tests</a>
+<ul type=none>
+  <li> 2.1. <a href=#organization_tests>Organization of Tcl Tests</a>
+  <li> 2.2. <a href=#run_tests>Commands to Run Tests</a>
+  <li> 2.3. <a href=#binary_test_failures>Investigating Binary Test Failures</a>
+</ul>
+  <li> 3. <a href=#source_code_tests>Source Tests</a>
+<ul type=none>
+  <li> 3.1. <a href=#commands_to_run_tests>Commands to Run SQLite Tests</a>
+  <li> 3.2. <a href=#zipvfs_tests>Running ZipVFS Tests</a>
+  <li> 3.3. <a href=#source_code_test_failures>Investigating Source Code Test Failures</a>
+</ul>
+  <li> 4. <a href=#testrunner_options>Extra testrunner.tcl Options</a>
+# 4. Extra testrunner.tcl Options
+  <li> 5. <a href=#cpu_cores>Controlling CPU Core Utilization</a>
+</ul>
+
+<a name=overview></a>
 # 1. Overview

 testrunner.tcl is a Tcl script used to run multiple SQLite tests using 
@@ -44,6 +64,7 @@ Sometimes testrunner.tcl uses the [testfixture] binary that it is run with
 to run tests (see "Binary Tests" below). Sometimes it builds testfixture and
 other binaries in specific configurations to test (see "Source Tests").

+<a name=binary_tests></a>
 # 2. Binary Tests

 The commands described in this section all run various combinations of the Tcl
@@ -61,6 +82,7 @@ these tests is therefore:
 The following sub-sections describe the various options that can be
 passed to testrunner.tcl to test binary testfixture builds.

+<a name=organization_tests></a>
 ## 2.1. Organization of Tcl Tests

 Tcl tests are stored in files that match the pattern *\*.test*. They are
@@ -91,6 +113,7 @@ Running **all** tests is to run all tests in the full test set, plus a dozen
 or so permutations. The specific permutations that are run as part of "all"
 are defined in file *testrunner_data.tcl*.

+<a name=run_tests></a>
 ## 2.2. Commands to Run Tests

 To run the "veryquick" test set, use either of the following:
@@ -114,6 +137,12 @@ a specified pattern (e.g. all tests that start with "fts5"), either of:
  ./testfixture $TESTDIR/testrunner.tcl 'fts5*'
 ```

+Strictly speaking, for a test to be run the pattern must match the script
+filename, not including the directory, using the rules of Tcl's 
+\[string match\] command. Except that before the matching is done, any "%"
+characters specified as part of the pattern are transformed to "\*".
+
+
 To run "all" tests (full + permutations):

 ```
@@ -141,6 +170,7 @@ Or, if the failure occured as part of a permutation:

 TODO: An example instead of "$PERMUTATION" and $PATH\_TO\_SCRIPT?

+<a name=source_code_tests></a>
 # 3. Source Code Tests

 The commands described in this section invoke the C compiler to build 
@@ -159,7 +189,8 @@ shell that supports SQLite 3.31.1 or newer via "package require sqlite3".

 TODO: ./configure + Makefile.msc build systems.

-## Commands to Run SQLite Tests
+<a name=commands_to_run_tests></a>
+## 3.1. Commands to Run SQLite Tests

 The **mdevtest** command is equivalent to running the veryquick tests and
 the [make fuzztest] target once for each of two --enable-all builds - one 
@@ -201,7 +232,18 @@ of the specific tests run.
  tclsh $TESTDIR/testrunner.tcl release
 ```

-## Running ZipVFS Tests
+As with <a href=#source code tests>source code tests</a>, one or more patterns
+may be appended to any of the above commands (mdevtest, sdevtest or release).
+In that case only Tcl tests (no fuzz or other tests) that match the specified
+pattern are run. For example, to run the just the Tcl rtree tests in all 
+builds and configurations supported by "release":
+
+```
+  tclsh $TESTDIR/testrunner.tcl release rtree%
+```
+
+<a name=zipvfs_tests></a>
+## 3.2. Running ZipVFS Tests

 testrunner.tcl can build a zipvfs-enabled testfixture and use it to run
 tests from the Zipvfs project with the following command:
@@ -217,7 +259,8 @@ test both SQLite and Zipvfs with a single command:
  tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS mdevtest
 ```

-## Investigating Source Code Test Failures
+<a name=source_code_test_failures></a>
+## 3.3. Investigating Source Code Test Failures

 Investigating a test failure that occurs during source code testing is a
 two step process:
@@ -244,9 +287,31 @@ target to build. This may be used either to run a [make] command test directly,
 or else to build a testfixture (or testfixture.exe) binary with which to
 run a Tcl test script, as <a href=#binary_test_failures>described above</a>.

+<a name=testrunner_options></a>
+# 4. Extra testrunner.tcl Options

+The testrunner.tcl script options in this section may be used with both source
+code and binary tests.

-# 4. Controlling CPU Core Utilization
+The **--buildonly** option instructs testrunner.tcl just to build the binaries
+required by a test, not to run any actual tests. For example:
+
+```
+  # Build binaries required by release test.
+  tclsh $TESTDIR/testrunner.tcl --buildonly release"
+```
+
+The **--dryrun** option prevents testrunner.tcl from building any binaries
+or running any tests. Instead, it just writes the shell commands that it
+would normally execute into the testrunner.log file. Example:
+
+```
+  # Log the shell commmands that make up the mdevtest test.
+  tclsh $TESTDIR/testrunner.tcl --dryrun mdevtest"
+```
+
+<a name=cpu_cores></a>
+# 5. Controlling CPU Core Utilization

 When running either binary or source code tests, testrunner.tcl reports the
 number of jobs it intends to use to stdout. e.g.
@@ -277,8 +342,3 @@ testrunner.log and testrunner.db files:



-
-
-
-
-
@@ -549,12 +549,11 @@ zSkipValidUtf8(const char *z, int nAccept, long ccm){
 #endif /*!(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE))*/

 #ifndef SQLITE_CIO_NO_TRANSLATE
-
-#ifdef CONSIO_SPUTB
+# ifdef CONSIO_SPUTB
 SQLITE_INTERNAL_LINKAGE int
 fPutbUtf8(FILE *pfO, const char *cBuf, int nAccept){
  assert(pfO!=0);
-# if CIO_WIN_WC_XLATE
+#  if CIO_WIN_WC_XLATE
  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
  PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
  if( pstReachesConsole(ppst) ){
@@ -564,13 +563,13 @@ fPutbUtf8(FILE *pfO, const char *cBuf, int nAccept){
    if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
    return rv;
  }else {
-# endif
+#  endif
    return (int)fwrite(cBuf, 1, nAccept, pfO);
-# if CIO_WIN_WC_XLATE
+#  if CIO_WIN_WC_XLATE
  }
-# endif
+#  endif
 }
-#endif /* defined(CONSIO_SPUTB) */
+# endif

 SQLITE_INTERNAL_LINKAGE int
 oPutbUtf8(const char *cBuf, int nAccept){
@@ -166,8 +166,8 @@ SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z);
 #ifdef CONSIO_SPUTB
 SQLITE_INTERNAL_LINKAGE int
 fPutbUtf8(FILE *pfOut, const char *cBuf, int nAccept);
-#endif
 /* Like fPutbUtf8 except stream is always the designated output. */
+#endif
 SQLITE_INTERNAL_LINKAGE int
 oPutbUtf8(const char *cBuf, int nAccept);
 /* Like fPutbUtf8 except stream is always the designated error. */
@@ -223,10 +223,12 @@ proc main {data} {
    Fts5ExtensionApi {
      set struct [get_fts5_struct $data "^struct Fts5ExtensionApi" "^.;"]
      set map [list]
+      set lKey [list]
      foreach {k v} [get_struct_members $data] {
        if {[string match x* $k]==0} continue
-        lappend map $k "<a href=#$k>$k</a>"
+        lappend lKey $k
      }
+      foreach k [lsort -decr $lKey] { lappend map $k "<a href=#$k>$k</a>" }
      output [string map $map $struct]
    }

@@ -261,9 +261,34 @@ struct Fts5PhraseIter {
 **
 ** xPhraseNextColumn()
 **   See xPhraseFirstColumn above.
+**
+** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)
+**   This is used to access token iToken of phrase iPhrase of the current
+**   query. Before returning, output parameter *ppToken is set to point
+**   to a buffer containing the requested token, and *pnToken to the
+**   size of this buffer in bytes.
+**
+**   The output text is not a copy of the query text that specified the
+**   token. It is the output of the tokenizer module. For tokendata=1
+**   tables, this includes any embedded 0x00 and trailing data.
+**
+** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)
+**   This is used to access token iToken of phrase hit iIdx within the
+**   current row. Output variable (*ppToken) is set to point to a buffer
+**   containing the matching document token, and (*pnToken) to the size
+**   of that buffer in bytes. This API is not available if the specified 
+**   token matches a prefix query term. In that case both output variables 
+**   are always set to 0.
+**
+**   The output text is not a copy of the document text that was tokenized.
+**   It is the output of the tokenizer module. For tokendata=1 tables, this 
+**   includes any embedded 0x00 and trailing data.
+**
+**   This API can be quite slow if used with an FTS5 table created with the
+**   "detail=none" or "detail=column" option.
 */
 struct Fts5ExtensionApi {
-  int iVersion;                   /* Currently always set to 2 */
+  int iVersion;                   /* Currently always set to 3 */

  void *(*xUserData)(Fts5Context*);

@@ -298,6 +323,13 @@ struct Fts5ExtensionApi {

  int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
  void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
+
+  /* Below this point are iVersion>=3 only */
+  int (*xQueryToken)(Fts5Context*, 
+      int iPhrase, int iToken, 
+      const char **ppToken, int *pnToken
+  );
+  int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
 };

 /* 
@@ -196,6 +196,7 @@ struct Fts5Config {
  char *zContent;                 /* content table */ 
  char *zContentRowid;            /* "content_rowid=" option value */ 
  int bColumnsize;                /* "columnsize=" option value (dflt==1) */
+  int bTokendata;                 /* "tokendata=" option value (dflt==0) */
  int eDetail;                    /* FTS5_DETAIL_XXX value */
  char *zContentExprlist;
  Fts5Tokenizer *pTok;
@@ -384,17 +385,19 @@ struct Fts5IndexIter {
 /*
 ** Values used as part of the flags argument passed to IndexQuery().
 */
-#define FTS5INDEX_QUERY_PREFIX     0x0001   /* Prefix query */
-#define FTS5INDEX_QUERY_DESC       0x0002   /* Docs in descending rowid order */
-#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004   /* Do not use prefix index */
-#define FTS5INDEX_QUERY_SCAN       0x0008   /* Scan query (fts5vocab) */
+#define FTS5INDEX_QUERY_PREFIX      0x0001  /* Prefix query */
+#define FTS5INDEX_QUERY_DESC        0x0002  /* Docs in descending rowid order */
+#define FTS5INDEX_QUERY_TEST_NOIDX  0x0004  /* Do not use prefix index */
+#define FTS5INDEX_QUERY_SCAN        0x0008  /* Scan query (fts5vocab) */

 /* The following are used internally by the fts5_index.c module. They are
 ** defined here only to make it easier to avoid clashes with the flags
 ** above. */
-#define FTS5INDEX_QUERY_SKIPEMPTY  0x0010
-#define FTS5INDEX_QUERY_NOOUTPUT   0x0020
-#define FTS5INDEX_QUERY_SKIPHASH   0x0040
+#define FTS5INDEX_QUERY_SKIPEMPTY   0x0010
+#define FTS5INDEX_QUERY_NOOUTPUT    0x0020
+#define FTS5INDEX_QUERY_SKIPHASH    0x0040
+#define FTS5INDEX_QUERY_NOTOKENDATA 0x0080
+#define FTS5INDEX_QUERY_SCANONETERM 0x0100

 /*
 ** Create/destroy an Fts5Index object.
@@ -463,6 +466,10 @@ void *sqlite3Fts5StructureRef(Fts5Index*);
 void sqlite3Fts5StructureRelease(void*);
 int sqlite3Fts5StructureTest(Fts5Index*, void*);

+/*
+** Used by xInstToken():
+*/
+int sqlite3Fts5IterToken(Fts5IndexIter*, i64, int, int, const char**, int*);

 /*
 ** Insert or remove data to or from the index. Each time a document is 
@@ -540,6 +547,13 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin);
 int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid);

+void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*);
+
+/* Used to populate hash tables for xInstToken in detail=none/column mode. */
+int sqlite3Fts5IndexIterWriteTokendata(
+    Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff
+);
+
 /*
 ** End of interface to code in fts5_index.c.
 **************************************************************************/
@@ -645,6 +659,7 @@ void sqlite3Fts5HashScanNext(Fts5Hash*);
 int sqlite3Fts5HashScanEof(Fts5Hash*);
 void sqlite3Fts5HashScanEntry(Fts5Hash *,
  const char **pzTerm,            /* OUT: term (nul-terminated) */
+  int *pnTerm,                    /* OUT: Size of term in bytes */
  const u8 **ppDoclist,           /* OUT: pointer to doclist */
  int *pnDoclist                  /* OUT: size of doclist in bytes */
 );
@@ -771,6 +786,10 @@ int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);

 int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);

+int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*);
+int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*);
+void sqlite3Fts5ExprClearTokens(Fts5Expr*);
+
 /*******************************************
 ** The fts5_expr.c API above this point is used by the other hand-written
 ** C code in this module. The interfaces below this point are called by
@@ -211,6 +211,14 @@ static int fts5HighlightCb(
  }

  if( iPos==p->iRangeEnd ){
+    if( p->bOpen ){
+      if( p->iter.iStart>=0 && iPos>=p->iter.iStart ){
+        fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
+        p->iOff = iEndOff;
+      }
+      fts5HighlightAppend(&rc, p, p->zClose, -1);
+      p->bOpen = 0;
+    }
    fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
    p->iOff = iEndOff;
  }
@@ -398,6 +398,16 @@ static int fts5ConfigParseSpecial(
    return rc;
  }

+  if( sqlite3_strnicmp("tokendata", zCmd, nCmd)==0 ){
+    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
+      *pzErr = sqlite3_mprintf("malformed tokendata=... directive");
+      rc = SQLITE_ERROR;
+    }else{
+      pConfig->bTokendata = (zArg[0]=='1');
+    }
+    return rc;
+  }
+
  *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
  return SQLITE_ERROR;
 }
@@ -100,7 +100,9 @@ struct Fts5ExprNode {
 struct Fts5ExprTerm {
  u8 bPrefix;                     /* True for a prefix term */
  u8 bFirst;                      /* True if token must be first in column */
-  char *zTerm;                    /* nul-terminated term */
+  char *pTerm;                    /* Term data */
+  int nQueryTerm;                 /* Effective size of term in bytes */
+  int nFullTerm;                  /* Size of term in bytes incl. tokendata */
  Fts5IndexIter *pIter;           /* Iterator for this term */
  Fts5ExprTerm *pSynonym;         /* Pointer to first in list of synonyms */
 };
@@ -967,7 +969,7 @@ static int fts5ExprNearInitAll(
            p->pIter = 0;
          }
          rc = sqlite3Fts5IndexQuery(
-              pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
+              pExpr->pIndex, p->pTerm, p->nQueryTerm,
              (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
              (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
              pNear->pColset,
@@ -1604,7 +1606,7 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
      Fts5ExprTerm *pSyn;
      Fts5ExprTerm *pNext;
      Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
-      sqlite3_free(pTerm->zTerm);
+      sqlite3_free(pTerm->pTerm);
      sqlite3Fts5IterClose(pTerm->pIter);
      for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){
        pNext = pSyn->pSynonym;
@@ -1702,6 +1704,7 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
 typedef struct TokenCtx TokenCtx;
 struct TokenCtx {
  Fts5ExprPhrase *pPhrase;
+  Fts5Config *pConfig;
  int rc;
 };

@@ -1735,8 +1738,10 @@ static int fts5ParseTokenize(
      rc = SQLITE_NOMEM;
    }else{
      memset(pSyn, 0, (size_t)nByte);
-      pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
-      memcpy(pSyn->zTerm, pToken, nToken);
+      pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
+      pSyn->nFullTerm = pSyn->nQueryTerm = nToken;
+      if( pCtx->pConfig->bTokendata ) pSyn->nQueryTerm = strlen(pSyn->pTerm);
+      memcpy(pSyn->pTerm, pToken, nToken);
      pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
      pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn;
    }
@@ -1761,7 +1766,11 @@ static int fts5ParseTokenize(
    if( rc==SQLITE_OK ){
      pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
      memset(pTerm, 0, sizeof(Fts5ExprTerm));
-      pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
+      pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
+      pTerm->nFullTerm = pTerm->nQueryTerm = nToken;
+      if( pCtx->pConfig->bTokendata && rc==SQLITE_OK ){ 
+        pTerm->nQueryTerm = strlen(pTerm->pTerm);
+      }
    }
  }

@@ -1828,6 +1837,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(

  memset(&sCtx, 0, sizeof(TokenCtx));
  sCtx.pPhrase = pAppend;
+  sCtx.pConfig = pConfig;

  rc = fts5ParseStringFromToken(pToken, &z);
  if( rc==SQLITE_OK ){
@@ -1877,8 +1887,7 @@ int sqlite3Fts5ExprClonePhrase(
  int rc = SQLITE_OK;             /* Return code */
  Fts5ExprPhrase *pOrig;          /* The phrase extracted from pExpr */
  Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
-  TokenCtx sCtx = {0,0};          /* Context object for fts5ParseTokenize */
-
+  TokenCtx sCtx = {0,0,0};        /* Context object for fts5ParseTokenize */
  pOrig = pExpr->apExprPhrase[iPhrase];
  pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
  if( rc==SQLITE_OK ){
@@ -1909,13 +1918,12 @@ int sqlite3Fts5ExprClonePhrase(

  if( pOrig->nTerm ){
    int i;                          /* Used to iterate through phrase terms */
+    sCtx.pConfig = pExpr->pConfig;
    for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
      int tflags = 0;
      Fts5ExprTerm *p;
      for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
-        const char *zTerm = p->zTerm;
-        rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
-            0, 0);
+        rc = fts5ParseTokenize((void*)&sCtx, tflags, p->pTerm,p->nFullTerm,0,0);
        tflags = FTS5_TOKEN_COLOCATED;
      }
      if( rc==SQLITE_OK ){
@@ -2296,11 +2304,13 @@ static Fts5ExprNode *fts5ParsePhraseToAnd(
        if( parseGrowPhraseArray(pParse) ){
          fts5ExprPhraseFree(pPhrase);
        }else{
+          Fts5ExprTerm *p = &pNear->apPhrase[0]->aTerm[ii];
+          Fts5ExprTerm *pTo = &pPhrase->aTerm[0];
          pParse->apPhrase[pParse->nPhrase++] = pPhrase;
          pPhrase->nTerm = 1;
-          pPhrase->aTerm[0].zTerm = sqlite3Fts5Strndup(
-              &pParse->rc, pNear->apPhrase[0]->aTerm[ii].zTerm, -1
-          );
+          pTo->pTerm = sqlite3Fts5Strndup(&pParse->rc, p->pTerm, p->nFullTerm);
+          pTo->nQueryTerm = p->nQueryTerm;
+          pTo->nFullTerm = p->nFullTerm;
          pRet->apChild[ii] = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 
              0, 0, sqlite3Fts5ParseNearset(pParse, 0, pPhrase)
          );
@@ -2485,16 +2495,17 @@ static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){

  /* Determine the maximum amount of space required. */
  for(p=pTerm; p; p=p->pSynonym){
-    nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2;
+    nByte += pTerm->nQueryTerm * 2 + 3 + 2;
  }
  zQuoted = sqlite3_malloc64(nByte);

  if( zQuoted ){
    int i = 0;
    for(p=pTerm; p; p=p->pSynonym){
-      char *zIn = p->zTerm;
+      char *zIn = p->pTerm;
+      char *zEnd = &zIn[p->nQueryTerm];
      zQuoted[i++] = '"';
-      while( *zIn ){
+      while( zIn<zEnd ){
        if( *zIn=='"' ) zQuoted[i++] = '"';
        zQuoted[i++] = *zIn++;
      }
@@ -2572,8 +2583,10 @@ static char *fts5ExprPrintTcl(

      zRet = fts5PrintfAppend(zRet, " {");
      for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){
-        char *zTerm = pPhrase->aTerm[iTerm].zTerm;
-        zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm);
+        Fts5ExprTerm *p = &pPhrase->aTerm[iTerm];
+        zRet = fts5PrintfAppend(zRet, "%s%.*s", iTerm==0?"":" ", 
+            p->nQueryTerm, p->pTerm
+        );
        if( pPhrase->aTerm[iTerm].bPrefix ){
          zRet = fts5PrintfAppend(zRet, "*");
        }
@@ -2974,6 +2987,17 @@ static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){
  return 0;
 }

+/*
+** pToken is a buffer nToken bytes in size that may or may not contain
+** an embedded 0x00 byte. If it does, return the number of bytes in
+** the buffer before the 0x00. If it does not, return nToken.
+*/
+static int fts5QueryTerm(const char *pToken, int nToken){
+  int ii;
+  for(ii=0; ii<nToken && pToken[ii]; ii++){}
+  return ii;
+}
+
 static int fts5ExprPopulatePoslistsCb(
  void *pCtx,                /* Copy of 2nd argument to xTokenize() */
  int tflags,                /* Mask of FTS5_TOKEN_* flags */
@@ -2985,22 +3009,33 @@ static int fts5ExprPopulatePoslistsCb(
  Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
  Fts5Expr *pExpr = p->pExpr;
  int i;
+  int nQuery = nToken;
+  i64 iRowid = pExpr->pRoot->iRowid;

  UNUSED_PARAM2(iUnused1, iUnused2);

-  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
+  if( nQuery>FTS5_MAX_TOKEN_SIZE ) nQuery = FTS5_MAX_TOKEN_SIZE;
+  if( pExpr->pConfig->bTokendata ){
+    nQuery = fts5QueryTerm(pToken, nQuery);
+  }
  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
  for(i=0; i<pExpr->nPhrase; i++){
-    Fts5ExprTerm *pTerm;
+    Fts5ExprTerm *pT;
    if( p->aPopulator[i].bOk==0 ) continue;
-    for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
-      int nTerm = (int)strlen(pTerm->zTerm);
-      if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
-       && memcmp(pTerm->zTerm, pToken, nTerm)==0
+    for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){
+      if( (pT->nQueryTerm==nQuery || (pT->nQueryTerm<nQuery && pT->bPrefix))
+       && memcmp(pT->pTerm, pToken, pT->nQueryTerm)==0
      ){
        int rc = sqlite3Fts5PoslistWriterAppend(
            &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
        );
+        if( rc==SQLITE_OK && pExpr->pConfig->bTokendata && !pT->bPrefix ){
+          int iCol = p->iOff>>32;
+          int iTokOff = p->iOff & 0x7FFFFFFF;
+          rc = sqlite3Fts5IndexIterWriteTokendata(
+              pT->pIter, pToken, nToken, iRowid, iCol, iTokOff
+          );
+        }
        if( rc ) return rc;
        break;
      }
@@ -3135,3 +3170,81 @@ int sqlite3Fts5ExprPhraseCollist(

  return rc;
 }
+
+/*
+** Does the work of the fts5_api.xQueryToken() API method.
+*/
+int sqlite3Fts5ExprQueryToken(
+  Fts5Expr *pExpr, 
+  int iPhrase, 
+  int iToken, 
+  const char **ppOut, 
+  int *pnOut
+){
+  Fts5ExprPhrase *pPhrase = 0;
+
+  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
+    return SQLITE_RANGE;
+  }
+  pPhrase = pExpr->apExprPhrase[iPhrase];
+  if( iToken<0 || iToken>=pPhrase->nTerm ){
+    return SQLITE_RANGE;
+  }
+
+  *ppOut = pPhrase->aTerm[iToken].pTerm;
+  *pnOut = pPhrase->aTerm[iToken].nFullTerm;
+  return SQLITE_OK;
+}
+
+/*
+** Does the work of the fts5_api.xInstToken() API method.
+*/
+int sqlite3Fts5ExprInstToken(
+  Fts5Expr *pExpr, 
+  i64 iRowid,
+  int iPhrase, 
+  int iCol, 
+  int iOff, 
+  int iToken, 
+  const char **ppOut, 
+  int *pnOut
+){
+  Fts5ExprPhrase *pPhrase = 0;
+  Fts5ExprTerm *pTerm = 0;
+  int rc = SQLITE_OK;
+
+  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
+    return SQLITE_RANGE;
+  }
+  pPhrase = pExpr->apExprPhrase[iPhrase];
+  if( iToken<0 || iToken>=pPhrase->nTerm ){
+    return SQLITE_RANGE;
+  }
+  pTerm = &pPhrase->aTerm[iToken];
+  if( pTerm->bPrefix==0 ){
+    if( pExpr->pConfig->bTokendata ){
+      rc = sqlite3Fts5IterToken(
+          pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut
+      );
+    }else{
+      *ppOut = pTerm->pTerm;
+      *pnOut = pTerm->nFullTerm;
+    }
+  }
+  return rc;
+}
+
+/*
+** Clear the token mappings for all Fts5IndexIter objects mannaged by 
+** the expression passed as the only argument.
+*/
+void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){
+  int ii;
+  for(ii=0; ii<pExpr->nPhrase; ii++){
+    Fts5ExprTerm *pT;
+    for(pT=&pExpr->apExprPhrase[ii]->aTerm[0]; pT; pT=pT->pSynonym){
+      sqlite3Fts5IndexIterClearTokendata(pT->pIter);
+    }
+  }
+}
+
@@ -36,10 +36,15 @@ struct Fts5Hash {

 /*
 ** Each entry in the hash table is represented by an object of the 
-** following type. Each object, its key (a nul-terminated string) and 
-** its current data are stored in a single memory allocation. The 
-** key immediately follows the object in memory. The position list
-** data immediately follows the key data in memory.
+** following type. Each object, its key, and its current data are stored 
+** in a single memory allocation. The key immediately follows the object 
+** in memory. The position list data immediately follows the key data 
+** in memory.
+**
+** The key is Fts5HashEntry.nKey bytes in size. It consists of a single
+** byte identifying the index (either the main term index or a prefix-index),
+** followed by the term data. For example: "0token". There is no 
+** nul-terminator - in this case nKey=6.
 **
 ** The data that follows the key is in a similar, but not identical format
 ** to the doclist data stored in the database. It is:
@@ -174,8 +179,7 @@ static int fts5HashResize(Fts5Hash *pHash){
      unsigned int iHash;
      Fts5HashEntry *p = apOld[i];
      apOld[i] = p->pHashNext;
-      iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p),
-                          (int)strlen(fts5EntryKey(p)));
+      iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), p->nKey);
      p->pHashNext = apNew[iHash];
      apNew[iHash] = p;
    }
@@ -259,7 +263,7 @@ int sqlite3Fts5HashWrite(
  for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
    char *zKey = fts5EntryKey(p);
    if( zKey[0]==bByte 
-     && p->nKey==nToken
+     && p->nKey==nToken+1
     && memcmp(&zKey[1], pToken, nToken)==0 
    ){
      break;
@@ -289,9 +293,9 @@ int sqlite3Fts5HashWrite(
    zKey[0] = bByte;
    memcpy(&zKey[1], pToken, nToken);
    assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) );
-    p->nKey = nToken;
+    p->nKey = nToken+1;
    zKey[nToken+1] = '\0';
-    p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry);
+    p->nData = nToken+1 + sizeof(Fts5HashEntry);
    p->pHashNext = pHash->aSlot[iHash];
    pHash->aSlot[iHash] = p;
    pHash->nEntry++;
@@ -408,12 +412,17 @@ static Fts5HashEntry *fts5HashEntryMerge(
      *ppOut = p1;
      p1 = 0;
    }else{
-      int i = 0;
      char *zKey1 = fts5EntryKey(p1);
      char *zKey2 = fts5EntryKey(p2);
-      while( zKey1[i]==zKey2[i] ) i++;
+      int nMin = MIN(p1->nKey, p2->nKey);

-      if( ((u8)zKey1[i])>((u8)zKey2[i]) ){
+      int cmp = memcmp(zKey1, zKey2, nMin);
+      if( cmp==0 ){
+        cmp = p1->nKey - p2->nKey;
+      }
+      assert( cmp!=0 );
+
+      if( cmp>0 ){
        /* p2 is smaller */
        *ppOut = p2;
        ppOut = &p2->pScanNext;
@@ -455,7 +464,7 @@ static int fts5HashEntrySort(
    Fts5HashEntry *pIter;
    for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
      if( pTerm==0 
-       || (pIter->nKey+1>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm))
+       || (pIter->nKey>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm))
      ){
        Fts5HashEntry *pEntry = pIter;
        pEntry->pScanNext = 0;
@@ -494,12 +503,11 @@ int sqlite3Fts5HashQuery(

  for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
    zKey = fts5EntryKey(p);
-    assert( p->nKey+1==(int)strlen(zKey) );
-    if( nTerm==p->nKey+1 && memcmp(zKey, pTerm, nTerm)==0 ) break;
+    if( nTerm==p->nKey && memcmp(zKey, pTerm, nTerm)==0 ) break;
  }

  if( p ){
-    int nHashPre = sizeof(Fts5HashEntry) + nTerm + 1;
+    int nHashPre = sizeof(Fts5HashEntry) + nTerm;
    int nList = p->nData - nHashPre;
    u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10));
    if( pRet ){
@@ -560,19 +568,22 @@ int sqlite3Fts5HashScanEof(Fts5Hash *p){
 void sqlite3Fts5HashScanEntry(
  Fts5Hash *pHash,
  const char **pzTerm,            /* OUT: term (nul-terminated) */
+  int *pnTerm,                    /* OUT: Size of term in bytes */
  const u8 **ppDoclist,           /* OUT: pointer to doclist */
  int *pnDoclist                  /* OUT: size of doclist in bytes */
 ){
  Fts5HashEntry *p;
  if( (p = pHash->pScan) ){
    char *zKey = fts5EntryKey(p);
-    int nTerm = (int)strlen(zKey);
+    int nTerm = p->nKey;
    fts5HashAddPoslistSize(pHash, p, 0);
    *pzTerm = zKey;
-    *ppDoclist = (const u8*)&zKey[nTerm+1];
-    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
+    *pnTerm = nTerm;
+    *ppDoclist = (const u8*)&zKey[nTerm];
+    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm);
  }else{
    *pzTerm = 0;
+    *pnTerm = 0;
    *ppDoclist = 0;
    *pnDoclist = 0;
  }
@@ -118,7 +118,7 @@ struct Fts5FullTable {
  Fts5Global *pGlobal;            /* Global (connection wide) data */
  Fts5Cursor *pSortCsr;           /* Sort data from this cursor */
  int iSavepoint;                 /* Successful xSavepoint()+1 */
-  int bInSavepoint;
+  
 #ifdef SQLITE_DEBUG
  struct Fts5TransactionState ts;
 #endif
@@ -656,12 +656,15 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
  }
  idxStr[iIdxStr] = '\0';

-  /* Set idxFlags flags for the ORDER BY clause */
+  /* Set idxFlags flags for the ORDER BY clause
+  **
+  ** Note that tokendata=1 tables cannot currently handle "ORDER BY rowid DESC".
+  */
  if( pInfo->nOrderBy==1 ){
    int iSort = pInfo->aOrderBy[0].iColumn;
    if( iSort==(pConfig->nCol+1) && bSeenMatch ){
      idxFlags |= FTS5_BI_ORDER_RANK;
-    }else if( iSort==-1 ){
+    }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){
      idxFlags |= FTS5_BI_ORDER_ROWID;
    }
    if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){
@@ -913,6 +916,16 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
  );
  assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) );

+  /* If this cursor uses FTS5_PLAN_MATCH and this is a tokendata=1 table,
+  ** clear any token mappings accumulated at the fts5_index.c level. In
+  ** other cases, specifically FTS5_PLAN_SOURCE and FTS5_PLAN_SORTED_MATCH,
+  ** we need to retain the mappings for the entire query.  */
+  if( pCsr->ePlan==FTS5_PLAN_MATCH 
+   && ((Fts5Table*)pCursor->pVtab)->pConfig->bTokendata 
+  ){
+    sqlite3Fts5ExprClearTokens(pCsr->pExpr);
+  }
+
  if( pCsr->ePlan<3 ){
    int bSkip = 0;
    if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
@@ -2063,12 +2076,6 @@ static int fts5ApiInst(
  ){
    if( iIdx<0 || iIdx>=pCsr->nInstCount ){
      rc = SQLITE_RANGE;
-#if 0
-    }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){
-      *piPhrase = pCsr->aInst[iIdx*3];
-      *piCol = pCsr->aInst[iIdx*3 + 2];
-      *piOff = -1;
-#endif
    }else{
      *piPhrase = pCsr->aInst[iIdx*3];
      *piCol = pCsr->aInst[iIdx*3 + 1];
@@ -2323,13 +2330,56 @@ static int fts5ApiPhraseFirstColumn(
  return rc;
 }

+/*
+** xQueryToken() API implemenetation.
+*/
+static int fts5ApiQueryToken(
+  Fts5Context* pCtx, 
+  int iPhrase, 
+  int iToken, 
+  const char **ppOut, 
+  int *pnOut
+){
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
+  return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut);
+}
+
+/*
+** xInstToken() API implemenetation.
+*/
+static int fts5ApiInstToken(
+  Fts5Context *pCtx,
+  int iIdx, 
+  int iToken,
+  const char **ppOut, int *pnOut
+){
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
+  int rc = SQLITE_OK;
+  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 
+   || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) 
+  ){
+    if( iIdx<0 || iIdx>=pCsr->nInstCount ){
+      rc = SQLITE_RANGE;
+    }else{
+      int iPhrase = pCsr->aInst[iIdx*3];
+      int iCol = pCsr->aInst[iIdx*3 + 1];
+      int iOff = pCsr->aInst[iIdx*3 + 2];
+      i64 iRowid = fts5CursorRowid(pCsr);
+      rc = sqlite3Fts5ExprInstToken(
+          pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut
+      );
+    }
+  }
+  return rc;
+}
+

 static int fts5ApiQueryPhrase(Fts5Context*, int, void*, 
    int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
 );

 static const Fts5ExtensionApi sFts5Api = {
-  2,                            /* iVersion */
+  3,                            /* iVersion */
  fts5ApiUserData,
  fts5ApiColumnCount,
  fts5ApiRowCount,
@@ -2349,6 +2399,8 @@ static const Fts5ExtensionApi sFts5Api = {
  fts5ApiPhraseNext,
  fts5ApiPhraseFirstColumn,
  fts5ApiPhraseNextColumn,
+  fts5ApiQueryToken,
+  fts5ApiInstToken
 };

 /*
@@ -2615,9 +2667,7 @@ static int fts5RenameMethod(
 ){
  int rc;
  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
-  pTab->bInSavepoint = 1;
  rc = sqlite3Fts5StorageRename(pTab->pStorage, zName);
-  pTab->bInSavepoint = 0;
  return rc;
 }

@@ -2634,26 +2684,12 @@ int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
 static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
  int rc = SQLITE_OK;
-  char *zSql = 0;
+
  fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
-
-  if( pTab->bInSavepoint==0 ){
-    zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')",
-        pTab->p.pConfig->zDb, pTab->p.pConfig->zName, pTab->p.pConfig->zName
-    );
-    if( zSql ){
-      pTab->bInSavepoint = 1;
-      rc = sqlite3_exec(pTab->p.pConfig->db, zSql, 0, 0, 0);
-      pTab->bInSavepoint = 0;
-      sqlite3_free(zSql);
-    }else{
-      rc = SQLITE_NOMEM;
-    }
-    if( rc==SQLITE_OK ){
-      pTab->iSavepoint = iSavepoint+1;
-    }
+  rc = sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
+  if( rc==SQLITE_OK ){
+    pTab->iSavepoint = iSavepoint+1;
  }
-
  return rc;
 }

@@ -2914,7 +2950,7 @@ static int fts5ShadowName(const char *zName){
 ** if anything is found amiss.  Return a NULL pointer if everything is
 ** OK.
 */
-static int fts5Integrity(
+static int fts5IntegrityMethod(
  sqlite3_vtab *pVtab,    /* the FTS5 virtual table to check */
  const char *zSchema,    /* Name of schema in which this table lives */
  const char *zTabname,   /* Name of the table itself */
@@ -2972,7 +3008,7 @@ static int fts5Init(sqlite3 *db){
    /* xRelease      */ fts5ReleaseMethod,
    /* xRollbackTo   */ fts5RollbackToMethod,
    /* xShadowName   */ fts5ShadowName,
-    /* xIntegrity    */ fts5Integrity
+    /* xIntegrity    */ fts5IntegrityMethod
  };

  int rc;
@@ -244,6 +244,9 @@ static int SQLITE_TCLAPI xF5tApi(
    { "xGetAuxdataInt",    1, "CLEAR" },              /* 15 */
    { "xPhraseForeach",    4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */
    { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */
+
+    { "xQueryToken",       2, "IPHRASE ITERM" },      /* 18 */
+    { "xInstToken",        2, "IDX ITERM" },          /* 19 */
    { 0, 0, 0}
  };

@@ -500,6 +503,38 @@ static int SQLITE_TCLAPI xF5tApi(
      break;
    }

+    CASE(18, "xQueryToken") {
+      const char *pTerm = 0;
+      int nTerm = 0;
+      int iPhrase = 0;
+      int iTerm = 0;
+
+      if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
+      if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
+      rc = p->pApi->xQueryToken(p->pFts, iPhrase, iTerm, &pTerm, &nTerm);
+      if( rc==SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
+      }
+
+      break;
+    }
+
+    CASE(19, "xInstToken") {
+      const char *pTerm = 0;
+      int nTerm = 0;
+      int iIdx = 0;
+      int iTerm = 0;
+
+      if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ) return TCL_ERROR;
+      if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
+      rc = p->pApi->xInstToken(p->pFts, iIdx, iTerm, &pTerm, &nTerm);
+      if( rc==SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
+      }
+
+      break;
+    }
+
    default: 
      assert( 0 );
      break;
@@ -1117,6 +1152,176 @@ static int SQLITE_TCLAPI f5tRegisterTok(
  return TCL_OK;
 }

+typedef struct OriginTextCtx OriginTextCtx;
+struct OriginTextCtx {
+  sqlite3 *db;
+  fts5_api *pApi;
+};
+
+typedef struct OriginTextTokenizer OriginTextTokenizer;
+struct OriginTextTokenizer {
+  Fts5Tokenizer *pTok;            /* Underlying tokenizer object */
+  fts5_tokenizer tokapi;          /* API implementation for pTok */
+};
+
+/*
+** Delete the OriginTextCtx object indicated by the only argument.
+*/
+static void f5tOrigintextTokenizerDelete(void *pCtx){
+  OriginTextCtx *p = (OriginTextCtx*)pCtx;
+  ckfree(p);
+}
+
+static int f5tOrigintextCreate(
+  void *pCtx, 
+  const char **azArg, 
+  int nArg, 
+  Fts5Tokenizer **ppOut
+){
+  OriginTextCtx *p = (OriginTextCtx*)pCtx;
+  OriginTextTokenizer *pTok = 0;
+  void *pTokCtx = 0;
+  int rc = SQLITE_OK;
+
+  pTok = (OriginTextTokenizer*)sqlite3_malloc(sizeof(OriginTextTokenizer));
+  if( pTok==0 ){
+    rc = SQLITE_NOMEM;
+  }else if( nArg<1 ){
+    rc = SQLITE_ERROR;
+  }else{
+    /* Locate the underlying tokenizer */
+    rc = p->pApi->xFindTokenizer(p->pApi, azArg[0], &pTokCtx, &pTok->tokapi);
+  }
+
+  /* Create the new tokenizer instance */
+  if( rc==SQLITE_OK ){
+    rc = pTok->tokapi.xCreate(pTokCtx, &azArg[1], nArg-1, &pTok->pTok);
+  }
+
+  if( rc!=SQLITE_OK ){
+    sqlite3_free(pTok);
+    pTok = 0;
+  }
+  *ppOut = (Fts5Tokenizer*)pTok;
+  return rc;
+}
+
+static void f5tOrigintextDelete(Fts5Tokenizer *pTokenizer){
+  OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
+  if( p->pTok ){
+    p->tokapi.xDelete(p->pTok);
+  }
+  sqlite3_free(p);
+}
+
+typedef struct OriginTextCb OriginTextCb;
+struct OriginTextCb {
+  void *pCtx;
+  const char *pText; 
+  int nText;
+  int (*xToken)(void *, int, const char *, int, int, int);
+
+  char *aBuf;                     /* Buffer to use */
+  int nBuf;                       /* Allocated size of aBuf[] */
+};
+
+static int xOriginToken(
+  void *pCtx,                     /* Copy of 2nd argument to xTokenize() */
+  int tflags,                     /* Mask of FTS5_TOKEN_* flags */
+  const char *pToken,             /* Pointer to buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStart,                     /* Byte offset of token within input text */
+  int iEnd                        /* Byte offset of end of token within input */
+){
+  OriginTextCb *p = (OriginTextCb*)pCtx;
+  int ret = 0;
+
+  if( nToken==(iEnd-iStart) && 0==memcmp(pToken, &p->pText[iStart], nToken) ){
+    /* Token exactly matches document text. Pass it through as is. */
+    ret = p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd);
+  }else{
+    int nReq = nToken + 1 + (iEnd-iStart);
+    if( nReq>p->nBuf ){
+      sqlite3_free(p->aBuf);
+      p->aBuf = sqlite3_malloc(nReq*2);
+      if( p->aBuf==0 ) return SQLITE_NOMEM;
+      p->nBuf = nReq*2;
+    }
+
+    memcpy(p->aBuf, pToken, nToken);
+    p->aBuf[nToken] = '\0';
+    memcpy(&p->aBuf[nToken+1], &p->pText[iStart], iEnd-iStart);
+    ret = p->xToken(p->pCtx, tflags, p->aBuf, nReq, iStart, iEnd);
+  }
+
+  return ret;
+}
+
+
+static int f5tOrigintextTokenize(
+  Fts5Tokenizer *pTokenizer, 
+  void *pCtx,
+  int flags,                      /* Mask of FTS5_TOKENIZE_* flags */
+  const char *pText, int nText, 
+  int (*xToken)(void *, int, const char *, int, int, int)
+){
+  OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
+  OriginTextCb cb;
+  int ret;
+
+  memset(&cb, 0, sizeof(cb));
+  cb.pCtx = pCtx;
+  cb.pText = pText;
+  cb.nText = nText;
+  cb.xToken = xToken;
+
+  ret = p->tokapi.xTokenize(p->pTok,(void*)&cb,flags,pText,nText,xOriginToken);
+  sqlite3_free(cb.aBuf);
+  return ret;
+}
+
+/*
+**      sqlite3_fts5_register_origintext DB
+**
+** Description...
+*/
+static int SQLITE_TCLAPI f5tRegisterOriginText(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3 *db = 0;
+  fts5_api *pApi = 0;
+  int rc;
+  fts5_tokenizer tok = {0, 0, 0};
+  OriginTextCtx *pCtx = 0;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }
+  if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR;
+
+  pCtx = (OriginTextCtx*)ckalloc(sizeof(OriginTextCtx));
+  pCtx->db = db;
+  pCtx->pApi = pApi;
+
+  tok.xCreate = f5tOrigintextCreate;
+  tok.xDelete = f5tOrigintextDelete;
+  tok.xTokenize = f5tOrigintextTokenize;
+  rc = pApi->xCreateTokenizer(
+      pApi, "origintext", (void*)pCtx, &tok, f5tOrigintextTokenizerDelete
+  );
+
+  Tcl_ResetResult(interp);
+  if( rc!=SQLITE_OK ){
+    Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
+    return TCL_ERROR;
+  }
+  return TCL_OK;
+}
+
 /*
 ** Entry point.
 */
@@ -1133,7 +1338,8 @@ int Fts5tcl_Init(Tcl_Interp *interp){
    { "sqlite3_fts5_may_be_corrupt",     f5tMayBeCorrupt, 0 },
    { "sqlite3_fts5_token_hash",         f5tTokenHash, 0 },
    { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 },
-    { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 }
+    { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 },
+    { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 }
  };
  int i;
  F5tTokenizerContext *pContext;
@@ -229,6 +229,12 @@ static const unsigned char sqlite3Utf8Trans1[] = {

 #endif /* ifndef SQLITE_AMALGAMATION */

+#define FTS5_SKIP_UTF8(zIn) {                               \
+  if( ((unsigned char)(*(zIn++)))>=0xc0 ){                              \
+    while( (((unsigned char)*zIn) & 0xc0)==0x80 ){ zIn++; }             \
+  }                                                    \
+}
+
 typedef struct Unicode61Tokenizer Unicode61Tokenizer;
 struct Unicode61Tokenizer {
  unsigned char aTokenChar[128];  /* ASCII range token characters */
@@ -1264,6 +1270,7 @@ static int fts5PorterTokenize(
 typedef struct TrigramTokenizer TrigramTokenizer;
 struct TrigramTokenizer {
  int bFold;                      /* True to fold to lower-case */
+  int iFoldParam;                 /* Parameter to pass to Fts5UnicodeFold() */
 };

 /*
@@ -1290,6 +1297,7 @@ static int fts5TriCreate(
  }else{
    int i;
    pNew->bFold = 1;
+    pNew->iFoldParam = 0;
    for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
      const char *zArg = azArg[i+1];
      if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
@@ -1298,10 +1306,21 @@ static int fts5TriCreate(
        }else{
          pNew->bFold = (zArg[0]=='0');
        }
+      }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
+        if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
+          rc = SQLITE_ERROR;
+        }else{
+          pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
+        }
      }else{
        rc = SQLITE_ERROR;
      }
    }
+
+    if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
+      rc = SQLITE_ERROR;
+    }
+
    if( rc!=SQLITE_OK ){
      fts5TriDelete((Fts5Tokenizer*)pNew);
      pNew = 0;
@@ -1324,40 +1343,62 @@ static int fts5TriTokenize(
  TrigramTokenizer *p = (TrigramTokenizer*)pTok;
  int rc = SQLITE_OK;
  char aBuf[32];
+  char *zOut = aBuf;
+  int ii;
  const unsigned char *zIn = (const unsigned char*)pText;
  const unsigned char *zEof = &zIn[nText];
  u32 iCode;
+  int aStart[3];                  /* Input offset of each character in aBuf[] */

  UNUSED_PARAM(unusedFlags);
-  while( 1 ){
-    char *zOut = aBuf;
-    int iStart = zIn - (const unsigned char*)pText;
-    const unsigned char *zNext; 

-    READ_UTF8(zIn, zEof, iCode);
-    if( iCode==0 ) break;
-    zNext = zIn;
-    if( zIn<zEof ){
-      if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
-      WRITE_UTF8(zOut, iCode);
+  /* Populate aBuf[] with the characters for the first trigram. */
+  for(ii=0; ii<3; ii++){
+    do {
+      aStart[ii] = zIn - (const unsigned char*)pText;
+      READ_UTF8(zIn, zEof, iCode);
+      if( iCode==0 ) return SQLITE_OK;
+      if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
+    }while( iCode==0 );
+    WRITE_UTF8(zOut, iCode);
+  }
+
+  /* At the start of each iteration of this loop:
+  **
+  **  aBuf:      Contains 3 characters. The 3 characters of the next trigram.
+  **  zOut:      Points to the byte following the last character in aBuf.
+  **  aStart[3]: Contains the byte offset in the input text corresponding
+  **             to the start of each of the three characters in the buffer.
+  */
+  assert( zIn<=zEof );
+  while( 1 ){
+    int iNext;                    /* Start of character following current tri */
+    const char *z1;
+
+    /* Read characters from the input up until the first non-diacritic */
+    do {
+      iNext = zIn - (const unsigned char*)pText;
      READ_UTF8(zIn, zEof, iCode);
      if( iCode==0 ) break;
-    }else{
-      break;
-    }
-    if( zIn<zEof ){
-      if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
-      WRITE_UTF8(zOut, iCode);
-      READ_UTF8(zIn, zEof, iCode);
-      if( iCode==0 ) break;
-      if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
-      WRITE_UTF8(zOut, iCode);
-    }else{
-      break;
-    }
-    rc = xToken(pCtx, 0, aBuf, zOut-aBuf, iStart, iStart + zOut-aBuf);
-    if( rc!=SQLITE_OK ) break;
-    zIn = zNext;
+      if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
+    }while( iCode==0 );
+
+    /* Pass the current trigram back to fts5 */
+    rc = xToken(pCtx, 0, aBuf, zOut-aBuf, aStart[0], iNext);
+    if( iCode==0 || rc!=SQLITE_OK ) break;
+
+    /* Remove the first character from buffer aBuf[]. Append the character
+    ** with codepoint iCode.  */
+    z1 = aBuf;
+    FTS5_SKIP_UTF8(z1);
+    memmove(aBuf, z1, zOut - z1);
+    zOut -= (z1 - aBuf);
+    WRITE_UTF8(zOut, iCode);
+
+    /* Update the aStart[] array */
+    aStart[0] = aStart[1];
+    aStart[1] = aStart[2];
+    aStart[2] = iNext;
  }

  return rc;
@@ -1380,7 +1421,9 @@ int sqlite3Fts5TokenizerPattern(
 ){
  if( xCreate==fts5TriCreate ){
    TrigramTokenizer *p = (TrigramTokenizer*)pTok;
-    return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB;
+    if( p->iFoldParam==0 ){
+      return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB;
+    }
  }
  return FTS5_PATTERN_NONE;
 }
@@ -629,7 +629,7 @@ static int fts5VocabFilterMethod(
  if( pEq ){
    zTerm = (const char *)sqlite3_value_text(pEq);
    nTerm = sqlite3_value_bytes(pEq);
-    f = 0;
+    f = FTS5INDEX_QUERY_NOTOKENDATA;
  }else{
    if( pGe ){
      zTerm = (const char *)sqlite3_value_text(pGe);
@@ -438,6 +438,20 @@ proc detail_is_none {} { detail_check ; expr {$::detail == "none"} }
 proc detail_is_col {}  { detail_check ; expr {$::detail == "col" } }
 proc detail_is_full {} { detail_check ; expr {$::detail == "full"} }

+proc foreach_tokenizer_mode {prefix script} {
+  set saved $::testprefix
+  foreach {d mapping} {
+    ""              {}
+    "-origintext"   {, tokenize="origintext unicode61", tokendata=1}
+  } {
+    set s [string map [list %TOKENIZER% $mapping] $script]
+    set ::testprefix "$prefix$d"
+    reset_db
+    sqlite3_fts5_register_origintext db
+    uplevel $s
+  }
+  set ::testprefix $saved
+}

 #-------------------------------------------------------------------------
 # Convert a poslist of the type returned by fts5_test_poslist() to a 
@@ -22,6 +22,7 @@ ifcapable !fts5 {
 }

 foreach_detail_mode $::testprefix {
+foreach_tokenizer_mode $::testprefix {

 do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
@@ -44,7 +45,7 @@ do_execsql_test 1.1 {
 #

 do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
 }
 do_execsql_test 2.1 {
  INSERT INTO t1 VALUES('a b c', 'd e f');
@@ -73,8 +74,9 @@ do_execsql_test 2.4 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 3.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
 }
 foreach {i x y} {
   1  {g f d b f} {h h e i a}
@@ -97,8 +99,9 @@ foreach {i x y} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 4.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }
 foreach {i x y} {
@@ -121,8 +124,9 @@ foreach {i x y} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 5.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }
 foreach {i x y} {
@@ -145,8 +149,9 @@ foreach {i x y} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 6.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }

@@ -181,6 +186,7 @@ do_execsql_test 6.6 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 expr srand(0)
 do_execsql_test 7.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x,y,z);
@@ -222,6 +228,7 @@ for {set i 1} {$i <= 10} {incr i} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 8.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3");
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
@@ -236,6 +243,7 @@ do_execsql_test 8.1 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db

 expr srand(0)

@@ -280,8 +288,9 @@ for {set i 1} {$i <= 10} {incr i} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 10.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
 }
 set d10 {
   1  {g f d b f} {h h e i a}
@@ -314,19 +323,19 @@ do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
 #-------------------------------------------------------------------------
 #
 do_catchsql_test 11.1 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL% %TOKENIZER%);
 } {1 {reserved fts5 column name: rank}}
 do_catchsql_test 11.2 {
-  CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL% %TOKENIZER%);
 } {1 {reserved fts5 table name: rank}}
 do_catchsql_test 11.3 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL% %TOKENIZER%);
 } {1 {reserved fts5 column name: rowid}}

 #-------------------------------------------------------------------------
 #
 do_execsql_test 12.1 {
-  CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
 } {}

 do_catchsql_test 12.2 {
@@ -341,8 +350,9 @@ do_test 12.3 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 13.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
 } {}

@@ -365,8 +375,9 @@ do_execsql_test 13.6 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 14.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  WITH d(x,y) AS (
    SELECT NULL, 'xyz xyz xyz xyz xyz xyz'
@@ -449,8 +460,9 @@ do_catchsql_test 16.2 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 17.1 {
-  CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO b2 VALUES('a');
  INSERT INTO b2 VALUES('b');
  INSERT INTO b2 VALUES('c');
@@ -466,8 +478,9 @@ do_test 17.2 {

 if {[string match n* %DETAIL%]==0} {
  reset_db
+  sqlite3_fts5_register_origintext db
  do_execsql_test 17.3 {
-    CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%);
+    CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
    INSERT INTO c2 VALUES('x x x', 'x x x');
    SELECT rowid FROM c2 WHERE c2 MATCH 'y:x';
  } {1}
@@ -476,8 +489,9 @@ if {[string match n* %DETAIL%]==0} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 17.1 {
-  CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO uio VALUES(NULL);
  INSERT INTO uio SELECT NULL FROM uio;
  INSERT INTO uio SELECT NULL FROM uio;
@@ -524,8 +538,8 @@ do_execsql_test 17.9 {
 #--------------------------------------------------------------------
 #
 do_execsql_test 18.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
-  CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1 VALUES('abc*', NULL);
  INSERT INTO t2 VALUES(1, 'abcdefg');
 }
@@ -540,8 +554,9 @@ do_execsql_test 18.3 {
 # fts5 table in the temp schema.
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 19.0 {
-  CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1 VALUES('x y z');
  INSERT INTO t1 VALUES('w x 1');
  SELECT rowid FROM t1 WHERE t1 MATCH 'x';
@@ -551,8 +566,9 @@ do_execsql_test 19.0 {
 # Test that 6 and 7 byte varints can be read.
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 20.0 {
-  CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL% %TOKENIZER%);
 }
 set ::ids [list \
  0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43]
@@ -570,7 +586,7 @@ do_test 20.1 {
 #
 do_execsql_test 21.0 {
  CREATE TEMP TABLE t8(a, b);
-  CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL% %TOKENIZER%);
 }

 do_execsql_test 21.1 {
@@ -581,7 +597,7 @@ do_execsql_test 21.1 {
 }

 do_execsql_test 22.0 {
-  CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t9(rowid, x) VALUES(2, 'bbb');
  BEGIN;
    INSERT INTO t9(rowid, x) VALUES(1, 'aaa');
@@ -596,7 +612,7 @@ do_execsql_test 22.1 {

 #-------------------------------------------------------------------------
 do_execsql_test 23.0 {
-  CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  CREATE TABLE t11(x);
 }
 do_execsql_test 23.1 {
@@ -608,7 +624,7 @@ do_execsql_test 23.2 {

 #-------------------------------------------------------------------------
 do_execsql_test 24.0 {
-  CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t12 VALUES('aaaa');
 }
 do_execsql_test 24.1 {
@@ -618,6 +634,9 @@ do_execsql_test 24.1 {
    INSERT INTO t12 VALUES('aaaa');
  END;
 }
+execsql_pp {
+  SELECT rowid, hex(block) FROM t12_data
+}
 do_execsql_test 24.2 {
  INSERT INTO t12(t12) VALUES('integrity-check');
 }
@@ -627,7 +646,7 @@ do_execsql_test 24.3 {

 #-------------------------------------------------------------------------
 do_execsql_test 25.0 {
-  CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
 }
 do_execsql_test 25.1 {
  BEGIN;
@@ -638,6 +657,7 @@ SELECT * FROM t13('BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
 }


+}
 }

 expand_all_sql db
@@ -966,6 +966,262 @@ do_catchsql_test 6.2 {
  UPDATE t1 SET content=randomblob(500) WHERE t1;
 } {1 {constraint failed}}

+#-------------------------------------------------------------------------
+reset_db
+do_test 7.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+.open --hexdb
+| size 40960 pagesize 4096 filename crash-d8b4a99207c10b.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04   ................
+|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     96: 00 00 00 00 0d 00 00 00 0d 0b 62 00 0f 97 0f 40   ..........b....@
+|    112: 0e d5 0e 75 0e 18 0d c0 0d 66 0d 0f 0c a4 0c 44   ...u.....f.....D
+|    128: 0b ec 0b a7 0b 62 00 00 00 00 00 00 00 00 00 00   .....b..........
+|   2912: 00 00 43 0d 06 17 11 11 08 75 74 61 62 6c 65 74   ..C......utablet
+|   2928: 34 74 34 43 52 45 41 54 45 20 56 49 52 54 55 41   4t4CREATE VIRTUA
+|   2944: 4c 20 54 41 42 4c 45 20 74 34 20 55 53 49 4e 47   L TABLE t4 USING
+|   2960: 20 66 74 73 35 76 6f 63 61 62 28 27 74 32 27 2c    fts5vocab('t2',
+|   2976: 20 27 72 6f 77 27 29 43 0c 06 17 11 11 08 75 74    'row')C......ut
+|   2992: 61 62 6c 65 74 33 74 33 43 52 45 41 54 45 20 56   ablet3t3CREATE V
+|   3008: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 33 20   IRTUAL TABLE t3 
+|   3024: 55 53 49 4e 47 20 66 74 73 35 76 6f 63 61 62 28   USING fts5vocab(
+|   3040: 27 74 31 27 2c 20 27 72 6f 77 27 29 56 0b 06 17   't1', 'row')V...
+|   3056: 1f 1f 01 7d 74 61 62 6c 65 74 32 5f 63 6f 6e 66   ....tablet2_conf
+|   3072: 69 67 74 32 5f 63 6f 6e 66 69 67 0a 43 52 45 41   igt2_config.CREA
+|   3088: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 63 6f 6e   TE TABLE 't2_con
+|   3104: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b   fig'(k PRIMARY K
+|   3120: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52   EY, v) WITHOUT R
+|   3136: 4f 57 49 44 5e 0a 07 17 21 21 01 81 07 74 61 62   OWID^...!!...tab
+|   3152: 6c 65 74 32 5f 63 6f 6e 74 65 6e 74 74 32 5f 63   let2_contentt2_c
+|   3168: 6f 6e 74 65 6e 74 09 43 52 45 41 54 45 20 54 41   ontent.CREATE TA
+|   3184: 42 4c 45 20 27 74 32 5f 63 6f 6e 74 65 6e 74 27   BLE 't2_content'
+|   3200: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d   (id INTEGER PRIM
+|   3216: 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c   ARY KEY, c0, c1,
+|   3232: 20 63 32 29 69 09 07 17 19 19 01 81 2d 74 61 62    c2)i.......-tab
+|   3248: 6c 65 74 32 5f 69 64 78 74 32 5f 69 64 78 08 43   let2_idxt2_idx.C
+|   3264: 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 32 5f   REATE TABLE 't2_
+|   3280: 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d   idx'(segid, term
+|   3296: 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20   , pgno, PRIMARY 
+|   3312: 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29   KEY(segid, term)
+|   3328: 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55   ) WITHOUT ROWIDU
+|   3344: 08 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 32 5f   ........tablet2_
+|   3360: 64 61 74 61 74 32 5f 64 61 74 61 07 43 52 45 41   datat2_data.CREA
+|   3376: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 64 61 74   TE TABLE 't2_dat
+|   3392: 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52   a'(id INTEGER PR
+|   3408: 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b   IMARY KEY, block
+|   3424: 20 42 4c 4f 42 29 58 07 07 17 11 11 08 81 1d 74    BLOB)X........t
+|   3440: 61 62 6c 65 74 32 74 32 43 52 45 41 54 45 20 56   ablet2t2CREATE V
+|   3456: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 32 20   IRTUAL TABLE t2 
+|   3472: 55 53 49 4e 47 20 66 74 73 35 28 27 61 27 2c 5b   USING fts5('a',[
+|   3488: 62 5d 2c 22 63 22 2c 64 65 74 61 69 6c 3d 6e 6f   b],.c.,detail=no
+|   3504: 6e 65 2c 63 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29   ne,columnsize=0)
+|   3520: 56 06 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f   V.......tablet1_
+|   3536: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 06   configt1_config.
+|   3552: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
+|   3568: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41   _config'(k PRIMA
+|   3584: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f   RY KEY, v) WITHO
+|   3600: 55 54 20 52 4f 57 49 44 5b 05 07 17 21 21 01 81   UT ROWID[...!!..
+|   3616: 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65   .tablet1_docsize
+|   3632: 74 31 5f 64 6f 63 73 69 7a 65 05 43 52 45 41 54   t1_docsize.CREAT
+|   3648: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73   E TABLE 't1_docs
+|   3664: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20   ize'(id INTEGER 
+|   3680: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20   PRIMARY KEY, sz 
+|   3696: 42 4c 4f 42 29 5e 04 07 17 21 21 01 81 07 74 61   BLOB)^...!!...ta
+|   3712: 62 6c 65 74 31 5f 63 6f 6e 74 65 6e 74 74 31 5f   blet1_contentt1_
+|   3728: 63 6f 6e 74 65 6e 74 04 43 52 45 41 54 45 20 54   content.CREATE T
+|   3744: 41 42 4c 45 20 27 74 31 5f 63 6f 6e 74 65 6e 74   ABLE 't1_content
+|   3760: 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49   '(id INTEGER PRI
+|   3776: 4d 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31   MARY KEY, c0, c1
+|   3792: 2c 20 63 32 29 69 03 07 17 19 19 01 81 2d 74 61   , c2)i.......-ta
+|   3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03   blet1_idxt1_idx.
+|   3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
+|   3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72   _idx'(segid, ter
+|   3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59   m, pgno, PRIMARY
+|   3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d    KEY(segid, term
+|   3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44   )) WITHOUT ROWID
+|   3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31   U........tablet1
+|   3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45   _datat1_data.CRE
+|   3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61   ATE TABLE 't1_da
+|   3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50   ta'(id INTEGER P
+|   3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63   RIMARY KEY, bloc
+|   3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b   k BLOB)g.......;
+|   4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20   tablet1t1CREATE 
+|   4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31   VIRTUAL TABLE t1
+|   4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 62 20    USING fts5(a,b 
+|   4048: 75 6e 69 6e 64 65 78 65 64 2c 63 2c 74 6f 6b 65   unindexed,c,toke
+|   4064: 6e 69 7a 65 3d 22 70 6f 72 74 65 72 20 61 73 63   nize=.porter asc
+|   4080: 69 69 22 2c 74 6f 6b 65 6e 64 61 74 61 3d 31 29   ii.,tokendata=1)
+| page 2 offset 4096
+|      0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c   ..h............|
+|     16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00   .*..............
+|   3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00   .......0........
+|   3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80   ..........7.....
+|   3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01   ...t.....0a.....
+|   3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01   b.....c.....g...
+|   3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01   ....h.......i...
+|   3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00   ............*...
+|   3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80   ............%...
+|   3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02   .....P.....0g...
+|   3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02   .....h........i.
+|   4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 7f f1 03   ........7.......
+|   4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01   .t.....0a.....b.
+|   4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02   ....c.....g.....
+|   4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 04   ..h.......i.....
+|   4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00   ................
+|   4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
+| page 3 offset 8192
+|      0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c   ................
+|   4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02   ................
+| page 4 offset 12288
+|      0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00   ................
+|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
+|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
+|   4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69    h i.......g h i
+|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
+|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
+| page 5 offset 16384
+|      0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03   ................
+|   4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03   ................
+| page 6 offset 20480
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| page 7 offset 24576
+|      0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00   ................
+|   3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84   ..............A.
+|   4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61   ............4.0a
+|   4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01   .....b.....c....
+|   4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01   .d...e...f...g..
+|   4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06   ...h.....i......
+|   4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f   ................
+|   4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
+| page 8 offset 28672
+|      0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
+| page 9 offset 32768
+|      0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00   ................
+|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
+|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
+|   4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69    h i.......g h i
+|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
+|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
+| page 10 offset 36864
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| end crash-d8b4a99207c10b.db
+}]} {}
+
+do_catchsql_test 7.1 {
+  SELECT snippet(t1, -1, '.', '..', '[', ']'), 
+         highlight(t1, 2, '[', ']') 
+           FROM t1('g + h') 
+           WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank;
+} {1 {database disk image is malformed}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_test 8.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+.open --hexdb
+| size 20480 pagesize 4096 filename crash-d57c01958e48ab.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 05   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 04   ................
+|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     96: 00 00 00 00 0d 00 00 00 05 0e 10 00 0f 97 0f 40   ...............@
+|    112: 0e d5 0e 68 0e 10 01 00 00 00 00 00 00 00 00 00   ...h............
+|   3600: 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f   V.......tablet1_
+|   3616: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 05   configt1_config.
+|   3632: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
+|   3648: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41   _config'(k PRIMA
+|   3664: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f   RY KEY, v) WITHO
+|   3680: 55 54 20 52 4f 57 49 44 6b 04 07 17 21 21 01 81   UT ROWIDk...!!..
+|   3696: 21 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65   !tablet1_docsize
+|   3712: 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 45 41 54   t1_docsize.CREAT
+|   3728: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73   E TABLE 't1_docs
+|   3744: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20   ize'(id INTEGER 
+|   3760: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20   PRIMARY KEY, sz 
+|   3776: 42 4c 4f 42 2c 20 6f 72 69 67 69 6e 20 49 4e 54   BLOB, origin INT
+|   3792: 45 47 45 52 29 69 03 07 17 19 19 01 81 2d 74 61   EGER)i.......-ta
+|   3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03   blet1_idxt1_idx.
+|   3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
+|   3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72   _idx'(segid, ter
+|   3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59   m, pgno, PRIMARY
+|   3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d    KEY(segid, term
+|   3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44   )) WITHOUT ROWID
+|   3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31   U........tablet1
+|   3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45   _datat1_data.CRE
+|   3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61   ATE TABLE 't1_da
+|   3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50   ta'(id INTEGER P
+|   3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63   RIMARY KEY, bloc
+|   3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b   k BLOB)g.......;
+|   4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20   tablet1t1CREATE 
+|   4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31   VIRTUAL TABLE t1
+|   4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 20 62    USING fts5(a, b
+|   4048: 2c 20 63 6f 6e 74 65 6e 74 3d 27 27 2c 20 63 6f   , content='', co
+|   4064: 6e 74 65 6e 74 6c 65 73 73 5f 64 65 6c 65 74 65   ntentless_delete
+|   4080: 3d 31 2c 20 74 6f 6b 65 6e 64 61 74 61 3d 31 29   =1, tokendata=1)
+| page 2 offset 4096
+|      0: 0d 0f eb 00 03 0e 17 00 0f e2 0e 17 0e 31 00 00   .............1..
+|     16: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
+|   3600: 00 00 00 00 00 00 00 18 0a 03 00 36 00 00 00 00   ...........6....
+|   3616: ff 00 00 01 01 01 01 00 01 01 01 01 01 01 00 00   ................
+|   3632: 07 83 29 84 80 80 80 80 01 04 00 86 56 00 00 01   ..).........V...
+|   3648: 96 04 30 61 61 61 01 02 02 01 04 02 04 01 08 02   ..0aaa..........
+|   3664: 04 04 04 01 10 02 04 04 04 04 04 04 04 01 20 02   .............. .
+|   3680: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 01   ................
+|   3696: 40 02 04 04 04 04 04 04 04 04 04 04 04 04 04 04   @...............
+|   3712: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3728: 04 01 81 00 02 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3744: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3760: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3776: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3792: 04 04 04 04 02 02 62 63 01 06 01 01 02 01 03 62   ......bc.......b
+|   3808: 62 62 02 02 03 01 04 03 06 01 08 03 06 06 06 01   bb..............
+|   3824: 10 03 06 06 06 06 06 06 06 01 20 03 06 06 06 06   .......... .....
+|   3840: 06 06 06 06 06 06 06 06 06 06 06 01 40 03 06 06   ............@...
+|   3856: 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06   ................
+|   3872: 06 06 06 06 06 06 06 06 06 06 16 06 06 02 02 63   ...............c
+|   3888: 64 02 06 01 01 02 01 03 63 63 63 03 02 05 01 04   d.......ccc.....
+|   3904: 05 0a 01 08 05 0a 0a 0a 01 10 05 0a 0a 0a 0a 0a   ................
+|   3920: 0a 0a 01 20 05 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a   ... ............
+|   3936: 0a 0a 0a 0a 02 02 64 65 03 06 01 01 02 01 03 64   ......de.......d
+|   3952: 64 64 04 02 09 01 04 09 12 01 08 09 12 12 12 01   dd..............
+|   3968: 10 09 12 12 12 12 12 12 12 02 02 65 66 04 06 01   ...........ef...
+|   3984: 01 02 01 03 65 65 65 05 02 11 01 04 11 22 01 08   ....eee.........
+|   4000: 11 22 22 22 02 02 66 67 05 06 01 01 02 01 03 66   ......fg.......f
+|   4016: 56 66 06 02 21 01 04 21 42 02 02 67 68 06 06 01   Vf..!..!B..gh...
+|   4032: 01 02 cb 03 67 67 67 07 02 41 02 02 68 69 07 06   ....ggg..A..hi..
+|   4048: 01 01 02 04 81 13 09 50 09 2e 09 1c 09 12 09 0c   .......P........
+|   4064: 09 08 07 01 03 00 14 07 81 77 07 00 00 00 15 22   .........w......
+|   4080: 00 00 00 00 ff 00 00 01 00 00 00 00 00 00 05 0c   ................
+| page 3 offset 8192
+|      0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
+| page 4 offset 12288
+|      0: 0d 00 00 00 07 0f c8 00 0f f8 0f f0 0f e8 0f e0   ................
+|     16: 0f d8 0f d0 0f c8 00 00 00 00 00 00 00 00 00 00   ................
+|   4032: 00 00 00 00 00 00 00 00 06 07 04 00 10 09 7f 01   ................
+|   4048: 06 06 04 00 10 09 3f 01 06 05 04 00 10 09 1f 01   ......?.........
+|   4064: 06 04 04 00 10 09 0f 01 06 03 04 00 10 09 07 01   ................
+|   4080: 06 02 04 00 10 09 03 01 06 01 04 00 10 09 01 01   ................
+| page 5 offset 16384
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| end crash-d57c01958e48ab.db
+}]} {}
+
+do_catchsql_test 8.1 {
+  SELECT rowid FROM t1('a* NOT ý') ;
+} {0 {1 2 3 4 5 6 7}}
+
 sqlite3_fts5_may_be_corrupt 0
 finish_test

@@ -0,0 +1,141 @@
+# 2010 June 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultG
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+set ::testprefix fts5faultH
+
+sqlite3_fts5_register_origintext db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+
+  BEGIN;
+    INSERT INTO t1 VALUES('oNe tWo thRee');
+    INSERT INTO t1 VALUES('One Two Three');
+    INSERT INTO t1 VALUES('onE twO threE');
+  COMMIT;
+  BEGIN;
+    INSERT INTO t1 VALUES('one two three');
+    INSERT INTO t1 VALUES('one two three');
+    INSERT INTO t1 VALUES('one two three');
+  COMMIT;
+}
+
+do_faultsim_test 1 -faults oom* -prep { 
+} -body {
+  execsql {
+    SELECT rowid FROM t1('three');
+  }
+} -test {
+  faultsim_integrity_check
+  faultsim_test_result {0 {1 2 3 4 5 6}}
+}
+
+
+reset_db
+sqlite3_fts5_register_origintext db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+
+  BEGIN;
+    INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(12, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(13, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(14, 'bbb BBB bbb');
+    INSERT INTO t1(rowid, x) VALUES(15, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(16, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(17, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(18, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(19, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(20, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(21, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(22, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(23, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(24, 'aaa bbb BBB');
+  COMMIT;
+}
+
+do_faultsim_test 2 -faults oom* -prep { 
+} -body {
+  execsql {
+    SELECT rowid FROM t1('BBB AND AAA');
+  }
+} -test {
+  faultsim_integrity_check
+  faultsim_test_result {0 {10 24}}
+}
+
+reset_db
+sqlite3_fts5_register_origintext db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+
+  INSERT INTO t1(rowid, x) VALUES(9, 'bbb Bbb BBB');
+  BEGIN;
+    INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(11, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(12, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(13, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(14, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(15, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(16, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(17, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(18, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(19, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(20, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(21, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(22, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(23, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(24, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(25, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(26, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(27, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(28, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(29, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(30, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(31, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(32, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(33, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(34, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(35, 'aaa bbb BBB');
+  COMMIT;
+}
+
+do_faultsim_test 3 -faults oom* -prep { 
+} -body {
+  execsql {
+    SELECT rowid FROM t1('BBB AND AAA');
+  }
+} -test {
+  faultsim_integrity_check
+  faultsim_test_result {0 {10 35}}
+}
+
+
+finish_test
@@ -91,7 +91,6 @@ do_execsql_test 2.2.1 {
  INSERT INTO vt0(c0) VALUES ('xyz');
 }

-breakpoint
 do_execsql_test 2.2.2 {
  ALTER TABLE t0 RENAME TO t1;
 }
@@ -500,6 +499,21 @@ do_execsql_test 17.5 {
  SELECT c0 FROM t0 WHERE c0 GLOB '*faul*';
 } {assertionfaultproblem}

+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 18.0 {
+  BEGIN;
+    CREATE VIRTUAL TABLE t1 USING fts5(text);
+    ALTER TABLE t1 RENAME TO t2;
+}
+
+do_execsql_test 18.1 {
+    DROP TABLE t2;
+}
+
+do_execsql_test 18.2 {
+  COMMIT;
+}

 finish_test

@@ -0,0 +1,297 @@
+# 2014 Jan 08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", detail=%DETAIL%
+  );
+  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO ft VALUES('Hello world');
+}
+
+do_execsql_test 1.2 {
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+proc b {x} { string map [list "\0" "."] $x }
+db func b b
+
+do_execsql_test 1.3 {
+  select b(term) from vocab;
+} {
+  hello.Hello
+  world
+}
+
+do_execsql_test 1.4 {
+  SELECT rowid FROM ft('Hello');
+} {1}
+
+#-------------------------------------------------------------------------
+reset_db
+
+# Return a random integer between 0 and n-1.
+#
+proc random {n} {
+  expr {abs(int(rand()*$n))}
+}
+
+proc select_one {list} {
+  set n [llength $list]
+  lindex $list [random $n]
+}
+
+proc term {} {
+  set first_letter {
+    a b c d e f g h i j k l m n o p q r s t u v w x y z
+    A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
+  }
+
+  set term [select_one $first_letter]
+  append term [random 100]
+}
+
+proc document {} {
+  set nTerm [expr [random 5] + 5]
+  set doc ""
+  for {set ii 0} {$ii < $nTerm} {incr ii} {
+    lappend doc [term]
+  }
+  set doc
+}
+db func document document
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", detail=%DETAIL%
+  );
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 128);
+  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
+}
+
+do_test 2.1 {
+  for {set ii 0} {$ii < 500} {incr ii} {
+    execsql { INSERT INTO ft VALUES( document() ) }
+  }
+} {}
+
+do_execsql_test 2.2 {
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+do_execsql_test 2.3 {
+  INSERT INTO ft(ft, rank) VALUES('merge', 16);
+}
+
+do_execsql_test 2.4 {
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+do_execsql_test 2.5 {
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", detail=%DETAIL%
+  );
+  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
+
+  INSERT INTO ft(rowid, x) VALUES(1, 'hello');
+  INSERT INTO ft(rowid, x) VALUES(2, 'Hello');
+  INSERT INTO ft(rowid, x) VALUES(3, 'HELLO');
+}
+
+#proc b {x} { string map [list "\0" "."] $x }
+#db func b b
+#execsql_pp { SELECT b(term) FROM vocab }
+
+do_execsql_test 3.1.1 { SELECT rowid FROM ft('hello') } 1
+do_execsql_test 3.1.2 { SELECT rowid FROM ft('Hello') } 2
+do_execsql_test 3.1.3 { SELECT rowid FROM ft('HELLO') } 3
+
+do_execsql_test 3.2 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(x, 
+      tokenize="origintext unicode61", 
+      tokendata=1,
+      detail=%DETAIL%
+  );
+  CREATE VIRTUAL TABLE vocab2 USING fts5vocab(ft2, instance);
+
+  INSERT INTO ft2(rowid, x) VALUES(1, 'hello');
+  INSERT INTO ft2(rowid, x) VALUES(2, 'Hello');
+  INSERT INTO ft2(rowid, x) VALUES(3, 'HELLO');
+
+  INSERT INTO ft2(rowid, x) VALUES(10, 'helloooo');
+}
+
+#proc b {x} { string map [list "\0" "."] $x }
+#db func b b
+#execsql_pp { SELECT b(term) FROM vocab }
+
+do_execsql_test 3.3.1 { SELECT rowid FROM ft2('hello') } {1 2 3}
+do_execsql_test 3.3.2 { SELECT rowid FROM ft2('Hello') } {1 2 3}
+do_execsql_test 3.3.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3}
+
+do_execsql_test 3.3.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+sqlite3_fts5_register_origintext db
+proc querytoken {cmd iPhrase iToken} { 
+  set txt [$cmd xQueryToken $iPhrase $iToken]
+  string map [list "\0" "."] $txt
+}
+sqlite3_fts5_create_function db querytoken querytoken
+
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
+  );
+  INSERT INTO ft VALUES('one two three four');
+}
+
+do_execsql_test 4.1 {
+  SELECT rowid, querytoken(ft, 0, 0) FROM ft('TwO')
+} {1 two.TwO}
+do_execsql_test 4.2 {
+  SELECT rowid, querytoken(ft, 0, 0) FROM ft('one TWO ThreE')
+} {1 one}
+do_execsql_test 4.3 {
+  SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE')
+} {1 two.TWO}
+
+if {"%DETAIL%"=="full"} {
+  # Phrase queries are only supported for detail=full.
+  #
+  do_execsql_test 4.4 {
+    SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"')
+  } {1 three.ThreE}
+  do_catchsql_test 4.5 {
+    SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"')
+  } {1 SQLITE_RANGE}
+  do_catchsql_test 4.6 {
+    SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"')
+  } {1 SQLITE_RANGE}
+  do_catchsql_test 4.7 {
+    SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"')
+  } {1 SQLITE_RANGE}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+sqlite3_fts5_register_origintext db
+proc insttoken {cmd iIdx iToken} { 
+  set txt [$cmd xInstToken $iIdx $iToken]
+  string map [list "\0" "."] $txt
+}
+sqlite3_fts5_create_function db insttoken insttoken
+fts5_aux_test_functions db
+
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
+  );
+  INSERT INTO ft VALUES('one ONE One oNe oNE one');
+}
+
+do_execsql_test 5.1 {
+  SELECT insttoken(ft, 0, 0), 
+         insttoken(ft, 1, 0),
+         insttoken(ft, 2, 0),
+         insttoken(ft, 3, 0),
+         insttoken(ft, 4, 0),
+         insttoken(ft, 5, 0)
+  FROM ft('one');
+} {
+  one one.ONE one.One one.oNe one.oNE one
+}
+
+do_execsql_test 5.2 {
+  SELECT insttoken(ft, 1, 0) FROM ft('one');
+} {
+  one.ONE
+}
+
+do_execsql_test 5.3 {
+  SELECT fts5_test_poslist(ft) FROM ft('one');
+} {
+  {0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5}
+}
+
+#-------------------------------------------------------------------------
+# Test the xInstToken() API with:
+#
+#   * a non tokendata=1 table.
+#   * prefix queries.
+#
+reset_db
+sqlite3_fts5_register_origintext db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, y, tokenize='origintext unicode61', detail=%DETAIL%
+  );
+
+  INSERT INTO ft VALUES('One Two', 'Three two');
+  INSERT INTO ft VALUES('three Three', 'one One');
+}
+proc tokens {cmd} { 
+  set ret [list]
+  for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+    set txt [$cmd xInstToken $iTok 0]
+    set txt [string map [list "\0" "."] $txt]
+    lappend ret $txt
+  }
+  set ret
+}
+sqlite3_fts5_create_function db tokens tokens
+
+do_execsql_test 6.1 {
+  SELECT rowid, tokens(ft) FROM ft('One');
+} {1 one.One 2 one.One}
+
+do_execsql_test 6.2 {
+  SELECT rowid, tokens(ft) FROM ft('on*');
+} {1 {{}} 2 {{} {}}}
+
+do_execsql_test 6.3 {
+  SELECT rowid, tokens(ft) FROM ft('Three*');
+} {1 {{}} 2 {{}}}
+
+}
+
+finish_test
+
@@ -0,0 +1,146 @@
+# 2014 Jan 08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+}
+
+do_execsql_test 1.1 {
+  BEGIN;
+  INSERT INTO ft VALUES('Hello');
+  INSERT INTO ft VALUES('hello');
+  INSERT INTO ft VALUES('HELLO');
+  INSERT INTO ft VALUES('today');
+  INSERT INTO ft VALUES('today');
+  INSERT INTO ft VALUES('today');
+  INSERT INTO ft VALUES('World');
+  INSERT INTO ft VALUES('world');
+  INSERT INTO ft VALUES('WORLD');
+  COMMIT;
+}
+
+do_execsql_test 1.2 { SELECT rowid FROM ft('hello'); } {1 2 3}
+do_execsql_test 1.3 { SELECT rowid FROM ft('today'); } {4 5 6}
+do_execsql_test 1.4 { SELECT rowid FROM ft('world'); } {7 8 9}
+
+do_execsql_test 1.5 {
+  SELECT count(*) FROM ft_data
+} 3
+
+do_execsql_test 1.6 {
+  DELETE FROM ft;
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
+  BEGIN;
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+    )
+    INSERT INTO ft SELECT 'Hello Hello Hello Hello Hello Hello Hello' FROM s;
+    INSERT INTO ft VALUES ('hELLO hELLO hELLO');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('World World World World World World World');
+    INSERT INTO ft VALUES('world world world world world world world');
+    INSERT INTO ft VALUES('WORLD WORLD WORLD WORLD WORLD WORLD WORLD');
+    INSERT INTO ft VALUES('World World World World World World World');
+    INSERT INTO ft VALUES('world world world world world world world');
+    INSERT INTO ft VALUES('WORLD WORLD WORLD WORLD WORLD WORLD WORLD');
+  COMMIT;
+}
+
+do_execsql_test 1.7 {
+  SELECT count(*) FROM ft_data;
+} 23
+
+do_execsql_test 1.8 { SELECT rowid FROM ft('hello') WHERE rowid>100; } {101}
+
+do_execsql_test 1.9 {
+  DELETE FROM ft;
+  INSERT INTO ft(ft) VALUES('optimize');
+  SELECT count(*) FROM ft_data;
+} {2}
+do_execsql_test 1.10 {
+  BEGIN;
+    INSERT INTO ft VALUES('Hello');
+    INSERT INTO ft VALUES('hello');
+    INSERT INTO ft VALUES('HELLO');
+    INSERT INTO ft VALUES('today');
+    INSERT INTO ft VALUES('today');
+    INSERT INTO ft VALUES('today');
+    INSERT INTO ft VALUES('World');
+    INSERT INTO ft VALUES('world');
+    INSERT INTO ft VALUES('WORLD');
+}
+
+do_execsql_test 1.11 { SELECT rowid FROM ft('hello'); } {1 2 3}
+do_execsql_test 1.12 { SELECT rowid FROM ft('today'); } {4 5 6}
+do_execsql_test 1.13 { SELECT rowid FROM ft('world'); } {7 8 9}
+do_execsql_test 1.14 { SELECT rowid FROM ft('hello') ORDER BY rank; } {1 2 3}
+
+#------------------------------------------------------------------------
+reset_db
+sqlite3_fts5_register_origintext db
+proc tokens {cmd} { 
+  set ret [list]
+  for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+    set txt [$cmd xInstToken $iTok 0]
+    set txt [string map [list "\0" "."] $txt]
+    lappend ret $txt
+  }
+  set ret
+}
+sqlite3_fts5_create_function db tokens tokens
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(
+    v, tokenize="origintext unicode61", tokendata=1, detail=none
+  );
+
+  INSERT INTO x1 VALUES('xxx Xxx XXX yyy YYY yyy');
+  INSERT INTO x1 VALUES('xxx yyy xxx yyy yyy yyy');
+}
+
+do_execsql_test 2.1 {
+  SELECT tokens(x1) FROM x1('xxx');
+} {
+  {xxx xxx.Xxx xxx.XXX} {xxx xxx}
+}
+
+do_execsql_test 2.2 {
+  UPDATE x1_content SET c0 = 'xxx xxX xxx yyy yyy yyy' WHERE id=1;
+}
+
+do_execsql_test 2.3 {
+  SELECT tokens(x1) FROM x1('xxx');
+} {
+  {xxx {} xxx} {xxx xxx}
+}
+
+finish_test
+
@@ -0,0 +1,101 @@
+# 2023 November 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext3
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+  reset_db
+
+  sqlite3_fts5_register_origintext db
+  fts5_aux_test_functions db
+  proc insttoken {cmd iIdx iToken} { 
+    set txt [$cmd xInstToken $iIdx $iToken]
+    string map [list "\0" "."] $txt
+  }
+  sqlite3_fts5_create_function db insttoken insttoken
+  
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE ft USING fts5(
+        x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
+    );
+  }
+  
+  do_execsql_test 1.1 {
+    INSERT INTO ft VALUES('Hello world HELLO WORLD hello');
+  }
+  
+  do_execsql_test 1.2 {
+    SELECT fts5_test_poslist(ft) FROM ft('hello');
+  } {{0.0.0 0.0.2 0.0.4}}
+
+  do_execsql_test 1.3 {
+    SELECT 
+      insttoken(ft, 0, 0),
+      insttoken(ft, 1, 0),
+      insttoken(ft, 2, 0)
+    FROM ft('hello');
+  } {hello.Hello hello.HELLO hello}
+
+  do_execsql_test 1.4 {
+    SELECT 
+      insttoken(ft, 0, 0),
+      insttoken(ft, 1, 0),
+      insttoken(ft, 2, 0)
+    FROM ft('hello') ORDER BY rank;
+  } {hello.Hello hello.HELLO hello}
+
+  do_execsql_test 1.5 {
+    CREATE VIRTUAL TABLE ft2 USING fts5(
+        x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
+    );
+    INSERT INTO ft2(rowid, x) VALUES(1, 'ONE one two three ONE');
+    INSERT INTO ft2(rowid, x) VALUES(2, 'TWO one two three TWO');
+    INSERT INTO ft2(rowid, x) VALUES(3, 'THREE one two three THREE');
+  }
+
+  do_execsql_test 1.6 {
+    SELECT insttoken(ft2, 0, 0), rowid FROM ft2('three') ORDER BY rank;
+  } {three.THREE 3 three 1 three 2}
+
+  do_execsql_test 1.7 {
+    INSERT INTO ft2(rowid, x) VALUES(10, 'aaa bbb BBB');
+    INSERT INTO ft2(rowid, x) VALUES(12, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(13, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(14, 'bbb BBB bbb');
+    INSERT INTO ft2(rowid, x) VALUES(15, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(16, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(17, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(18, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(19, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(20, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(21, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(22, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(23, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(24, 'aaa bbb BBB');
+  }
+
+  do_execsql_test 1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24}
+  do_execsql_test 1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24}
+
+}
+
+finish_test
+
@@ -0,0 +1,66 @@
+# 2023 November 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext4
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+}
+
+do_execsql_test 1.1 {
+  BEGIN;
+    INSERT INTO ft SELECT 'the first thing';
+
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<90000
+    )
+    INSERT INTO ft SELECT 'The second thing' FROM s;
+
+    INSERT INTO ft SELECT 'the first thing';
+  COMMIT;
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+
+foreach {tn sql expr} {
+  1 { SELECT rowid FROM ft('the') }       {$mem > 250000}
+  2 { SELECT rowid FROM ft('first') }     {$mem <  50000}
+  3 { SELECT rowid FROM ft('the first') } {$mem <  50000}
+} {
+  db close
+  sqlite3 db test.db
+  sqlite3_fts5_register_origintext db
+
+  execsql $sql
+  do_test 1.2.$tn {
+    set mem [lindex [sqlite3_db_status db CACHE_USED 0] 1]
+    expr $expr
+  } 1
+}
+
+proc b {x} { string map [list "\0" "."] $x }
+db func b b
+# execsql_pp { SELECT segid, b(term), pgno from ft_idx }
+
+finish_test
+
@@ -0,0 +1,273 @@
+# 2023 Dec 04
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests for tables that use both tokendata=1 and contentless_delete=1.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+# Return a random integer between 0 and n-1.
+#
+proc random {n} { expr {abs(int(rand()*$n))} }
+
+# Select an element of the list passed as the only argument at random and
+# return it. 
+#
+proc select_one {list} {
+  set n [llength $list]
+  lindex $list [random $n]
+}
+
+# Given a term that consists entirely of alphabet characters, return all
+# permutations of the term using upper and lower case characters. e.g.
+#
+#    "abc" -> {CBA cBA CbA cbA CBa cBa Cba cba}
+#
+proc casify {term {lRet {{}}}} {
+  if {$term==""} { return $lRet }
+  set t [string range $term 1 end]
+  set f1 [string toupper [string range $term 0 0]]
+  set f2 [string tolower [string range $term 0 0]]
+  set ret [list]
+  foreach x $lRet {
+    lappend ret "$x$f1"
+    lappend ret "$x$f2"
+  }
+  return [casify $t $ret]
+}
+
+proc vocab {} {
+  list abc def ghi jkl mno pqr stu vwx yza
+}
+
+# Return a random 3 letter term.
+#
+proc term {} {
+  if {[info exists ::expanded_vocab]==0} {
+    foreach v [vocab] { lappend ::expanded_vocab {*}[casify $v] }
+  }
+
+  select_one $::expanded_vocab
+}
+
+# Return a document - between 3 and 10 terms.
+#
+proc document {} {
+  set nTerm [expr [random 3] + 7]
+  set doc ""
+  for {set ii 0} {$ii < $nTerm} {incr ii} {
+    lappend doc [term]
+  }
+  set doc
+}
+db func document document
+
+#-------------------------------------------------------------------------
+
+expr srand(6)
+
+set NDOC  200
+set NLOOP 50
+
+sqlite3_fts5_register_origintext db
+
+proc tokens {cmd} { 
+  set ret [list]
+  for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+    set txt [$cmd xInstToken $iTok 0]
+    set txt [string map [list "\0" "."] $txt]
+    lappend ret $txt
+  }
+  set ret
+}
+sqlite3_fts5_create_function db tokens tokens
+
+proc rankfunc {cmd} { 
+  $cmd xRowid
+}
+sqlite3_fts5_create_function db rankfunc rankfunc
+
+proc ctrl_tokens {term args} {
+  set ret [list]
+  set term [string tolower $term]
+  foreach doc $args {
+    foreach a $doc {
+      if {[string tolower $a]==$term} {
+        if {$a==$term} {
+          lappend ret $a
+        } else {
+          lappend ret [string tolower $a].$a
+        }
+      }
+    }
+  }
+  set ret
+}
+db func ctrl_tokens ctrl_tokens
+
+proc do_all_vocab_test {tn} {
+  foreach ::v [concat [vocab] nnn] {
+    set answer [execsql {
+      SELECT id, ctrl_tokens($::v, x) FROM ctrl WHERE x LIKE '%' || $::v || '%'
+    }]
+    do_execsql_test $tn.$::v.1 {
+      SELECT rowid, tokens(ft) FROM ft($::v)
+    } $answer
+    do_execsql_test $tn.$::v.2 {
+      SELECT rowid, tokens(ft) FROM ft($::v) ORDER BY rank
+    } $answer
+  }
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", content=, contentless_delete=1,
+      tokendata=1
+  );
+
+  CREATE TABLE ctrl(id INTEGER PRIMARY KEY, x TEXT);
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
+  INSERT INTO ft(ft, rank) VALUES('rank', 'rankfunc()');
+}
+do_test 1.1 {
+  for {set ii 0} {$ii < $NDOC} {incr ii} {
+    set doc [document]
+    execsql {
+      INSERT INTO ft(rowid, x) VALUES($ii, $doc);
+      INSERT INTO ctrl(id, x) VALUES($ii, $doc);
+    }
+  }
+} {}
+
+#execsql_pp { SELECT * FROM ctrl }
+#execsql_pp { SELECT * FROM ft }
+#fts5_aux_test_functions db
+#execsql_pp { SELECT rowid, tokens(ft), fts5_test_poslist(ft) FROM ft('ghi'); }
+
+do_all_vocab_test 1.2
+
+for {set ii 0} {$ii < $NLOOP} {incr ii} {
+  set lRowid [execsql { SELECT id FROM ctrl WHERE random() % 2 }]
+  foreach r $lRowid {
+    execsql { DELETE FROM ft WHERE rowid = $r }
+    execsql { DELETE FROM ctrl WHERE rowid = $r }
+
+    set doc [document]
+    execsql { INSERT INTO ft(rowid, x) VALUES($r, $doc) }
+    execsql { INSERT INTO ctrl(id, x) VALUES($r, $doc) }
+  }
+  do_all_vocab_test 1.3.$ii
+}
+
+#-------------------------------------------------------------------------
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(
+      x, y, tokenize="origintext unicode61", content=, contentless_delete=1,
+      tokendata=1
+  );
+
+  CREATE TABLE ctrl2(id INTEGER PRIMARY KEY, x TEXT, y TEXT);
+  INSERT INTO ft2(ft2, rank) VALUES('pgsz', 64);
+  INSERT INTO ft2(ft2, rank) VALUES('rank', 'rankfunc()');
+}
+do_test 2.1 {
+  for {set ii 0} {$ii < $NDOC} {incr ii} {
+    set doc1 [document]
+    set doc2 [document]
+    execsql {
+      INSERT INTO ft2(rowid, x, y) VALUES($ii, $doc, $doc2);
+      INSERT INTO ctrl2(id, x, y) VALUES($ii, $doc, $doc2);
+    }
+  }
+} {}
+
+proc do_all_vocab_test2 {tn} {
+  foreach ::v [vocab] {
+    set answer [execsql {
+      SELECT id, ctrl_tokens($::v, x, y) FROM ctrl2
+      WHERE x LIKE '%' || $::v || '%' OR y LIKE '%' || $::v || '%';
+    }]
+    do_execsql_test $tn.$::v.1 {
+      SELECT rowid, tokens(ft2) FROM ft2($::v)
+    } $answer
+    do_execsql_test $tn.$::v.2 {
+      SELECT rowid, tokens(ft2) FROM ft2($::v) ORDER BY rank
+    } $answer
+  }
+}
+
+do_all_vocab_test2 2.2
+
+for {set ii 0} {$ii < $NLOOP} {incr ii} {
+  set lRowid [execsql { SELECT id FROM ctrl2 WHERE random() % 2 }]
+  foreach r $lRowid {
+    execsql { DELETE FROM ft2 WHERE rowid = $r }
+    execsql { DELETE FROM ctrl2 WHERE rowid = $r }
+
+    set doc1 [document]
+    set doc2 [document]
+    execsql { INSERT INTO ft2(rowid, x, y) VALUES($r, $doc, $doc1) }
+    execsql { INSERT INTO ctrl2(id, x, y) VALUES($r, $doc, $doc2) }
+  }
+  do_all_vocab_test 2.3.$ii
+}
+
+#-------------------------------------------------------------------------
+
+unset -nocomplain ::expanded_vocab
+proc vocab {} {
+  list abcde fghij klmno
+}
+
+proc do_all_vocab_test3 {tn} {
+  foreach ::v [concat [vocab] nnn] {
+    set answer [execsql {
+      SELECT rowid, ctrl_tokens($::v, w) FROM ctrl3 WHERE w LIKE '%' || $::v || '%'
+    }]
+    do_execsql_test $tn.$::v.1 {
+      SELECT rowid, tokens(ft3) FROM ft3($::v)
+    } $answer
+    do_execsql_test $tn.$::v.2 {
+      SELECT rowid, tokens(ft3) FROM ft3($::v) ORDER BY rank
+    } $answer
+  }
+}
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft3 USING fts5(
+      w, tokenize="origintext unicode61", content=, contentless_delete=1,
+      tokendata=1
+  );
+  INSERT INTO ft3(ft3, rank) VALUES('rank', 'rankfunc()');
+  CREATE TABLE ctrl3(w);
+}
+
+do_execsql_test 3.1 {
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<2
+  )
+  INSERT INTO ctrl3 SELECT document() FROM s;
+  INSERT INTO ft3(rowid, w) SELECT rowid, w FROM ctrl3;
+}
+
+do_all_vocab_test3 3.2
+
+
+finish_test
+
@@ -343,7 +343,9 @@ do_execsql_test 17.0 {
    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
  COMMIT;
 }
-do_execsql_test 17.1 { SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 }
+do_execsql_test 17.1 { 
+  SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 
+}
 do_execsql_test 17.2 {
  BEGIN;
    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
@@ -0,0 +1,89 @@
+# 2023 Nov 03
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focusing on the built-in fts5 tokenizers. 
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5tokenizer2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+sqlite3_fts5_create_tokenizer db tst get_tst_tokenizer
+proc get_tst_tokenizer {args} {
+  return "tst_tokenizer"
+}
+proc tst_tokenizer {flags txt} {
+  set token ""
+  set lTok [list]
+
+  foreach c [split $txt {}] {
+    if {$token==""} {
+      append token $c
+    } else {
+      set t1 [string is upper $token]
+      set t2 [string is upper $c]
+
+      if {$t1!=$t2} {
+        lappend lTok $token
+        set token ""
+      }
+      append token $c
+    }
+  }
+  if {$token!=""} { lappend lTok $token }
+
+  set iOff 0
+  foreach t $lTok {
+    set n [string length $t]
+    sqlite3_fts5_token $t $iOff [expr $iOff+$n]
+    incr iOff $n
+  }
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(t, tokenize=tst);
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO t1 VALUES('AAdontBBmess');
+}
+
+do_execsql_test 1.2 {
+  SELECT snippet(t1, 0, '>', '<', '...', 4) FROM t1('BB');
+} {AAdont>BB<mess}
+
+do_execsql_test 1.3 {
+  SELECT highlight(t1, 0, '>', '<') FROM t1('BB');
+} {AAdont>BB<mess}
+
+do_execsql_test 1.4 {
+  SELECT highlight(t1, 0, '>', '<') FROM t1('AA');
+} {>AA<dontBBmess}
+
+do_execsql_test 1.5 {
+  SELECT highlight(t1, 0, '>', '<') FROM t1('dont');
+} {AA>dont<BBmess}
+
+do_execsql_test 1.6 {
+  SELECT highlight(t1, 0, '>', '<') FROM t1('mess');
+} {AAdontBB>mess<}
+
+do_execsql_test 1.7 {
+  SELECT highlight(t1, 0, '>', '<') FROM t1('BB mess');
+} {AAdont>BBmess<}
+
+
+finish_test
@@ -0,0 +1,109 @@
+# 2023 October 24
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# Tests for the fts5 "trigram" tokenizer.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5trigram2
+
+do_execsql_test 1.0 "
+  CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize='trigram remove_diacritics 1');
+  INSERT INTO t1 VALUES('abc\u0303defghijklm');
+  INSERT INTO t1 VALUES('a\u0303b\u0303c\u0303defghijklm');
+"
+
+do_execsql_test 1.1 {
+  SELECT highlight(t1, 0, '(', ')') FROM t1('abc');
+} [list \
+  "(abc\u0303)defghijklm"                          \
+  "(a\u0303b\u0303c\u0303)defghijklm"              \
+]
+
+do_execsql_test 1.2 {
+  SELECT highlight(t1, 0, '(', ')') FROM t1('bcde');
+} [list \
+  "a(bc\u0303de)fghijklm"                          \
+  "a\u0303(b\u0303c\u0303de)fghijklm"              \
+]
+
+do_execsql_test 1.3 {
+  SELECT highlight(t1, 0, '(', ')') FROM t1('cdef');
+} [list \
+  "ab(c\u0303def)ghijklm"                          \
+  "a\u0303b\u0303(c\u0303def)ghijklm"              \
+]
+
+do_execsql_test 1.4 {
+  SELECT highlight(t1, 0, '(', ')') FROM t1('def');
+} [list \
+  "abc\u0303(def)ghijklm"                          \
+  "a\u0303b\u0303c\u0303(def)ghijklm"              \
+]
+
+
+#-------------------------------------------------------------------------
+do_catchsql_test 2.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(
+      z, tokenize='trigram case_sensitive 1 remove_diacritics 1'
+  );
+} {1 {error in tokenizer constructor}}
+
+do_execsql_test 2.1 {
+  CREATE VIRTUAL TABLE t2 USING fts5(
+      z, tokenize='trigram case_sensitive 0 remove_diacritics 1'
+  );
+}
+do_execsql_test 2.2 "
+  INSERT INTO t2 VALUES('\u00E3bcdef');
+  INSERT INTO t2 VALUES('b\u00E3cdef');
+  INSERT INTO t2 VALUES('bc\u00E3def');
+  INSERT INTO t2 VALUES('bcd\u00E3ef');
+"
+
+do_execsql_test 2.3 {
+  SELECT highlight(t2, 0, '(', ')') FROM t2('abc');
+} "(\u00E3bc)def"
+do_execsql_test 2.4 {
+  SELECT highlight(t2, 0, '(', ')') FROM t2('bac');
+} "(b\u00E3c)def"
+do_execsql_test 2.5 {
+  SELECT highlight(t2, 0, '(', ')') FROM t2('bca');
+} "(bc\u00E3)def"
+do_execsql_test 2.6 "
+  SELECT highlight(t2, 0, '(', ')') FROM t2('\u00E3bc');
+" "(\u00E3bc)def"
+
+#-------------------------------------------------------------------------
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t3 USING fts5(
+      z, tokenize='trigram remove_diacritics 1'
+  );
+} {}
+do_execsql_test 3.1 "
+  INSERT INTO t3 VALUES ('\u0303abc\u0303');
+"
+do_execsql_test 3.2 {
+  SELECT highlight(t3, 0, '(', ')') FROM t3('abc');
+} "\u0303(abc\u0303)"
+
+#-------------------------------------------------------------------------
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t4 USING fts5(z, tokenize=trigram);
+} {}
+
+breakpoint
+do_execsql_test 4.1 {
+  INSERT INTO t4 VALUES('ABCD');
+} {}
+
+finish_test
@@ -280,6 +280,30 @@ do_catchsql_test 5.2 {
  INSERT INTO t1 SELECT randomblob(3000) FROM v1
 } {1 {query aborted}}

+#-------------------------------------------------------------------------
+reset_db
+sqlite3_fts5_may_be_corrupt 1
+
+do_execsql_test 6.0 {
+  BEGIN TRANSACTION;
+    CREATE VIRTUAL TABLE t1 USING fts5(a,b unindexed,c,tokenize="porter ascii",tokendata=1);
+    REPLACE INTO t1_data VALUES(1,X'03090009');
+    REPLACE INTO t1_data VALUES(10,X'000000000103030003010101020101030101');
+    REPLACE INTO t1_data VALUES(137438953473,X'0000002e023061010202010162010203010163010204010167010601020201016801060102030101690106010204040606060808');
+    REPLACE INTO t1_data VALUES(274877906945,X'0000001f013067020802010202010168020803010203010169020804010204040909');
+    REPLACE INTO t1_data VALUES(412316860417,X'0000002e023061030202010162030203010163030204010167030601020201016803060102030101690306010204040606060808');
+  COMMIT;
+}
+
+do_execsql_test 6.1 {
+  CREATE VIRTUAL TABLE t3 USING fts5vocab('t1', 'row');
+}
+
+do_catchsql_test 6.2 {
+  SELECT * FROM t3;
+} {1 {database disk image is malformed}}
+
+sqlite3_fts5_may_be_corrupt 0

 finish_test

@@ -35,6 +35,8 @@ $(dir.bld.c):

 javac.flags ?= -Xlint:unchecked -Xlint:deprecation
 java.flags ?=
+javac.flags += -encoding utf8
+# -------------^^^^^^^^^^^^^^ required for Windows builds
 jnicheck ?= 1
 ifeq (1,$(jnicheck))
  java.flags += -Xcheck:jni
@@ -79,6 +81,7 @@ $(bin.version-info): $(dir.tool)/version-info.c $(sqlite3.h) $(dir.top)/Makefile
 # Be explicit about which Java files to compile so that we can work on
 # in-progress files without requiring them to be in a compilable statae.
 JAVA_FILES.main := $(patsubst %,$(dir.src.jni)/annotation/%,\
+  Experimental.java \
  NotNull.java \
  Nullable.java \
 ) $(patsubst %,$(dir.src.capi)/%,\
@@ -91,7 +94,7 @@ JAVA_FILES.main := $(patsubst %,$(dir.src.jni)/annotation/%,\
  CollationNeededCallback.java \
  CommitHookCallback.java \
  ConfigLogCallback.java \
-  ConfigSqllogCallback.java \
+  ConfigSqlLogCallback.java \
  NativePointerHolder.java \
  OutputPointer.java \
  PrepareMultiCallback.java \
@@ -110,6 +113,7 @@ JAVA_FILES.main := $(patsubst %,$(dir.src.jni)/annotation/%,\
  WindowFunction.java \
  XDestroyCallback.java \
  sqlite3.java \
+  sqlite3_blob.java \
  sqlite3_context.java \
  sqlite3_stmt.java \
  sqlite3_value.java \
@@ -120,6 +124,7 @@ JAVA_FILES.main := $(patsubst %,$(dir.src.jni)/annotation/%,\
  Sqlite.java \
  SqliteException.java \
  ValueHolder.java \
+  WindowFunction.java \
 )

 JAVA_FILES.unittest := $(patsubst %,$(dir.src.jni)/%,\
@@ -159,12 +164,13 @@ endif
 CLASS_FILES :=
 define CLASSFILE_DEPS
 all: $(1).class
+$(1).class: $(1).java
 CLASS_FILES += $(1).class
 endef
 $(foreach B,$(basename \
  $(JAVA_FILES.main) $(JAVA_FILES.unittest) $(JAVA_FILES.tester)),\
  $(eval $(call CLASSFILE_DEPS,$(B))))
-$(CLASS_FILES): $(JAVA_FILES) $(MAKEFILE)
+$(CLASS_FILES): $(MAKEFILE)
 	$(bin.javac) $(javac.flags) -h $(dir.bld.c) -cp $(classpath) $(JAVA_FILES)

 #.PHONY: classfiles
@@ -226,7 +232,8 @@ SQLITE_OPT += -DSQLITE_ENABLE_RTREE \
  -DSQLITE_ENABLE_OFFSET_SQL_FUNC \
  -DSQLITE_ENABLE_PREUPDATE_HOOK \
  -DSQLITE_ENABLE_NORMALIZE \
-  -DSQLITE_ENABLE_SQLLOG
+  -DSQLITE_ENABLE_SQLLOG \
+  -DSQLITE_ENABLE_COLUMN_METADATA
 endif

 ifeq (1,$(opt.debug))
@@ -316,7 +323,7 @@ test-one: $(test.deps)
 	$(bin.java) $(test.flags.jvm) org.sqlite.jni.wrapper1.Tester2 $(Tester2.flags)
 test-sqllog: $(test.deps)
 	@echo "Testing with -sqllog..."
-	$(bin.java) $(test.flags.jvm) -sqllog
+	$(bin.java) $(test.flags.jvm) org.sqlite.jni.capi.Tester1 $(Tester1.flags) -sqllog
 test-mt: $(test.deps)
 	@echo "Testing in multi-threaded mode:";
 	$(bin.java) $(test.flags.jvm) org.sqlite.jni.capi.Tester1 \
@@ -16,9 +16,8 @@ Technical support is available in the forum:
 > **FOREWARNING:** this subproject is very much in development and
  subject to any number of changes. Please do not rely on any
  information about its API until this disclaimer is removed.  The JNI
-  bindings released with version 3.43 are a "tech preview" and 3.44
-  will be "final," at which point strong backward compatibility
-  guarantees will apply.
+  bindings released with version 3.43 are a "tech preview." Once
+  finalized, strong backward compatibility guarantees will apply.

 Project goals/requirements:

@@ -43,11 +42,13 @@ Non-goals:
 - Creation of high-level OO wrapper APIs. Clients are free to create
  them off of the C-style API.

+- Virtual tables are unlikely to be supported due to the amount of
+  glue code needed to fit them into Java.
+
 - Support for mixed-mode operation, where client code accesses SQLite
  both via the Java-side API and the C API via their own native
-  code. In such cases, proxy functionalities (primarily callback
-  handler wrappers of all sorts) may fail because the C-side use of
-  the SQLite APIs will bypass those proxies.
+  code. Such cases would be a minefield of potential mis-interactions
+  between this project's JNI bindings and mixed-mode client code.


 Hello World
@@ -123,15 +124,13 @@ sensible default argument values. In all such cases they are thin
 proxies around the corresponding C APIs and do not introduce new
 semantics.

-In some very few cases, Java-specific capabilities have been added in
+In a few cases, Java-specific capabilities have been added in
 new APIs, all of which have "_java" somewhere in their names.
 Examples include:

 - `sqlite3_result_java_object()`
 - `sqlite3_column_java_object()`
- `sqlite3_column_java_casted()`
 - `sqlite3_value_java_object()`
- `sqlite3_value_java_casted()`

 which, as one might surmise, collectively enable the passing of
 arbitrary Java objects from user-defined SQL functions through to the
@@ -150,6 +149,9 @@ pending statements have been closed. Be aware that Java garbage
 collection _cannot_ close a database or finalize a prepared statement.
 Those things require explicit API calls.

+Classes for which it is sensible support Java's `AutoCloseable`
+interface so can be used with try-with-resources constructs.
+

 Golden Rule #2: _Never_ Throw from Callbacks (Unless...)
 ------------------------------------------------------------------------
@@ -159,14 +161,15 @@ retain C-like semantics. For example, they are not permitted to throw
 or propagate exceptions and must return error information (if any) via
 result codes or `null`. The only cases where the C-style APIs may
 throw is through client-side misuse, e.g. passing in a null where it
-shouldn't be used. The APIs clearly mark function parameters which
-should not be null, but does not actively defend itself against such
-misuse. Some C-style APIs explicitly accept `null` as a no-op for
-usability's sake, and some of the JNI APIs deliberately return an
-error code, instead of segfaulting, when passed a `null`.
+may cause a `NullPointerException`. The APIs clearly mark function
+parameters which should not be null, but does not generally actively
+defend itself against such misuse. Some C-style APIs explicitly accept
+`null` as a no-op for usability's sake, and some of the JNI APIs
+deliberately return an error code, instead of segfaulting, when passed
+a `null`.

 Client-defined callbacks _must never throw exceptions_ unless _very
-explicitly documented_ as being throw-safe. Exceptions are generally
+explitly documented_ as being throw-safe. Exceptions are generally
 reserved for higher-level bindings which are constructed to
 specifically deal with them and ensure that they do not leak C-level
 resources. In some cases, callback handlers are permitted to throw, in
@@ -292,14 +295,14 @@ int sqlite3_create_function(sqlite3 db, String funcName, int nArgs,
 `SQLFunction` is not used directly, but is instead instantiated via
 one of its three subclasses:

- `SQLFunction.Scalar` implements simple scalar functions using but a
+- `ScalarFunction` implements simple scalar functions using but a
  single callback.
- `SQLFunction.Aggregate` implements aggregate functions using two
+- `AggregateFunction` implements aggregate functions using two
  callbacks.
- `SQLFunction.Window` implements window functions using four
+- `WindowFunction` implements window functions using four
  callbacks.

-Search [`Tester1.java`](/file/ext/jni/src/org/sqlite/jni/Tester1.java) for
+Search [`Tester1.java`](/file/ext/jni/src/org/sqlite/jni/capi/Tester1.java) for
 `SQLFunction` for how it's used.

 Reminder: see the disclaimer at the top of this document regarding the
@@ -427,8 +427,6 @@ extern "C" {
 #define org_sqlite_jni_capi_CApi_SQLITE_OPEN_EXRESCODE 33554432L
 #undef org_sqlite_jni_capi_CApi_SQLITE_PREPARE_PERSISTENT
 #define org_sqlite_jni_capi_CApi_SQLITE_PREPARE_PERSISTENT 1L
-#undef org_sqlite_jni_capi_CApi_SQLITE_PREPARE_NORMALIZE
-#define org_sqlite_jni_capi_CApi_SQLITE_PREPARE_NORMALIZE 2L
 #undef org_sqlite_jni_capi_CApi_SQLITE_PREPARE_NO_VTAB
 #define org_sqlite_jni_capi_CApi_SQLITE_PREPARE_NO_VTAB 4L
 #undef org_sqlite_jni_capi_CApi_SQLITE_OK
@@ -707,8 +705,12 @@ extern "C" {
 #define org_sqlite_jni_capi_CApi_SQLITE_DETERMINISTIC 2048L
 #undef org_sqlite_jni_capi_CApi_SQLITE_DIRECTONLY
 #define org_sqlite_jni_capi_CApi_SQLITE_DIRECTONLY 524288L
+#undef org_sqlite_jni_capi_CApi_SQLITE_SUBTYPE
+#define org_sqlite_jni_capi_CApi_SQLITE_SUBTYPE 1048576L
 #undef org_sqlite_jni_capi_CApi_SQLITE_INNOCUOUS
 #define org_sqlite_jni_capi_CApi_SQLITE_INNOCUOUS 2097152L
+#undef org_sqlite_jni_capi_CApi_SQLITE_RESULT_SUBTYPE
+#define org_sqlite_jni_capi_CApi_SQLITE_RESULT_SUBTYPE 16777216L
 #undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_SCAN_UNIQUE
 #define org_sqlite_jni_capi_CApi_SQLITE_INDEX_SCAN_UNIQUE 1L
 #undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_EQ
@@ -775,6 +777,22 @@ JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_init
 JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1java_1uncache_1thread
  (JNIEnv *, jclass);

+/*
+ * Class:     org_sqlite_jni_capi_CApi
+ * Method:    sqlite3_jni_supports_nio
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1jni_1supports_1nio
+  (JNIEnv *, jclass);
+
+/*
+ * Class:     org_sqlite_jni_capi_CApi
+ * Method:    sqlite3_jni_db_error
+ * Signature: (Lorg/sqlite/jni/capi/sqlite3;ILjava/lang/String;)I
+ */
+JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1jni_1db_1error
+  (JNIEnv *, jclass, jobject, jint, jstring);
+
 /*
 * Class:     org_sqlite_jni_capi_CApi
 * Method:    sqlite3_aggregate_context
@@ -871,6 +889,14 @@ JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1int64
 JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1java_1object
  (JNIEnv *, jclass, jlong, jint, jobject);

+/*
+ * Class:     org_sqlite_jni_capi_CApi
+ * Method:    sqlite3_bind_nio_buffer
+ * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;ILjava/nio/ByteBuffer;II)I
+ */
+JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1nio_1buffer
+  (JNIEnv *, jclass, jobject, jint, jobject, jint, jint);
+
 /*
 * Class:     org_sqlite_jni_capi_CApi
 * Method:    sqlite3_bind_null
@@ -975,6 +1001,14 @@ JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1open
 JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1read
  (JNIEnv *, jclass, jlong, jbyteArray, jint);

+/*
+ * Class:     org_sqlite_jni_capi_CApi
+ * Method:    sqlite3_blob_read_nio_buffer
+ * Signature: (JILjava/nio/ByteBuffer;II)I
+ */
+JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1read_1nio_1buffer
+  (JNIEnv *, jclass, jlong, jint, jobject, jint, jint);
+
 /*
 * Class:     org_sqlite_jni_capi_CApi
 * Method:    sqlite3_blob_reopen
@@ -991,6 +1025,14 @@ JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1reopen
 JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1write
  (JNIEnv *, jclass, jlong, jbyteArray, jint);

+/*
+ * Class:     org_sqlite_jni_capi_CApi
+ * Method:    sqlite3_blob_write_nio_buffer
+ * Signature: (JILjava/nio/ByteBuffer;II)I
+ */
+JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1write_1nio_1buffer
+  (JNIEnv *, jclass, jlong, jint, jobject, jint, jint);
+
 /*
 * Class:     org_sqlite_jni_capi_CApi
 * Method:    sqlite3_busy_handler
@@ -1087,6 +1129,14 @@ JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1bytes16
 JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1count
  (JNIEnv *, jclass, jlong);

+/*
+ * Class:     org_sqlite_jni_capi_CApi
+ * Method:    sqlite3_column_database_name
+ * Signature: (JI)Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1database_1name
+  (JNIEnv *, jclass, jlong, jint);
+
 /*
 * Class:     org_sqlite_jni_capi_CApi
 * Method:    sqlite3_column_decltype
@@ -1119,6 +1169,14 @@ JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1int
 JNIEXPORT jlong JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1int64
  (JNIEnv *, jclass, jobject, jint);

+/*
+ * Class:     org_sqlite_jni_capi_CApi
+ * Method:    sqlite3_column_java_object
+ * Signature: (JI)Ljava/lang/Object;
+ */
+JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1java_1object
+  (JNIEnv *, jclass, jlong, jint);
+
 /*
 * Class:     org_sqlite_jni_capi_CApi
 * Method:    sqlite3_column_name
@@ -1129,11 +1187,11 @@ JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1name

 /*
 * Class:     org_sqlite_jni_capi_CApi
- * Method:    sqlite3_column_database_name
- * Signature: (JI)Ljava/lang/String;
+ * Method:    sqlite3_column_nio_buffer
+ * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)Ljava/nio/ByteBuffer;
 */
-JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1database_1name
-  (JNIEnv *, jclass, jlong, jint);
+JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1nio_1buffer
+  (JNIEnv *, jclass, jobject, jint);

 /*
 * Class:     org_sqlite_jni_capi_CApi
@@ -1225,26 +1283,26 @@ JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1complete

 /*
 * Class:     org_sqlite_jni_capi_CApi
- * Method:    sqlite3_config
+ * Method:    sqlite3_config__enable
 * Signature: (I)I
 */
-JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1config__I
+JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1config_1_1enable
  (JNIEnv *, jclass, jint);

 /*
 * Class:     org_sqlite_jni_capi_CApi
- * Method:    sqlite3_config
- * Signature: (Lorg/sqlite/jni/capi/ConfigSqllogCallback;)I
+ * Method:    sqlite3_config__CONFIG_LOG
+ * Signature: (Lorg/sqlite/jni/capi/ConfigLogCallback;)I
 */
-JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1config__Lorg_sqlite_jni_capi_ConfigSqllogCallback_2
+JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1config_1_1CONFIG_1LOG
  (JNIEnv *, jclass, jobject);

 /*
 * Class:     org_sqlite_jni_capi_CApi
- * Method:    sqlite3_config
- * Signature: (Lorg/sqlite/jni/capi/ConfigLogCallback;)I
+ * Method:    sqlite3_config__SQLLOG
+ * Signature: (Lorg/sqlite/jni/capi/ConfigSqlLogCallback;)I
 */
-JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1config__Lorg_sqlite_jni_capi_ConfigLogCallback_2
+JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1config_1_1SQLLOG
  (JNIEnv *, jclass, jobject);

 /*
@@ -1394,9 +1452,9 @@ JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1extended_1errcode
 /*
 * Class:     org_sqlite_jni_capi_CApi
 * Method:    sqlite3_extended_result_codes
- * Signature: (Lorg/sqlite/jni/capi/sqlite3;Z)Z
+ * Signature: (Lorg/sqlite/jni/capi/sqlite3;Z)I
 */
-JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1extended_1result_1codes
+JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1extended_1result_1codes
  (JNIEnv *, jclass, jobject, jboolean);

 /*
@@ -1679,14 +1737,6 @@ JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1error_1nom
 JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1error_1code
  (JNIEnv *, jclass, jobject, jint);

-/*
- * Class:     org_sqlite_jni_capi_CApi
- * Method:    sqlite3_result_null
- * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;)V
- */
-JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1null
-  (JNIEnv *, jclass, jobject);
-
 /*
 * Class:     org_sqlite_jni_capi_CApi
 * Method:    sqlite3_result_int
@@ -1711,6 +1761,30 @@ JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1int64
 JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1java_1object
  (JNIEnv *, jclass, jobject, jobject);

+/*
+ * Class:     org_sqlite_jni_capi_CApi
+ * Method:    sqlite3_result_nio_buffer
+ * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;Ljava/nio/ByteBuffer;II)V
+ */
+JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1nio_1buffer
+  (JNIEnv *, jclass, jobject, jobject, jint, jint);
+
+/*
+ * Class:     org_sqlite_jni_capi_CApi
+ * Method:    sqlite3_result_null
+ * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;)V
+ */
+JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1null
+  (JNIEnv *, jclass, jobject);
+
+/*
+ * Class:     org_sqlite_jni_capi_CApi
+ * Method:    sqlite3_result_subtype
+ * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;I)V
+ */
+JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1subtype
+  (JNIEnv *, jclass, jobject, jint);
+
 /*
 * Class:     org_sqlite_jni_capi_CApi
 * Method:    sqlite3_result_value
@@ -1850,10 +1924,10 @@ JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1status64
 /*
 * Class:     org_sqlite_jni_capi_CApi
 * Method:    sqlite3_step
- * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;)I
+ * Signature: (J)I
 */
 JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1step
-  (JNIEnv *, jclass, jobject);
+  (JNIEnv *, jclass, jlong);

 /*
 * Class:     org_sqlite_jni_capi_CApi
@@ -2063,6 +2137,14 @@ JNIEXPORT jlong JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1int64
 JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1java_1object
  (JNIEnv *, jclass, jlong);

+/*
+ * Class:     org_sqlite_jni_capi_CApi
+ * Method:    sqlite3_value_nio_buffer
+ * Signature: (Lorg/sqlite/jni/capi/sqlite3_value;)Ljava/nio/ByteBuffer;
+ */
+JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1nio_1buffer
+  (JNIEnv *, jclass, jobject);
+
 /*
 * Class:     org_sqlite_jni_capi_CApi
 * Method:    sqlite3_value_nochange
@@ -0,0 +1,30 @@
+/*
+** 2023-09-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file houses the Experimental annotation for the sqlite3 C API.
+*/
+package org.sqlite.jni.annotation;
+import java.lang.annotation.*;
+
+/**
+   This annotation is for flagging methods, constructors, and types
+   which are expressly experimental and subject to any amount of
+   change or outright removal. Client code should not rely on such
+   features.
+*/
+@Documented
+@Retention(RetentionPolicy.SOURCE)
+@Target({
+    ElementType.METHOD,
+    ElementType.CONSTRUCTOR,
+    ElementType.TYPE
+})
+public @interface Experimental{}
@@ -9,21 +9,22 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** This file houses the NotNull annotaion for the sqlite3 C API.
+** This file houses the NotNull annotation for the sqlite3 C API.
 */
 package org.sqlite.jni.annotation;
+import java.lang.annotation.*;

 /**
   This annotation is for flagging parameters which may not legally be
   null or point to closed/finalized C-side resources.

   <p>In the case of Java types which map directly to C struct types
-   (e.g. {@link org.sqlite.jni.sqlite3}, {@link
-   org.sqlite.jni.sqlite3_stmt}, and {@link
-   org.sqlite.jni.sqlite3_context}), a closed/finalized resource is
-   also considered to be null for purposes this annotation because the
-   C-side effect of passing such a handle is the same as if null is
-   passed.</p>
+   (e.g. {@link org.sqlite.jni.capi.sqlite3}, {@link
+   org.sqlite.jni.capi.sqlite3_stmt}, and {@link
+   org.sqlite.jni.capi.sqlite3_context}), a closed/finalized resource
+   is also considered to be null for purposes this annotation because
+   the C-side effect of passing such a handle is the same as if null
+   is passed.</p>

   <p>When used in the context of Java interfaces which are called
   from the C APIs, this annotation communicates that the C API will
@@ -31,12 +32,23 @@ package org.sqlite.jni.annotation;

   <p>Passing a null, for this annotation's definition of null, for
   any parameter marked with this annoation specifically invokes
-   undefined behavior.</p>
+   undefined behavior (see below).</p>

   <p>Passing 0 (i.e. C NULL) or a negative value for any long-type
   parameter marked with this annoation specifically invokes undefined
-   behavior. Such values are treated as C pointers in the JNI
-   layer.</p>
+   behavior (see below). Such values are treated as C pointers in the
+   JNI layer.</p>
+
+   <p><b>Undefined behaviour:</b> the JNI build uses the {@code
+   SQLITE_ENABLE_API_ARMOR} build flag, meaning that the C code
+   invoked with invalid NULL pointers and the like will not invoke
+   undefined behavior in the conventional C sense, but may, for
+   example, return result codes which are not documented for the
+   affected APIs or may otherwise behave unpredictably. In no known
+   cases will such arguments result in C-level code dereferencing a
+   NULL pointer or accessing out-of-bounds (or otherwise invalid)
+   memory. In other words, they may cause unexpected behavior but
+   should never cause an outright crash or security issue.</p>

   <p>Note that the C-style API does not throw any exceptions on its
   own because it has a no-throw policy in order to retain its C-style
@@ -48,12 +60,12 @@ package org.sqlite.jni.annotation;
   code.</p>

   <p>This annotation is solely for the use by the classes in the
-   org.sqlite package and subpackages, but is made public so that
+   org.sqlite.jni package and subpackages, but is made public so that
   javadoc will link to it from the annotated functions. It is not
   part of the public API and client-level code must not rely on
   it.</p>
 */
-@java.lang.annotation.Documented
-@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.SOURCE)
-@java.lang.annotation.Target(java.lang.annotation.ElementType.PARAMETER)
+@Documented
+@Retention(RetentionPolicy.SOURCE)
+@Target(ElementType.PARAMETER)
 public @interface NotNull{}
@@ -9,9 +9,10 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** This file houses the Nullable annotaion for the sqlite3 C API.
+** This file houses the Nullable annotation for the sqlite3 C API.
 */
 package org.sqlite.jni.annotation;
+import java.lang.annotation.*;

 /**
   This annotation is for flagging parameters which may legally be
@@ -26,7 +27,7 @@ package org.sqlite.jni.annotation;
   annotated functions. It is not part of the public API and
   client-level code must not rely on it.
 */
-@java.lang.annotation.Documented
-@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.SOURCE)
-@java.lang.annotation.Target(java.lang.annotation.ElementType.PARAMETER)
+@Documented
+@Retention(RetentionPolicy.SOURCE)
+@Target(ElementType.PARAMETER)
 public @interface Nullable{}
@@ -42,9 +42,75 @@ public abstract class AggregateFunction<T> implements SQLFunction {
  */
  public void xDestroy() {}

+  /**
+     PerContextState assists aggregate and window functions in
+     managing their accumulator state across calls to the UDF's
+     callbacks.
+
+     <p>T must be of a type which can be legally stored as a value in
+     java.util.HashMap<KeyType,T>.
+
+     <p>If a given aggregate or window function is called multiple times
+     in a single SQL statement, e.g. SELECT MYFUNC(A), MYFUNC(B)...,
+     then the clients need some way of knowing which call is which so
+     that they can map their state between their various UDF callbacks
+     and reset it via xFinal(). This class takes care of such
+     mappings.
+
+     <p>This class works by mapping
+     sqlite3_context.getAggregateContext() to a single piece of
+     state, of a client-defined type (the T part of this class), which
+     persists across a "matching set" of the UDF's callbacks.
+
+     <p>This class is a helper providing commonly-needed functionality
+     - it is not required for use with aggregate or window functions.
+     Client UDFs are free to perform such mappings using custom
+     approaches. The provided {@link AggregateFunction} and {@link
+     WindowFunction} classes use this.
+  */
+  public static final class PerContextState<T> {
+    private final java.util.Map<Long,ValueHolder<T>> map
+      = new java.util.HashMap<>();
+
+    /**
+       Should be called from a UDF's xStep(), xValue(), and xInverse()
+       methods, passing it that method's first argument and an initial
+       value for the persistent state. If there is currently no
+       mapping for the given context within the map, one is created
+       using the given initial value, else the existing one is used
+       and the 2nd argument is ignored.  It returns a ValueHolder<T>
+       which can be used to modify that state directly without
+       requiring that the client update the underlying map's entry.
+
+       <p>The caller is obligated to eventually call
+       takeAggregateState() to clear the mapping.
+    */
+    public ValueHolder<T> getAggregateState(sqlite3_context cx, T initialValue){
+      final Long key = cx.getAggregateContext(true);
+      ValueHolder<T> rc = null==key ? null : map.get(key);
+      if( null==rc ){
+        map.put(key, rc = new ValueHolder<>(initialValue));
+      }
+      return rc;
+    }
+
+    /**
+       Should be called from a UDF's xFinal() method and passed that
+       method's first argument. This function removes the value
+       associated with cx.getAggregateContext() from the map and
+       returns it, returning null if no other UDF method has been
+       called to set up such a mapping. The latter condition will be
+       the case if a UDF is used in a statement which has no result
+       rows.
+    */
+    public T takeAggregateState(sqlite3_context cx){
+      final ValueHolder<T> h = map.remove(cx.getAggregateContext(false));
+      return null==h ? null : h.value;
+    }
+  }
+
  /** Per-invocation state for the UDF. */
-  private final SQLFunction.PerContextState<T> map =
-    new SQLFunction.PerContextState<>();
+  private final PerContextState<T> map = new PerContextState<>();

  /**
     To be called from the implementation's xStep() method, as well
@@ -20,7 +20,8 @@ import org.sqlite.jni.annotation.*;
 public interface AuthorizerCallback extends CallbackProxy {
  /**
     Must function as described for the C-level
-     sqlite3_set_authorizer() callback.
+     sqlite3_set_authorizer() callback. If it throws, the error is
+     converted to a db-level error and the exception is suppressed.
  */
  int call(int opId, @Nullable String s1, @Nullable String s2,
           @Nullable String s3, @Nullable String s4);
@@ -24,8 +24,9 @@ package org.sqlite.jni.capi;
   propagated.  For callback interfaces which support returning error
   info to the core, the JNI binding will convert any exceptions to
   C-level error information. For callback interfaces which do not
-   support, all exceptions will necessarily be suppressed in order to
-   retain the C-style no-throw semantics.
+   support returning error information, all exceptions will
+   necessarily be suppressed in order to retain the C-style no-throw
+   semantics and avoid invoking undefined behavior in the C layer.

   <p>Callbacks of this style follow a common naming convention:

@@ -21,8 +21,9 @@ public interface CollationNeededCallback extends CallbackProxy {
     Has the same semantics as the C-level sqlite3_create_collation()
     callback.

-     <p>If it throws, the exception message is passed on to the db and
-     the exception is suppressed.
+     <p>Because the C API has no mechanism for reporting errors
+     from this callbacks, any exceptions thrown by this callback
+     are suppressed.
  */
-  int call(sqlite3 db, int eTextRep, String collationName);
+  void call(sqlite3 db, int eTextRep, String collationName);
 }
@@ -19,7 +19,8 @@ package org.sqlite.jni.capi;
 public interface CommitHookCallback extends CallbackProxy {
  /**
     Works as documented for the C-level sqlite3_commit_hook()
-     callback.  Must not throw.
+     callback. If it throws, the exception is translated into
+     a db-level error.
  */
  int call();
 }
@@ -16,10 +16,10 @@ package org.sqlite.jni.capi;
 /**
   A callback for use with sqlite3_config().
 */
-public interface ConfigSqllogCallback {
+public interface ConfigSqlLogCallback {
  /**
     Must function as described for a C-level callback for
-     {@link CApi#sqlite3_config(ConfigSqllogCallback)}, with the slight signature change.
+     {@link CApi#sqlite3_config(ConfigSqlLogCallback)}, with the slight signature change.
  */
  void call(sqlite3 db, String msg, int msgType );
 }
@@ -228,4 +228,26 @@ public final class OutputPointer {
    /** Sets the current value. */
    public final void set(byte[] v){value = v;}
  }
+
+  /**
+     Output pointer for use with native routines which return
+     blobs via java.nio.ByteBuffer.
+
+     See {@link org.sqlite.jni.capi.CApi#sqlite3_jni_supports_nio}
+  */
+  public static final class ByteBuffer {
+    /**
+       This is public for ease of use. Accessors are provided for
+       consistency with the higher-level types.
+    */
+    public java.nio.ByteBuffer value;
+    /** Initializes with the value null. */
+    public ByteBuffer(){this(null);}
+    /** Initializes with the value v. */
+    public ByteBuffer(java.nio.ByteBuffer v){value = v;}
+    /** Returns the current value. */
+    public final java.nio.ByteBuffer get(){return value;}
+    /** Sets the current value. */
+    public final void set(java.nio.ByteBuffer v){value = v;}
+  }
 }
@@ -25,7 +25,10 @@ public interface PrepareMultiCallback extends CallbackProxy {
     sqlite3_prepare_multi() will _not_ finalize st - it is up
     to the call() implementation how st is handled.

-     Must return 0 on success or an SQLITE_... code on error.
+     Must return 0 on success or an SQLITE_... code on error. If it
+     throws, sqlite3_prepare_multi() will transform the exception into
+     a db-level error in order to retain the C-style error semantics
+     of the API.

     See the {@link Finalize} class for a wrapper which finalizes the
     statement after calling a proxy PrepareMultiCallback.
@@ -37,7 +40,7 @@ public interface PrepareMultiCallback extends CallbackProxy {
     any sqlite3_stmt passed to its callback.
  */
  public static final class Finalize implements PrepareMultiCallback {
-    private PrepareMultiCallback p;
+    private final PrepareMultiCallback p;
    /**
       p is the proxy to call() when this.call() is called.
    */
@@ -19,7 +19,8 @@ package org.sqlite.jni.capi;
 public interface PreupdateHookCallback extends CallbackProxy {
  /**
     Must function as described for the C-level sqlite3_preupdate_hook()
-     callback.
+     callback. If it throws, the exception is translated to a
+     db-level error and the exception is suppressed.
  */
  void call(sqlite3 db, int op, String dbName, String dbTable,
            long iKey1, long iKey2 );
@@ -18,8 +18,9 @@ package org.sqlite.jni.capi;
 */
 public interface RollbackHookCallback extends CallbackProxy {
  /**
-     Works as documented for the C-level sqlite3_rollback_hook()
-     callback.
+     Must function as documented for the C-level sqlite3_rollback_hook()
+     callback. If it throws, the exception is translated into
+     a db-level error.
  */
  void call();
 }
@@ -33,71 +33,4 @@ package org.sqlite.jni.capi;
 */
 public interface SQLFunction {

-  /**
-     PerContextState assists aggregate and window functions in
-     managing their accumulator state across calls to the UDF's
-     callbacks.
-
-     <p>T must be of a type which can be legally stored as a value in
-     java.util.HashMap<KeyType,T>.
-
-     <p>If a given aggregate or window function is called multiple times
-     in a single SQL statement, e.g. SELECT MYFUNC(A), MYFUNC(B)...,
-     then the clients need some way of knowing which call is which so
-     that they can map their state between their various UDF callbacks
-     and reset it via xFinal(). This class takes care of such
-     mappings.
-
-     <p>This class works by mapping
-     sqlite3_context.getAggregateContext() to a single piece of
-     state, of a client-defined type (the T part of this class), which
-     persists across a "matching set" of the UDF's callbacks.
-
-     <p>This class is a helper providing commonly-needed functionality
-     - it is not required for use with aggregate or window functions.
-     Client UDFs are free to perform such mappings using custom
-     approaches. The provided {@link AggregateFunction} and {@link
-     WindowFunction} classes use this.
-  */
-  public static final class PerContextState<T> {
-    private final java.util.Map<Long,ValueHolder<T>> map
-      = new java.util.HashMap<>();
-
-    /**
-       Should be called from a UDF's xStep(), xValue(), and xInverse()
-       methods, passing it that method's first argument and an initial
-       value for the persistent state. If there is currently no
-       mapping for the given context within the map, one is created
-       using the given initial value, else the existing one is used
-       and the 2nd argument is ignored.  It returns a ValueHolder<T>
-       which can be used to modify that state directly without
-       requiring that the client update the underlying map's entry.
-
-       <p>The caller is obligated to eventually call
-       takeAggregateState() to clear the mapping.
-    */
-    public ValueHolder<T> getAggregateState(sqlite3_context cx, T initialValue){
-      final Long key = cx.getAggregateContext(true);
-      ValueHolder<T> rc = null==key ? null : map.get(key);
-      if( null==rc ){
-        map.put(key, rc = new ValueHolder<>(initialValue));
-      }
-      return rc;
-    }
-
-    /**
-       Should be called from a UDF's xFinal() method and passed that
-       method's first argument. This function removes the value
-       associated with cx.getAggregateContext() from the map and
-       returns it, returning null if no other UDF method has been
-       called to set up such a mapping. The latter condition will be
-       the case if a UDF is used in a statement which has no result
-       rows.
-    */
-    public T takeAggregateState(sqlite3_context cx){
-      final ValueHolder<T> h = map.remove(cx.getAggregateContext(false));
-      return null==h ? null : h.value;
-    }
-  }
-
 }
@@ -38,6 +38,14 @@ import java.util.concurrent.Future;
@java.lang.annotation.Target({java.lang.annotation.ElementType.METHOD})
@interface SingleThreadOnly{}

+/**
+   Annotation for Tester1 tests which must only be run if
+   sqlite3_jni_supports_nio() is true.
+*/
+@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.RUNTIME)
+@java.lang.annotation.Target({java.lang.annotation.ElementType.METHOD})
+@interface RequiresJniNio{}
+
 public class Tester1 implements Runnable {
  //! True when running in multi-threaded mode.
  private static boolean mtMode = false;
@@ -46,7 +54,7 @@ public class Tester1 implements Runnable {
  //! True to shuffle the order of the tests.
  private static boolean shuffle = false;
  //! True to dump the list of to-run tests to stdout.
-  private static boolean listRunTests = false;
+  private static int listRunTests = 0;
  //! True to squelch all out() and outln() output.
  private static boolean quietMode = false;
  //! Total number of runTests() calls.
@@ -327,7 +335,7 @@ public class Tester1 implements Runnable {


    rc = sqlite3_prepare_v3(db, "INSERT INTO t2(a) VALUES(1),(2),(3)",
-                            SQLITE_PREPARE_NORMALIZE, outStmt);
+                            0, outStmt);
    affirm(0 == rc);
    stmt = outStmt.get();
    affirm(0 != stmt.getNativePointer());
@@ -382,6 +390,15 @@ public class Tester1 implements Runnable {
    stmt = prepare(db, "SELECT a FROM t ORDER BY a DESC;");
    affirm( sqlite3_stmt_readonly(stmt) );
    affirm( !sqlite3_stmt_busy(stmt) );
+    if( sqlite3_compileoption_used("ENABLE_COLUMN_METADATA") ){
+      /* Unlike in native C code, JNI won't trigger an
+         UnsatisfiedLinkError until these are called (on Linux, at
+         least). */
+      affirm("t".equals(sqlite3_column_table_name(stmt,0)));
+      affirm("main".equals(sqlite3_column_database_name(stmt,0)));
+      affirm("a".equals(sqlite3_column_origin_name(stmt,0)));
+    }
+
    int total2 = 0;
    while( SQLITE_ROW == sqlite3_step(stmt) ){
      affirm( sqlite3_stmt_busy(stmt) );
@@ -477,6 +494,7 @@ public class Tester1 implements Runnable {
    stmt = prepare(db, "SELECT a FROM t ORDER BY a DESC;");
    StringBuilder sbuf = new StringBuilder();
    n = 0;
+    final boolean tryNio = sqlite3_jni_supports_nio();
    while( SQLITE_ROW == sqlite3_step(stmt) ){
      final sqlite3_value sv = sqlite3_value_dup(sqlite3_column_value(stmt,0));
      final String txt = sqlite3_column_text16(stmt, 0);
@@ -491,6 +509,15 @@ public class Tester1 implements Runnable {
                           StandardCharsets.UTF_8)) );
      affirm( txt.length() == sqlite3_value_bytes16(sv)/2 );
      affirm( txt.equals(sqlite3_value_text16(sv)) );
+      if( tryNio ){
+        java.nio.ByteBuffer bu = sqlite3_value_nio_buffer(sv);
+        byte ba[] = sqlite3_value_blob(sv);
+        affirm( ba.length == bu.capacity() );
+        int i = 0;
+        for( byte b : ba ){
+          affirm( b == bu.get(i++) );
+        }
+      }
      sqlite3_value_free(sv);
      ++n;
    }
@@ -548,6 +575,79 @@ public class Tester1 implements Runnable {
    sqlite3_close_v2(db);
  }

+  @RequiresJniNio
+  private void testBindByteBuffer(){
+    /* TODO: these tests need to be much more extensive to check the
+       begin/end range handling. */
+
+    java.nio.ByteBuffer zeroCheck =
+      java.nio.ByteBuffer.allocateDirect(0);
+    affirm( null != zeroCheck );
+    zeroCheck = null;
+    sqlite3 db = createNewDb();
+    execSql(db, "CREATE TABLE t(a)");
+
+    final java.nio.ByteBuffer buf = java.nio.ByteBuffer.allocateDirect(10);
+    buf.put((byte)0x31)/*note that we'll skip this one*/
+      .put((byte)0x32)
+      .put((byte)0x33)
+      .put((byte)0x34)
+      .put((byte)0x35)/*we'll skip this one too*/;
+
+    final int expectTotal = buf.get(1) + buf.get(2) + buf.get(3);
+    sqlite3_stmt stmt = prepare(db, "INSERT INTO t(a) VALUES(?);");
+    affirm( SQLITE_ERROR == sqlite3_bind_blob(stmt, 1, buf, -1, 0),
+            "Buffer offset may not be negative." );
+    affirm( 0 == sqlite3_bind_blob(stmt, 1, buf, 1, 3) );
+    affirm( SQLITE_DONE == sqlite3_step(stmt) );
+    sqlite3_finalize(stmt);
+    stmt = prepare(db, "SELECT a FROM t;");
+    int total = 0;
+    affirm( SQLITE_ROW == sqlite3_step(stmt) );
+    byte blob[] = sqlite3_column_blob(stmt, 0);
+    java.nio.ByteBuffer nioBlob =
+      sqlite3_column_nio_buffer(stmt, 0);
+    affirm(3 == blob.length);
+    affirm(blob.length == nioBlob.capacity());
+    affirm(blob.length == nioBlob.limit());
+    int i = 0;
+    for(byte b : blob){
+      affirm( i<=3 );
+      affirm(b == buf.get(1 + i));
+      affirm(b == nioBlob.get(i));
+      ++i;
+      total += b;
+    }
+    affirm( SQLITE_DONE == sqlite3_step(stmt) );
+    sqlite3_finalize(stmt);
+    affirm(total == expectTotal);
+
+    SQLFunction func =
+      new ScalarFunction(){
+        public void xFunc(sqlite3_context cx, sqlite3_value[] args){
+          sqlite3_result_blob(cx, buf, 1, 3);
+        }
+      };
+
+    affirm( 0 == sqlite3_create_function(db, "myfunc", -1, SQLITE_UTF8, func) );
+    stmt = prepare(db, "SELECT myfunc()");
+    affirm( SQLITE_ROW == sqlite3_step(stmt) );
+    blob = sqlite3_column_blob(stmt, 0);
+    affirm(3 == blob.length);
+    i = 0;
+    total = 0;
+    for(byte b : blob){
+      affirm( i<=3 );
+      affirm(b == buf.get(1 + i++));
+      total += b;
+    }
+    affirm( SQLITE_DONE == sqlite3_step(stmt) );
+    sqlite3_finalize(stmt);
+    affirm(total == expectTotal);
+
+    sqlite3_close_v2(db);
+  }
+
  private void testSql(){
    sqlite3 db = createNewDb();
    sqlite3_stmt stmt = prepare(db, "SELECT 1");
@@ -593,9 +693,9 @@ public class Tester1 implements Runnable {
      };
    final CollationNeededCallback collLoader = new CollationNeededCallback(){
        @Override
-        public int call(sqlite3 dbArg, int eTextRep, String collationName){
+        public void call(sqlite3 dbArg, int eTextRep, String collationName){
          affirm(dbArg == db/* as opposed to a temporary object*/);
-          return sqlite3_create_collation(dbArg, "reversi", eTextRep, myCollation);
+          sqlite3_create_collation(dbArg, "reversi", eTextRep, myCollation);
        }
      };
    int rc = sqlite3_collation_needed(db, collLoader);
@@ -803,13 +903,17 @@ public class Tester1 implements Runnable {
    affirm( 0==rc );
    int n = 0;
    if( SQLITE_ROW == sqlite3_step(stmt) ){
+      affirm( testResult.value == sqlite3_column_java_object(stmt, 0) );
+      affirm( testResult.value == sqlite3_column_java_object(stmt, 0, sqlite3.class) );
+      affirm( null == sqlite3_column_java_object(stmt, 0, sqlite3_stmt.class) );
+      affirm( null == sqlite3_column_java_object(stmt,1) );
      final sqlite3_value v = sqlite3_column_value(stmt, 0);
      affirm( testResult.value == sqlite3_value_java_object(v) );
-      affirm( testResult.value == sqlite3_value_java_casted(v, sqlite3.class) );
+      affirm( testResult.value == sqlite3_value_java_object(v, sqlite3.class) );
      affirm( testResult.value ==
-              sqlite3_value_java_casted(v, testResult.value.getClass()) );
-      affirm( testResult.value == sqlite3_value_java_casted(v, Object.class) );
-      affirm( null == sqlite3_value_java_casted(v, String.class) );
+              sqlite3_value_java_object(v, testResult.value.getClass()) );
+      affirm( testResult.value == sqlite3_value_java_object(v, Object.class) );
+      affirm( null == sqlite3_value_java_object(v, String.class) );
      ++n;
    }
    sqlite3_finalize(stmt);
@@ -824,15 +928,28 @@ public class Tester1 implements Runnable {
      // To confirm that xFinal() is called with no aggregate state
      // when the corresponding result set is empty.
      new ValueHolder<>(false);
+    final ValueHolder<sqlite3_value[]> neverEverDoThisInClientCode = new ValueHolder<>(null);
+    final ValueHolder<sqlite3_context> neverEverDoThisInClientCode2 = new ValueHolder<>(null);
    SQLFunction func = new AggregateFunction<Integer>(){
        @Override
        public void xStep(sqlite3_context cx, sqlite3_value[] args){
+          if( null==neverEverDoThisInClientCode.value ){
+            /* !!!NEVER!!! hold a reference to an sqlite3_value or
+               sqlite3_context object like this in client code! They
+               are ONLY legal for the duration of their single
+               call. We do it here ONLY to test that the defenses
+               against clients doing this are working. */
+            neverEverDoThisInClientCode.value = args;
+          }
          final ValueHolder<Integer> agg = this.getAggregateState(cx, 0);
          agg.value += sqlite3_value_int(args[0]);
          affirm( agg == this.getAggregateState(cx, 0) );
        }
        @Override
        public void xFinal(sqlite3_context cx){
+          if( null==neverEverDoThisInClientCode2.value ){
+            neverEverDoThisInClientCode2.value = cx;
+          }
          final Integer v = this.takeAggregateState(cx);
          if(null == v){
            xFinalNull.value = true;
@@ -857,6 +974,10 @@ public class Tester1 implements Runnable {
    }
    affirm( 1==n );
    affirm(!xFinalNull.value);
+    affirm( null!=neverEverDoThisInClientCode.value );
+    affirm( null!=neverEverDoThisInClientCode2.value );
+    affirm( 0<neverEverDoThisInClientCode.value.length );
+    affirm( 0==neverEverDoThisInClientCode2.value.getNativePointer() );
    sqlite3_reset(stmt);
    affirm( 1==sqlite3_stmt_status(stmt, SQLITE_STMTSTATUS_RUN, false) );
    // Ensure that the accumulator is reset on subsequent calls...
@@ -925,7 +1046,6 @@ public class Tester1 implements Runnable {
                         "ORDER BY x ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING"+
                         ") AS sum_y "+
                         "FROM twin ORDER BY x;");
-    affirm( 0 == rc );
    int n = 0;
    while( SQLITE_ROW == sqlite3_step(stmt) ){
      final String s = sqlite3_column_text16(stmt, 0);
@@ -1028,48 +1148,48 @@ public class Tester1 implements Runnable {
  @SingleThreadOnly /* because threads inherently break this test */
  private static void testBusy(){
    final String dbName = "_busy-handler.db";
-    final OutputPointer.sqlite3 outDb = new OutputPointer.sqlite3();
-    final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt();
-
-    int rc = sqlite3_open(dbName, outDb);
-    ++metrics.dbOpen;
-    affirm( 0 == rc );
-    final sqlite3 db1 = outDb.get();
-    execSql(db1, "CREATE TABLE IF NOT EXISTS t(a)");
-    rc = sqlite3_open(dbName, outDb);
-    ++metrics.dbOpen;
-    affirm( 0 == rc );
-    affirm( outDb.get() != db1 );
-    final sqlite3 db2 = outDb.get();
-
-    affirm( "main".equals( sqlite3_db_name(db1, 0) ) );
-    rc = sqlite3_db_config(db1, SQLITE_DBCONFIG_MAINDBNAME, "foo");
-    affirm( sqlite3_db_filename(db1, "foo").endsWith(dbName) );
-    affirm( "foo".equals( sqlite3_db_name(db1, 0) ) );
-
-    final ValueHolder<Integer> xBusyCalled = new ValueHolder<>(0);
-    BusyHandlerCallback handler = new BusyHandlerCallback(){
-        @Override public int call(int n){
-          //outln("busy handler #"+n);
-          return n > 2 ? 0 : ++xBusyCalled.value;
-        }
-      };
-    rc = sqlite3_busy_handler(db2, handler);
-    affirm(0 == rc);
-
-    // Force a locked condition...
-    execSql(db1, "BEGIN EXCLUSIVE");
-    rc = sqlite3_prepare_v2(db2, "SELECT * from t", outStmt);
-    affirm( SQLITE_BUSY == rc);
-    affirm( null == outStmt.get() );
-    affirm( 3 == xBusyCalled.value );
-    sqlite3_close_v2(db1);
-    sqlite3_close_v2(db2);
    try{
-      final java.io.File f = new java.io.File(dbName);
-      f.delete();
-    }catch(Exception e){
-      /* ignore */
+      final OutputPointer.sqlite3 outDb = new OutputPointer.sqlite3();
+      final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt();
+
+      int rc = sqlite3_open(dbName, outDb);
+      ++metrics.dbOpen;
+      affirm( 0 == rc );
+      final sqlite3 db1 = outDb.get();
+      execSql(db1, "CREATE TABLE IF NOT EXISTS t(a)");
+      rc = sqlite3_open(dbName, outDb);
+      ++metrics.dbOpen;
+      affirm( 0 == rc );
+      affirm( outDb.get() != db1 );
+      final sqlite3 db2 = outDb.get();
+
+      affirm( "main".equals( sqlite3_db_name(db1, 0) ) );
+      rc = sqlite3_db_config(db1, SQLITE_DBCONFIG_MAINDBNAME, "foo");
+      affirm( sqlite3_db_filename(db1, "foo").endsWith(dbName) );
+      affirm( "foo".equals( sqlite3_db_name(db1, 0) ) );
+      affirm( SQLITE_MISUSE == sqlite3_db_config(db1, 0, 0, null) );
+
+      final ValueHolder<Integer> xBusyCalled = new ValueHolder<>(0);
+      BusyHandlerCallback handler = new BusyHandlerCallback(){
+          @Override public int call(int n){
+            //outln("busy handler #"+n);
+            return n > 2 ? 0 : ++xBusyCalled.value;
+          }
+        };
+      rc = sqlite3_busy_handler(db2, handler);
+      affirm(0 == rc);
+
+      // Force a locked condition...
+      execSql(db1, "BEGIN EXCLUSIVE");
+      rc = sqlite3_prepare_v2(db2, "SELECT * from t", outStmt);
+      affirm( SQLITE_BUSY == rc);
+      affirm( null == outStmt.get() );
+      affirm( 3 == xBusyCalled.value );
+      sqlite3_close_v2(db1);
+      sqlite3_close_v2(db2);
+    }finally{
+      try{(new java.io.File(dbName)).delete();}
+      catch(Exception e){/* ignore */}
    }
  }

@@ -1093,6 +1213,7 @@ public class Tester1 implements Runnable {

  private void testCommitHook(){
    final sqlite3 db = createNewDb();
+    sqlite3_extended_result_codes(db, true);
    final ValueHolder<Integer> counter = new ValueHolder<>(0);
    final ValueHolder<Integer> hookResult = new ValueHolder<>(0);
    final CommitHookCallback theHook = new CommitHookCallback(){
@@ -1135,7 +1256,7 @@ public class Tester1 implements Runnable {
    affirm( 5 == counter.value );
    hookResult.value = SQLITE_ERROR;
    int rc = execSql(db, false, "BEGIN; update t set a='j' where a='i'; COMMIT;");
-    affirm( SQLITE_CONSTRAINT == rc );
+    affirm( SQLITE_CONSTRAINT_COMMITHOOK == rc );
    affirm( 6 == counter.value );
    sqlite3_close_v2(db);
  }
@@ -1354,6 +1475,9 @@ public class Tester1 implements Runnable {
    authRc.value = SQLITE_DENY;
    int rc = execSql(db, false, "UPDATE t SET a=2");
    affirm( SQLITE_AUTH==rc );
+    sqlite3_set_authorizer(db, null);
+    rc = execSql(db, false, "UPDATE t SET a=2");
+    affirm( 0==rc );
    // TODO: expand these tests considerably
    sqlite3_close(db);
  }
@@ -1415,7 +1539,7 @@ public class Tester1 implements Runnable {

    val.value = 0;
    final AutoExtensionCallback ax2 = new AutoExtensionCallback(){
-        @Override public synchronized int call(sqlite3 db){
+        @Override public int call(sqlite3 db){
          ++val.value;
          return 0;
        }
@@ -1626,7 +1750,7 @@ public class Tester1 implements Runnable {
    sqlite3_finalize(stmt);

    b = sqlite3_blob_open(db, "main", "t", "a",
-                          sqlite3_last_insert_rowid(db), 1);
+                          sqlite3_last_insert_rowid(db), 0);
    affirm( null!=b );
    rc = sqlite3_blob_reopen(b, 2);
    affirm( 0==rc );
@@ -1636,6 +1760,86 @@ public class Tester1 implements Runnable {
    affirm( 100==tgt[0] && 101==tgt[1] && 102==tgt[2], "DEF" );
    rc = sqlite3_blob_close(b);
    affirm( 0==rc );
+
+    if( !sqlite3_jni_supports_nio() ){
+      outln("WARNING: skipping tests for ByteBuffer-using sqlite3_blob APIs ",
+            "because this platform lacks that support.");
+      sqlite3_close_v2(db);
+      return;
+    }
+    /* Sanity checks for the java.nio.ByteBuffer-taking overloads of
+       sqlite3_blob_read/write(). */
+    execSql(db, "UPDATE t SET a=zeroblob(10)");
+    b = sqlite3_blob_open(db, "main", "t", "a", 1, 1);
+    affirm( null!=b );
+    java.nio.ByteBuffer bb = java.nio.ByteBuffer.allocateDirect(10);
+    for( byte i = 0; i < 10; ++i ){
+      bb.put((int)i, (byte)(48+i & 0xff));
+    }
+    rc = sqlite3_blob_write(b, 1, bb, 1, 10);
+    affirm( rc==SQLITE_ERROR, "b length < (srcOffset + bb length)" );
+    rc = sqlite3_blob_write(b, -1, bb);
+    affirm( rc==SQLITE_ERROR, "Target offset may not be negative" );
+    rc = sqlite3_blob_write(b, 0, bb, -1, -1);
+    affirm( rc==SQLITE_ERROR, "Source offset may not be negative" );
+    rc = sqlite3_blob_write(b, 1, bb, 1, 8);
+    affirm( rc==0 );
+    // b's contents: 0 49  50  51  52  53  54  55  56  0
+    //        ascii: 0 '1' '2' '3' '4' '5' '6' '7' '8' 0
+    byte br[] = new byte[10];
+    java.nio.ByteBuffer bbr =
+      java.nio.ByteBuffer.allocateDirect(bb.limit());
+    rc = sqlite3_blob_read( b, br, 0 );
+    affirm( rc==0 );
+    rc = sqlite3_blob_read( b, bbr );
+    affirm( rc==0 );
+    java.nio.ByteBuffer bbr2 = sqlite3_blob_read_nio_buffer(b, 0, 12);
+    affirm( null==bbr2, "Read size is too big");
+    bbr2 = sqlite3_blob_read_nio_buffer(b, -1, 3);
+    affirm( null==bbr2, "Source offset is negative");
+    bbr2 = sqlite3_blob_read_nio_buffer(b, 5, 6);
+    affirm( null==bbr2, "Read pos+size is too big");
+    bbr2 = sqlite3_blob_read_nio_buffer(b, 4, 7);
+    affirm( null==bbr2, "Read pos+size is too big");
+    bbr2 = sqlite3_blob_read_nio_buffer(b, 4, 6);
+    affirm( null!=bbr2 );
+    java.nio.ByteBuffer bbr3 =
+      java.nio.ByteBuffer.allocateDirect(2 * bb.limit());
+    java.nio.ByteBuffer bbr4 =
+      java.nio.ByteBuffer.allocateDirect(5);
+    rc = sqlite3_blob_read( b, bbr3 );
+    affirm( rc==0 );
+    rc = sqlite3_blob_read( b, bbr4 );
+    affirm( rc==0 );
+    affirm( sqlite3_blob_bytes(b)==bbr3.limit() );
+    affirm( 5==bbr4.limit() );
+    sqlite3_blob_close(b);
+    affirm( 0==br[0] );
+    affirm( 0==br[9] );
+    affirm( 0==bbr.get(0) );
+    affirm( 0==bbr.get(9) );
+    affirm( bbr2.limit() == 6 );
+    affirm( 0==bbr3.get(0) );
+    {
+      Exception ex = null;
+      try{ bbr3.get(11); }
+      catch(Exception e){ex = e;}
+      affirm( ex instanceof IndexOutOfBoundsException,
+              "bbr3.limit() was reset by read()" );
+      ex = null;
+    }
+    affirm( 0==bbr4.get(0) );
+    for( int i = 1; i < 9; ++i ){
+      affirm( br[i] == 48 + i );
+      affirm( br[i] == bbr.get(i) );
+      affirm( br[i] == bbr3.get(i) );
+      if( i>3 ){
+        affirm( br[i] == bbr2.get(i-4) );
+      }
+      if( i < bbr4.limit() ){
+        affirm( br[i] == bbr4.get(i) );
+      }
+    }
    sqlite3_close_v2(db);
  }

@@ -1648,9 +1852,13 @@ public class Tester1 implements Runnable {
    };
    final List<sqlite3_stmt> liStmt = new ArrayList<sqlite3_stmt>();
    final PrepareMultiCallback proxy = new PrepareMultiCallback.StepAll();
+    final ValueHolder<String> toss = new ValueHolder<>(null);
    PrepareMultiCallback m = new PrepareMultiCallback() {
        @Override public int call(sqlite3_stmt st){
          liStmt.add(st);
+          if( null!=toss.value ){
+            throw new RuntimeException(toss.value);
+          }
          return proxy.call(st);
        }
      };
@@ -1660,6 +1868,10 @@ public class Tester1 implements Runnable {
    for( sqlite3_stmt st : liStmt ){
      sqlite3_finalize(st);
    }
+    toss.value = "This is an exception.";
+    rc = sqlite3_prepare_multi(db, "SELECT 1", m);
+    affirm( SQLITE_ERROR==rc );
+    affirm( sqlite3_errmsg(db).indexOf(toss.value)>0 );
    sqlite3_close_v2(db);
  }

@@ -1698,7 +1910,7 @@ public class Tester1 implements Runnable {
      mlist = new ArrayList<>( testMethods.subList(0, testMethods.size()) );
      java.util.Collections.shuffle(mlist);
    }
-    if( listRunTests ){
+    if( (!fromThread && listRunTests>0) || listRunTests>1 ){
      synchronized(this.getClass()){
        if( !fromThread ){
          out("Initial test"," list: ");
@@ -1760,8 +1972,11 @@ public class Tester1 implements Runnable {
     -naps: sleep small random intervals between tests in order to add
     some chaos for cross-thread contention.

+
     -list-tests: outputs the list of tests being run, minus some
-      which are hard-coded. This is noisy in multi-threaded mode.
+      which are hard-coded. In multi-threaded mode, use this twice to
+      to emit the list run by each thread (which may differ from the initial
+      list, in particular if -shuffle is used).

     -fail: forces an exception to be thrown during the test run.  Use
     with -shuffle to make its appearance unpredictable.
@@ -1790,7 +2005,7 @@ public class Tester1 implements Runnable {
        }else if(arg.equals("shuffle")){
          shuffle = true;
        }else if(arg.equals("list-tests")){
-          listRunTests = true;
+          ++listRunTests;
        }else if(arg.equals("fail")){
          forceFail = true;
        }else if(arg.equals("sqllog")){
@@ -1809,7 +2024,7 @@ public class Tester1 implements Runnable {

    if( sqlLog ){
      if( sqlite3_compileoption_used("ENABLE_SQLLOG") ){
-        final ConfigSqllogCallback log = new ConfigSqllogCallback() {
+        final ConfigSqlLogCallback log = new ConfigSqlLogCallback() {
            @Override public void call(sqlite3 db, String msg, int op){
              switch(op){
                case 0: outln("Opening db: ",db); break;
@@ -1820,7 +2035,7 @@ public class Tester1 implements Runnable {
          };
        int rc = sqlite3_config( log );
        affirm( 0==rc );
-        rc = sqlite3_config( (ConfigSqllogCallback)null );
+        rc = sqlite3_config( (ConfigSqlLogCallback)null );
        affirm( 0==rc );
        rc = sqlite3_config( log );
        affirm( 0==rc );
@@ -1858,18 +2073,20 @@ public class Tester1 implements Runnable {
          if( forceFail ){
            testMethods.add(m);
          }
+        }else if( m.isAnnotationPresent( RequiresJniNio.class )
+                  && !sqlite3_jni_supports_nio() ){
+          outln("Skipping test for lack of JNI java.nio.ByteBuffer support: ",
+                name,"()\n");
+          ++nSkipped;
        }else if( !m.isAnnotationPresent( ManualTest.class ) ){
          if( nThread>1 && m.isAnnotationPresent( SingleThreadOnly.class ) ){
-            if( 0==nSkipped++ ){
-              out("Skipping tests in multi-thread mode:");
-            }
-            out(" "+name+"()");
+            out("Skipping test in multi-thread mode: ",name,"()\n");
+            ++nSkipped;
          }else if( name.startsWith("test") ){
            testMethods.add(m);
          }
        }
      }
-      if( nSkipped>0 ) out("\n");
    }

    final long timeStart = System.currentTimeMillis();
@@ -1901,6 +2118,7 @@ public class Tester1 implements Runnable {
          sqlite3_libversion_number(),"\n",
          sqlite3_libversion(),"\n",SQLITE_SOURCE_ID,"\n",
          "SQLITE_THREADSAFE=",sqlite3_threadsafe());
+    outln("JVM NIO support? ",sqlite3_jni_supports_nio() ? "YES" : "NO");
    final boolean showLoopCount = (nRepeat>1 && nThread>1);
    if( showLoopCount ){
      outln("Running ",nRepeat," loop(s) with ",nThread," thread(s) each.");
@@ -19,7 +19,8 @@ package org.sqlite.jni.capi;
 public interface UpdateHookCallback extends CallbackProxy {
  /**
     Must function as described for the C-level sqlite3_update_hook()
-     callback.
+     callback. If it throws, the exception is translated into
+     a db-level error.
  */
  void call(int opId, String dbName, String tableName, long rowId);
 }
@@ -9,14 +9,16 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** This file contains a set of tests for the sqlite3 JNI bindings.
+** This file contains the ValueHolder utility class for the sqlite3
+** JNI bindings.
 */
 package org.sqlite.jni.capi;

 /**
   A helper class which simply holds a single value. Its primary use
   is for communicating values out of anonymous classes, as doing so
-   requires a "final" reference.
+   requires a "final" reference, as well as communicating aggregate
+   SQL function state across calls to such functions.
 */
 public class ValueHolder<T> {
  public T value;
@@ -38,6 +38,6 @@ public final class sqlite3 extends NativePointerHolder<sqlite3>
  }

  @Override public void close(){
-    CApi.sqlite3_close_v2(this.clearNativePointer());
+    CApi.sqlite3_close_v2(this);
  }
 }
@@ -25,7 +25,6 @@ public final class sqlite3_blob extends NativePointerHolder<sqlite3_blob>
  private sqlite3_blob(){}

  @Override public void close(){
-    CApi.sqlite3_blob_close(this.clearNativePointer());
+    CApi.sqlite3_blob_close(this);
  }
-
 }
@@ -25,6 +25,6 @@ public final class sqlite3_stmt extends NativePointerHolder<sqlite3_stmt>
  private sqlite3_stmt(){}

  @Override public void close(){
-    CApi.sqlite3_finalize(this.clearNativePointer());
+    CApi.sqlite3_finalize(this);
  }
 }
@@ -12,10 +12,6 @@
 ** This file is part of the wrapper1 interface for sqlite3.
 */
 package org.sqlite.jni.wrapper1;
-import org.sqlite.jni.capi.CApi;
-import org.sqlite.jni.annotation.*;
-import org.sqlite.jni.capi.sqlite3_context;
-import org.sqlite.jni.capi.sqlite3_value;

 /**
   EXPERIMENTAL/INCOMPLETE/UNTESTED
@@ -51,9 +47,75 @@ public abstract class AggregateFunction<T> implements SqlFunction  {
  */
  public void xDestroy() {}

+  /**
+     PerContextState assists aggregate and window functions in
+     managing their accumulator state across calls to the UDF's
+     callbacks.
+
+     <p>T must be of a type which can be legally stored as a value in
+     java.util.HashMap<KeyType,T>.
+
+     <p>If a given aggregate or window function is called multiple times
+     in a single SQL statement, e.g. SELECT MYFUNC(A), MYFUNC(B)...,
+     then the clients need some way of knowing which call is which so
+     that they can map their state between their various UDF callbacks
+     and reset it via xFinal(). This class takes care of such
+     mappings.
+
+     <p>This class works by mapping
+     sqlite3_context.getAggregateContext() to a single piece of
+     state, of a client-defined type (the T part of this class), which
+     persists across a "matching set" of the UDF's callbacks.
+
+     <p>This class is a helper providing commonly-needed functionality
+     - it is not required for use with aggregate or window functions.
+     Client UDFs are free to perform such mappings using custom
+     approaches. The provided {@link AggregateFunction} and {@link
+     WindowFunction} classes use this.
+  */
+  public static final class PerContextState<T> {
+    private final java.util.Map<Long,ValueHolder<T>> map
+      = new java.util.HashMap<>();
+
+    /**
+       Should be called from a UDF's xStep(), xValue(), and xInverse()
+       methods, passing it that method's first argument and an initial
+       value for the persistent state. If there is currently no
+       mapping for the given context within the map, one is created
+       using the given initial value, else the existing one is used
+       and the 2nd argument is ignored.  It returns a ValueHolder<T>
+       which can be used to modify that state directly without
+       requiring that the client update the underlying map's entry.
+
+       <p>The caller is obligated to eventually call
+       takeAggregateState() to clear the mapping.
+    */
+    public ValueHolder<T> getAggregateState(SqlFunction.Arguments args, T initialValue){
+      final Long key = args.getContext().getAggregateContext(true);
+      ValueHolder<T> rc = null==key ? null : map.get(key);
+      if( null==rc ){
+        map.put(key, rc = new ValueHolder<>(initialValue));
+      }
+      return rc;
+    }
+
+    /**
+       Should be called from a UDF's xFinal() method and passed that
+       method's first argument. This function removes the value
+       associated with with the arguments' aggregate context from the
+       map and returns it, returning null if no other UDF method has
+       been called to set up such a mapping. The latter condition will
+       be the case if a UDF is used in a statement which has no result
+       rows.
+    */
+    public T takeAggregateState(SqlFunction.Arguments args){
+      final ValueHolder<T> h = map.remove(args.getContext().getAggregateContext(false));
+      return null==h ? null : h.value;
+    }
+  }
+
  /** Per-invocation state for the UDF. */
-  private final SqlFunction.PerContextState<T> map =
-    new SqlFunction.PerContextState<>();
+  private final PerContextState<T> map = new PerContextState<>();

  /**
     To be called from the implementation's xStep() method, as well
@@ -22,6 +22,14 @@ import org.sqlite.jni.capi.sqlite3_value;
 */
 public interface SqlFunction  {

+  public static final int DETERMINISTIC = CApi.SQLITE_DETERMINISTIC;
+  public static final int INNOCUOUS = CApi.SQLITE_INNOCUOUS;
+  public static final int DIRECTONLY = CApi.SQLITE_DIRECTONLY;
+  public static final int SUBTYPE = CApi.SQLITE_SUBTYPE;
+  public static final int RESULT_SUBTYPE = CApi.SQLITE_RESULT_SUBTYPE;
+  public static final int UTF8 = CApi.SQLITE_UTF8;
+  public static final int UTF16 = CApi.SQLITE_UTF16;
+
  /**
     The Arguments type is an abstraction on top of the lower-level
     UDF function argument types. It provides _most_ of the functionality
@@ -45,12 +53,118 @@ public interface SqlFunction  {
    */
    Arguments(sqlite3_context cx, sqlite3_value args[]){
      this.cx = cx;
-      this.args = args==null ? new sqlite3_value[0] : args;;
+      this.args = args==null ? new sqlite3_value[0] : args;
      this.length = this.args.length;
    }

    /**
-       Wrapper for a single SqlFunction argument. Primarily intended
+       Returns the sqlite3_value at the given argument index or throws
+       an IllegalArgumentException exception if ndx is out of range.
+    */
+    private sqlite3_value valueAt(int ndx){
+      if(ndx<0 || ndx>=args.length){
+        throw new IllegalArgumentException(
+          "SQL function argument index "+ndx+" is out of range."
+        );
+      }
+      return args[ndx];
+    }
+
+    //! Returns the underlying sqlite3_context for these arguments.
+    sqlite3_context getContext(){return cx;}
+
+    /**
+       Returns the Sqlite (db) object associated with this UDF call,
+       or null if the UDF is somehow called without such an object or
+       the db has been closed in an untimely manner (e.g. closed by a
+       UDF call).
+    */
+    public Sqlite getDb(){
+      return Sqlite.fromNative( CApi.sqlite3_context_db_handle(cx) );
+    }
+
+    public int getArgCount(){ return args.length; }
+
+    public int getInt(int argNdx){return CApi.sqlite3_value_int(valueAt(argNdx));}
+    public long getInt64(int argNdx){return CApi.sqlite3_value_int64(valueAt(argNdx));}
+    public double getDouble(int argNdx){return CApi.sqlite3_value_double(valueAt(argNdx));}
+    public byte[] getBlob(int argNdx){return CApi.sqlite3_value_blob(valueAt(argNdx));}
+    public byte[] getText(int argNdx){return CApi.sqlite3_value_text(valueAt(argNdx));}
+    public String getText16(int argNdx){return CApi.sqlite3_value_text16(valueAt(argNdx));}
+    public int getBytes(int argNdx){return CApi.sqlite3_value_bytes(valueAt(argNdx));}
+    public int getBytes16(int argNdx){return CApi.sqlite3_value_bytes16(valueAt(argNdx));}
+    public Object getObject(int argNdx){return CApi.sqlite3_value_java_object(valueAt(argNdx));}
+    public <T> T getObject(int argNdx, Class<T> type){
+      return CApi.sqlite3_value_java_object(valueAt(argNdx), type);
+    }
+
+    public int getType(int argNdx){return CApi.sqlite3_value_type(valueAt(argNdx));}
+    public int getSubtype(int argNdx){return CApi.sqlite3_value_subtype(valueAt(argNdx));}
+    public int getNumericType(int argNdx){return CApi.sqlite3_value_numeric_type(valueAt(argNdx));}
+    public int getNoChange(int argNdx){return CApi.sqlite3_value_nochange(valueAt(argNdx));}
+    public boolean getFromBind(int argNdx){return CApi.sqlite3_value_frombind(valueAt(argNdx));}
+    public int getEncoding(int argNdx){return CApi.sqlite3_value_encoding(valueAt(argNdx));}
+
+    public void resultInt(int v){ CApi.sqlite3_result_int(cx, v); }
+    public void resultInt64(long v){ CApi.sqlite3_result_int64(cx, v); }
+    public void resultDouble(double v){ CApi.sqlite3_result_double(cx, v); }
+    public void resultError(String msg){CApi.sqlite3_result_error(cx, msg);}
+    public void resultError(Exception e){CApi.sqlite3_result_error(cx, e);}
+    public void resultErrorTooBig(){CApi.sqlite3_result_error_toobig(cx);}
+    public void resultErrorCode(int rc){CApi.sqlite3_result_error_code(cx, rc);}
+    public void resultObject(Object o){CApi.sqlite3_result_java_object(cx, o);}
+    public void resultNull(){CApi.sqlite3_result_null(cx);}
+    /**
+       Analog to sqlite3_result_value(), using the Value object at the
+       given argument index.
+    */
+    public void resultArg(int argNdx){CApi.sqlite3_result_value(cx, valueAt(argNdx));}
+    public void resultSubtype(int subtype){CApi.sqlite3_result_subtype(cx, subtype);}
+    public void resultZeroBlob(long n){
+      // Throw on error? If n is too big,
+      // sqlite3_result_error_toobig() is automatically called.
+      CApi.sqlite3_result_zeroblob64(cx, n);
+    }
+
+    public void resultBlob(byte[] blob){CApi.sqlite3_result_blob(cx, blob);}
+    public void resultText(byte[] utf8){CApi.sqlite3_result_text(cx, utf8);}
+    public void resultText(String txt){CApi.sqlite3_result_text(cx, txt);}
+    public void resultText16(byte[] utf16){CApi.sqlite3_result_text16(cx, utf16);}
+    public void resultText16(String txt){CApi.sqlite3_result_text16(cx, txt);}
+
+    /**
+       Callbacks should invoke this on OOM errors, instead of throwing
+       OutOfMemoryError, because the latter cannot be propagated
+       through the C API.
+    */
+    public void resultNoMem(){CApi.sqlite3_result_error_nomem(cx);}
+
+    /**
+       Analog to sqlite3_set_auxdata() but throws if argNdx is out of
+       range.
+    */
+    public void setAuxData(int argNdx, Object o){
+      /* From the API docs: https://www.sqlite.org/c3ref/get_auxdata.html
+
+         The value of the N parameter to these interfaces should be
+         non-negative. Future enhancements may make use of negative N
+         values to define new kinds of function caching behavior.
+      */
+      valueAt(argNdx);
+      CApi.sqlite3_set_auxdata(cx, argNdx, o);
+    }
+
+    /**
+       Analog to sqlite3_get_auxdata() but throws if argNdx is out of
+       range.
+    */
+    public Object getAuxData(int argNdx){
+      valueAt(argNdx);
+      return CApi.sqlite3_get_auxdata(cx, argNdx);
+    }
+
+    /**
+       Represents a single SqlFunction argument. Primarily intended
       for use with the Arguments class's Iterable interface.
    */
    public final static class Arg {
@@ -72,7 +186,7 @@ public interface SqlFunction  {
      public int getBytes(){return a.getBytes(ndx);}
      public int getBytes16(){return a.getBytes16(ndx);}
      public Object getObject(){return a.getObject(ndx);}
-      public <T> T getObjectCasted(Class<T> type){ return a.getObjectCasted(ndx, type); }
+      public <T> T getObject(Class<T> type){ return a.getObject(ndx, type); }
      public int getType(){return a.getType(ndx);}
      public Object getAuxData(){return a.getAuxData(ndx);}
      public void setAuxData(Object o){a.setAuxData(ndx, o);}
@@ -87,147 +201,6 @@ public interface SqlFunction  {
      return java.util.Arrays.stream(proxies).iterator();
    }

-    /**
-       Returns the sqlite3_value at the given argument index or throws
-       an IllegalArgumentException exception if ndx is out of range.
-    */
-    private sqlite3_value valueAt(int ndx){
-      if(ndx<0 || ndx>=args.length){
-        throw new IllegalArgumentException(
-          "SQL function argument index "+ndx+" is out of range."
-        );
-      }
-      return args[ndx];
-    }
-
-    sqlite3_context getContext(){return cx;}
-
-    public int getArgCount(){ return args.length; }
-
-    public int getInt(int arg){return CApi.sqlite3_value_int(valueAt(arg));}
-    public long getInt64(int arg){return CApi.sqlite3_value_int64(valueAt(arg));}
-    public double getDouble(int arg){return CApi.sqlite3_value_double(valueAt(arg));}
-    public byte[] getBlob(int arg){return CApi.sqlite3_value_blob(valueAt(arg));}
-    public byte[] getText(int arg){return CApi.sqlite3_value_text(valueAt(arg));}
-    public String getText16(int arg){return CApi.sqlite3_value_text16(valueAt(arg));}
-    public int getBytes(int arg){return CApi.sqlite3_value_bytes(valueAt(arg));}
-    public int getBytes16(int arg){return CApi.sqlite3_value_bytes16(valueAt(arg));}
-    public Object getObject(int arg){return CApi.sqlite3_value_java_object(valueAt(arg));}
-    public <T> T getObjectCasted(int arg, Class<T> type){
-      return CApi.sqlite3_value_java_casted(valueAt(arg), type);
-    }
-
-    public int getType(int arg){return CApi.sqlite3_value_type(valueAt(arg));}
-    public int getSubtype(int arg){return CApi.sqlite3_value_subtype(valueAt(arg));}
-    public int getNumericType(int arg){return CApi.sqlite3_value_numeric_type(valueAt(arg));}
-    public int getNoChange(int arg){return CApi.sqlite3_value_nochange(valueAt(arg));}
-    public boolean getFromBind(int arg){return CApi.sqlite3_value_frombind(valueAt(arg));}
-    public int getEncoding(int arg){return CApi.sqlite3_value_encoding(valueAt(arg));}
-
-    public void resultInt(int v){ CApi.sqlite3_result_int(cx, v); }
-    public void resultInt64(long v){ CApi.sqlite3_result_int64(cx, v); }
-    public void resultDouble(double v){ CApi.sqlite3_result_double(cx, v); }
-    public void resultError(String msg){CApi.sqlite3_result_error(cx, msg);}
-    public void resultError(Exception e){CApi.sqlite3_result_error(cx, e);}
-    public void resultErrorTooBig(){CApi.sqlite3_result_error_toobig(cx);}
-    public void resultErrorCode(int rc){CApi.sqlite3_result_error_code(cx, rc);}
-    public void resultObject(Object o){CApi.sqlite3_result_java_object(cx, o);}
-    public void resultNull(){CApi.sqlite3_result_null(cx);}
-    public void resultArg(int argNdx){CApi.sqlite3_result_value(cx, valueAt(argNdx));}
-    public void resultZeroBlob(long n){
-      // Throw on error? If n is too big,
-      // sqlite3_result_error_toobig() is automatically called.
-      CApi.sqlite3_result_zeroblob64(cx, n);
-    }
-
-    public void resultBlob(byte[] blob){CApi.sqlite3_result_blob(cx, blob);}
-    public void resultText(byte[] utf8){CApi.sqlite3_result_text(cx, utf8);}
-    public void resultText(String txt){CApi.sqlite3_result_text(cx, txt);}
-    public void resultText16(byte[] utf16){CApi.sqlite3_result_text16(cx, utf16);}
-    public void resultText16(String txt){CApi.sqlite3_result_text16(cx, txt);}
-
-    public void setAuxData(int arg, Object o){
-      /* From the API docs: https://www.sqlite.org/c3ref/get_auxdata.html
-
-         The value of the N parameter to these interfaces should be
-         non-negative. Future enhancements may make use of negative N
-         values to define new kinds of function caching behavior.
-      */
-      valueAt(arg);
-      CApi.sqlite3_set_auxdata(cx, arg, o);
-    }
-
-    public Object getAuxData(int arg){
-      valueAt(arg);
-      return CApi.sqlite3_get_auxdata(cx, arg);
-    }
-  }
-
-  /**
-     PerContextState assists aggregate and window functions in
-     managing their accumulator state across calls to the UDF's
-     callbacks.
-
-     <p>T must be of a type which can be legally stored as a value in
-     java.util.HashMap<KeyType,T>.
-
-     <p>If a given aggregate or window function is called multiple times
-     in a single SQL statement, e.g. SELECT MYFUNC(A), MYFUNC(B)...,
-     then the clients need some way of knowing which call is which so
-     that they can map their state between their various UDF callbacks
-     and reset it via xFinal(). This class takes care of such
-     mappings.
-
-     <p>This class works by mapping
-     sqlite3_context.getAggregateContext() to a single piece of
-     state, of a client-defined type (the T part of this class), which
-     persists across a "matching set" of the UDF's callbacks.
-
-     <p>This class is a helper providing commonly-needed functionality
-     - it is not required for use with aggregate or window functions.
-     Client UDFs are free to perform such mappings using custom
-     approaches. The provided {@link AggregateFunction} and {@link
-     WindowFunction} classes use this.
-  */
-  public static final class PerContextState<T> {
-    private final java.util.Map<Long,ValueHolder<T>> map
-      = new java.util.HashMap<>();
-
-    /**
-       Should be called from a UDF's xStep(), xValue(), and xInverse()
-       methods, passing it that method's first argument and an initial
-       value for the persistent state. If there is currently no
-       mapping for the given context within the map, one is created
-       using the given initial value, else the existing one is used
-       and the 2nd argument is ignored.  It returns a ValueHolder<T>
-       which can be used to modify that state directly without
-       requiring that the client update the underlying map's entry.
-
-       <p>The caller is obligated to eventually call
-       takeAggregateState() to clear the mapping.
-    */
-    public ValueHolder<T> getAggregateState(SqlFunction.Arguments args, T initialValue){
-      final Long key = args.getContext().getAggregateContext(true);
-      ValueHolder<T> rc = null==key ? null : map.get(key);
-      if( null==rc ){
-        map.put(key, rc = new ValueHolder<>(initialValue));
-      }
-      return rc;
-    }
-
-    /**
-       Should be called from a UDF's xFinal() method and passed that
-       method's first argument. This function removes the value
-       associated with with the arguments' aggregate context from the
-       map and returns it, returning null if no other UDF method has
-       been called to set up such a mapping. The latter condition will
-       be the case if a UDF is used in a statement which has no result
-       rows.
-    */
-    public T takeAggregateState(SqlFunction.Arguments args){
-      final ValueHolder<T> h = map.remove(args.getContext().getAggregateContext(false));
-      return null==h ? null : h.value;
-    }
  }

  /**
@@ -235,7 +208,7 @@ public interface SqlFunction  {
     for use with the org.sqlite.jni.capi.ScalarFunction interface.
  */
  static final class ScalarAdapter extends org.sqlite.jni.capi.ScalarFunction {
-    final ScalarFunction impl;
+    private final ScalarFunction impl;
    ScalarAdapter(ScalarFunction impl){
      this.impl = impl;
    }
@@ -261,8 +234,9 @@ public interface SqlFunction  {
     Internal-use adapter for wrapping this package's AggregateFunction
     for use with the org.sqlite.jni.capi.AggregateFunction interface.
  */
-  static final class AggregateAdapter extends org.sqlite.jni.capi.AggregateFunction {
-    final AggregateFunction impl;
+  static /*cannot be final without duplicating the whole body in WindowAdapter*/
+  class AggregateAdapter extends org.sqlite.jni.capi.AggregateFunction {
+    private final AggregateFunction impl;
    AggregateAdapter(AggregateFunction impl){
      this.impl = impl;
    }
@@ -282,8 +256,9 @@ public interface SqlFunction  {
    }

    /**
-       As for the xFinal() argument of the C API's sqlite3_create_function().
-       If the proxied function throws, it is translated into a sqlite3_result_error().
+       As for the xFinal() argument of the C API's
+       sqlite3_create_function().  If the proxied function throws, it
+       is translated into a sqlite3_result_error().
    */
    public void xFinal(sqlite3_context cx){
      try{
@@ -298,4 +273,46 @@ public interface SqlFunction  {
    }
  }

+  /**
+     Internal-use adapter for wrapping this package's WindowFunction
+     for use with the org.sqlite.jni.capi.WindowFunction interface.
+  */
+  static final class WindowAdapter extends AggregateAdapter {
+    private final WindowFunction impl;
+    WindowAdapter(WindowFunction impl){
+      super(impl);
+      this.impl = impl;
+    }
+
+    /**
+       Proxies this.impl.xInverse(), adapting the call arguments to that
+       function's signature. If the proxied function throws, it is
+       translated to sqlite_result_error() with the exception's
+       message.
+    */
+    public void xInverse(sqlite3_context cx, sqlite3_value[] args){
+      try{
+        impl.xInverse( new SqlFunction.Arguments(cx, args) );
+      }catch(Exception e){
+        CApi.sqlite3_result_error(cx, e);
+      }
+    }
+
+    /**
+       As for the xValue() argument of the C API's sqlite3_create_window_function().
+       If the proxied function throws, it is translated into a sqlite3_result_error().
+    */
+    public void xValue(sqlite3_context cx){
+      try{
+        impl.xValue( new SqlFunction.Arguments(cx, null) );
+      }catch(Exception e){
+        CApi.sqlite3_result_error(cx, e);
+      }
+    }
+
+    public void xDestroy(){
+      impl.xDestroy();
+    }
+  }
+
 }
@@ -12,7 +12,7 @@
 ** This file is part of the wrapper1 interface for sqlite3.
 */
 package org.sqlite.jni.wrapper1;
-import static org.sqlite.jni.capi.CApi.*;
+import org.sqlite.jni.capi.CApi;
 import org.sqlite.jni.capi.sqlite3;

 /**
@@ -22,10 +22,10 @@ import org.sqlite.jni.capi.sqlite3;
   and C via JNI.
 */
 public final class SqliteException extends java.lang.RuntimeException {
-  int errCode = SQLITE_ERROR;
-  int xerrCode = SQLITE_ERROR;
-  int errOffset = -1;
-  int sysErrno = 0;
+  private int errCode = CApi.SQLITE_ERROR;
+  private int xerrCode = CApi.SQLITE_ERROR;
+  private int errOffset = -1;
+  private int sysErrno = 0;

  /**
     Records the given error string and uses SQLITE_ERROR for both the
@@ -38,28 +38,31 @@ public final class SqliteException extends java.lang.RuntimeException {
  /**
     Uses sqlite3_errstr(sqlite3ResultCode) for the error string and
     sets both the error code and extended error code to the given
-     value.
+     value. This approach includes no database-level information and
+     systemErrno() will be 0, so is intended only for use with sqlite3
+     APIs for which a result code is not an error but which the
+     higher-level wrapper should treat as one.
  */
  public SqliteException(int sqlite3ResultCode){
-    super(sqlite3_errstr(sqlite3ResultCode));
+    super(CApi.sqlite3_errstr(sqlite3ResultCode));
    errCode = xerrCode = sqlite3ResultCode;
  }

  /**
     Records the current error state of db (which must not be null and
-     must refer to an opened db object). Note that this does NOT close
+     must refer to an opened db object). Note that this does not close
     the db.

-     Design note: closing the db on error is likely only useful during
+     Design note: closing the db on error is really only useful during
     a failed db-open operation, and the place(s) where that can
     happen are inside this library, not client-level code.
  */
  SqliteException(sqlite3 db){
-    super(sqlite3_errmsg(db));
-    errCode = sqlite3_errcode(db);
-    xerrCode = sqlite3_extended_errcode(db);
-    errOffset = sqlite3_error_offset(db);
-    sysErrno = sqlite3_system_errno(db);
+    super(CApi.sqlite3_errmsg(db));
+    errCode = CApi.sqlite3_errcode(db);
+    xerrCode = CApi.sqlite3_extended_errcode(db);
+    errOffset = CApi.sqlite3_error_offset(db);
+    sysErrno = CApi.sqlite3_system_errno(db);
  }

  /**
@@ -71,7 +74,7 @@ public final class SqliteException extends java.lang.RuntimeException {
  }

  public SqliteException(Sqlite.Stmt stmt){
-    this( stmt.db() );
+    this(stmt.getDb());
  }

  public int errcode(){ return errCode; }
@@ -9,13 +9,13 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** This file contains a set of tests for the sqlite3 JNI bindings.
+** This file contains the ValueHolder utility class.
 */
 package org.sqlite.jni.wrapper1;

 /**
   A helper class which simply holds a single value. Its primary use
-   is for communicating values out of anonymous classes, as doing so
+   is for communicating values out of anonymous callbacks, as doing so
   requires a "final" reference.
 */
 public class ValueHolder<T> {
@@ -0,0 +1,42 @@
+/*
+** 2023-10-16
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the wrapper1 interface for sqlite3.
+*/
+package org.sqlite.jni.wrapper1;
+
+/**
+   A SqlFunction implementation for window functions. The T type
+   represents the type of data accumulated by this function while it
+   works. e.g. a SUM()-like UDF might use Integer or Long and a
+   CONCAT()-like UDF might use a StringBuilder or a List<String>.
+*/
+public abstract class WindowFunction<T> extends AggregateFunction<T>  {
+
+  /**
+     As for the xInverse() argument of the C API's
+     sqlite3_create_window_function().  If this function throws, the
+     exception is reported via sqlite3_result_error().
+  */
+  public abstract void xInverse(SqlFunction.Arguments args);
+
+  /**
+     As for the xValue() argument of the C API's
+     sqlite3_create_window_function(). If this function throws, it is
+     translated into sqlite3_result_error().
+
+     Note that the passed-in object will not actually contain any
+     arguments for xValue() but will contain the context object needed
+     for setting the call's result or error state.
+  */
+  public abstract void xValue(SqlFunction.Arguments args);
+
+}
@@ -582,6 +582,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
            bNextPage = 1;
          }else{
            iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload);
+            if( nPayload>0x7fffff00 ) nPayload &= 0x3fff;
          }
    
          /* If this is a leaf intkey cell, load the rowid */
@@ -854,7 +854,7 @@ dir.sql := sql
 speedtest1 := ../../speedtest1
 speedtest1.c := ../../test/speedtest1.c
 speedtest1.sql := $(dir.sql)/speedtest1.sql
-speedtest1.cliflags := --size 25 --big-transactions
+speedtest1.cliflags := --size 10 --big-transactions
 $(speedtest1):
 	$(MAKE) -C ../.. speedtest1
 $(speedtest1.sql): $(speedtest1) $(MAKEFILE)
@@ -63,6 +63,7 @@ _sqlite3_file_control
 _sqlite3_finalize
 _sqlite3_free
 _sqlite3_get_auxdata
+_sqlite3_get_autocommit
 _sqlite3_initialize
 _sqlite3_keyword_count
 _sqlite3_keyword_name
@@ -188,6 +188,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
    ["sqlite3_file_control", "int", "sqlite3*", "string", "int", "*"],
    ["sqlite3_finalize", "int", "sqlite3_stmt*"],
    ["sqlite3_free", undefined,"*"],
+    ["sqlite3_get_autocommit", "int", "sqlite3*"],
    ["sqlite3_get_auxdata", "*", "sqlite3_context*", "int"],
    ["sqlite3_initialize", undefined],
    /*["sqlite3_interrupt", undefined, "sqlite3*"
@@ -62,7 +62,7 @@

  ```
  {
-    type: string, // one of: 'open', 'close', 'exec', 'config-get'
+    type: string, // one of: 'open', 'close', 'exec', 'export', 'config-get'

    messageId: OPTIONAL arbitrary value. The worker will copy it as-is
    into response messages to assist in client-side dispatching.
@@ -325,6 +325,37 @@
  passed only a string), noting that options.resultRows and
  options.columnNames may be populated by the call to db.exec().

+
+  ====================================================================
+  "export" the current db
+
+  To export the underlying database as a byte array...
+
+  Message format:
+
+  ```
+  {
+    type: "export",
+    messageId: ...as above...,
+    dbId: ...as above...
+  }
+  ```
+
+  Response:
+
+  ```
+  {
+    type: "export",
+    messageId: ...as above...,
+    dbId: ...as above...
+    result: {
+      byteArray: Uint8Array (as per sqlite3_js_db_export()),
+      filename: the db filename,
+      mimetype: "application/x-sqlite3"
+    }
+  }
+  ```
+
 */
 globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
 sqlite3.initWorker1API = function(){
@@ -0,0 +1,86 @@
+<!doctype html>
+<html lang="en-us">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon">
+    <link rel="stylesheet" href="common/testing.css"/>
+    <title>sqlite3-api batch SQL runner for the SAHPool VFS</title>
+  </head>
+  <body>
+    <header id='titlebar'><span>sqlite3-api batch SQL runner for the SAHPool VFS</span></header>
+    <div>
+      <span class='input-wrapper'>
+        <input type='checkbox' class='disable-during-eval' id='cb-reverse-log-order' checked></input>
+        <label for='cb-reverse-log-order' id='lbl-reverse-log-order'>Reverse log order</label>
+      </span>
+    </div>
+    <div id='test-output' class='reverse'></div>
+    <script>
+      (function(){
+        const E = (sel)=>document.querySelector(sel);
+
+        const eOut = E('#test-output');
+        const log2 = function(cssClass,...args){
+          let ln;
+          if(1 || cssClass){
+            ln = document.createElement('div');
+            if(cssClass) ln.classList.add(cssClass);
+            ln.append(document.createTextNode(args.join(' ')));
+          }else{
+            // This doesn't work with the "reverse order" option!
+            ln = document.createTextNode(args.join(' ')+'\n');
+          }
+          eOut.append(ln);
+        };
+        const log = (...args)=>{
+          //console.log(...args);
+          log2('', ...args);
+        };
+        const logErr = function(...args){
+          console.error(...args);
+          log2('error', ...args);
+        };
+        const logWarn = function(...args){
+          console.warn(...args);
+          log2('warning', ...args);
+        };
+
+        const cbReverseLog = E('#cb-reverse-log-order');
+        const lblReverseLog = E('#lbl-reverse-log-order');
+        if(cbReverseLog.checked){
+            lblReverseLog.classList.add('warning');
+            eOut.classList.add('reverse');
+        }
+        cbReverseLog.addEventListener('change', function(){
+            if(this.checked){
+                eOut.classList.add('reverse');
+                lblReverseLog.classList.add('warning');
+            }else{
+                eOut.classList.remove('reverse');
+                lblReverseLog.classList.remove('warning');
+            }
+        }, false);
+
+        const w = new Worker('batch-runner-sahpool.js?sqlite3.dir=jswasm');
+        w.onmessage = function(msg){
+          msg = msg.data;
+          switch(msg.type){
+            case 'stdout': log(...msg.data); break;
+            case 'warn': logWarn(...msg.data); break;
+            case 'error': logErr(...msg.data); break;
+            default:
+              logErr("Unhandled worker message type:",msg);
+              break;
+          }
+        };
+      })();
+    </script>
+    <style>
+      #test-output {
+          white-space: break-spaces;
+          overflow: auto;
+      }
+    </style>
+  </body>
+</html>
@@ -0,0 +1,341 @@
+/*
+  2023-11-30
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  A basic batch SQL runner for the SAHPool VFS. This file must be run in
+  a worker thread. This is not a full-featured app, just a way to get some
+  measurements for batch execution of SQL for the OPFS SAH Pool VFS.
+*/
+'use strict';
+
+const wMsg = function(msgType,...args){
+  postMessage({
+    type: msgType,
+    data: args
+  });
+};
+const toss = function(...args){throw new Error(args.join(' '))};
+const warn = (...args)=>{ wMsg('warn',...args); };
+const error = (...args)=>{ wMsg('error',...args); };
+const log = (...args)=>{ wMsg('stdout',...args); }
+let sqlite3;
+const urlParams = new URL(globalThis.location.href).searchParams;
+const cacheSize = (()=>{
+  if(urlParams.has('cachesize')) return +urlParams.get('cachesize');
+  return 200;
+})();
+
+
+/** Throws if the given sqlite3 result code is not 0. */
+const checkSqliteRc = (dbh,rc)=>{
+  if(rc) toss("Prepare failed:",sqlite3.capi.sqlite3_errmsg(dbh));
+};
+
+const sqlToDrop = [
+  "SELECT type,name FROM sqlite_schema ",
+  "WHERE name NOT LIKE 'sqlite\\_%' escape '\\' ",
+  "AND name NOT LIKE '\\_%' escape '\\'"
+].join('');
+
+const clearDbSqlite = function(db){
+  // This would be SO much easier with the oo1 API, but we specifically want to
+  // inject metrics we can't get via that API, and we cannot reliably (OPFS)
+  // open the same DB twice to clear it using that API, so...
+  const rc = sqlite3.wasm.exports.sqlite3_wasm_db_reset(db.handle);
+  log("reset db rc =",rc,db.id, db.filename);
+};
+
+const App = {
+  db: undefined,
+  cache:Object.create(null),
+  log: log,
+  warn: warn,
+  error: error,
+  metrics: {
+    fileCount: 0,
+    runTimeMs: 0,
+    prepareTimeMs: 0,
+    stepTimeMs: 0,
+    stmtCount: 0,
+    strcpyMs: 0,
+    sqlBytes: 0
+  },
+  fileList: undefined,
+  execSql: async function(name,sql){
+    const db = this.db;
+    const banner = "========================================";
+    this.log(banner,
+             "Running",name,'('+sql.length,'bytes)');
+    const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm;
+    let pStmt = 0, pSqlBegin;
+    const metrics = db.metrics = Object.create(null);
+    metrics.prepTotal = metrics.stepTotal = 0;
+    metrics.stmtCount = 0;
+    metrics.malloc = 0;
+    metrics.strcpy = 0;
+    if(this.gotErr){
+      this.error("Cannot run SQL: error cleanup is pending.");
+      return;
+    }
+    // Run this async so that the UI can be updated for the above header...
+    const endRun = ()=>{
+      metrics.evalSqlEnd = performance.now();
+      metrics.evalTimeTotal = (metrics.evalSqlEnd - metrics.evalSqlStart);
+      this.log("metrics:",JSON.stringify(metrics, undefined, ' '));
+      this.log("prepare() count:",metrics.stmtCount);
+      this.log("Time in prepare_v2():",metrics.prepTotal,"ms",
+               "("+(metrics.prepTotal / metrics.stmtCount),"ms per prepare())");
+      this.log("Time in step():",metrics.stepTotal,"ms",
+               "("+(metrics.stepTotal / metrics.stmtCount),"ms per step())");
+      this.log("Total runtime:",metrics.evalTimeTotal,"ms");
+      this.log("Overhead (time - prep - step):",
+               (metrics.evalTimeTotal - metrics.prepTotal - metrics.stepTotal)+"ms");
+      this.log(banner,"End of",name);
+      this.metrics.prepareTimeMs += metrics.prepTotal;
+      this.metrics.stepTimeMs += metrics.stepTotal;
+      this.metrics.stmtCount += metrics.stmtCount;
+      this.metrics.strcpyMs += metrics.strcpy;
+      this.metrics.sqlBytes += sql.length;
+    };
+
+    const runner = function(resolve, reject){
+      ++this.metrics.fileCount;
+      metrics.evalSqlStart = performance.now();
+      const stack = wasm.scopedAllocPush();
+      try {
+        let t, rc;
+        let sqlByteLen = sql.byteLength;
+        const [ppStmt, pzTail] = wasm.scopedAllocPtr(2);
+        t = performance.now();
+        pSqlBegin = wasm.scopedAlloc( sqlByteLen + 1/*SQL + NUL*/) || toss("alloc(",sqlByteLen,") failed");
+        metrics.malloc = performance.now() - t;
+        metrics.byteLength = sqlByteLen;
+        let pSql = pSqlBegin;
+        const pSqlEnd = pSqlBegin + sqlByteLen;
+        t = performance.now();
+        wasm.heap8().set(sql, pSql);
+        wasm.poke(pSql + sqlByteLen, 0);
+        //log("SQL:",wasm.cstrToJs(pSql));
+        metrics.strcpy = performance.now() - t;
+        let breaker = 0;
+        while(pSql && wasm.peek8(pSql)){
+          wasm.pokePtr(ppStmt, 0);
+          wasm.pokePtr(pzTail, 0);
+          t = performance.now();
+          rc = capi.sqlite3_prepare_v2(
+            db.handle, pSql, sqlByteLen, ppStmt, pzTail
+          );
+          metrics.prepTotal += performance.now() - t;
+          checkSqliteRc(db.handle, rc);
+          pStmt = wasm.peekPtr(ppStmt);
+          pSql = wasm.peekPtr(pzTail);
+          sqlByteLen = pSqlEnd - pSql;
+          if(!pStmt) continue/*empty statement*/;
+          ++metrics.stmtCount;
+          t = performance.now();
+          rc = capi.sqlite3_step(pStmt);
+          capi.sqlite3_finalize(pStmt);
+          pStmt = 0;
+          metrics.stepTotal += performance.now() - t;
+          switch(rc){
+            case capi.SQLITE_ROW:
+            case capi.SQLITE_DONE: break;
+            default: checkSqliteRc(db.handle, rc); toss("Not reached.");
+          }
+        }
+        resolve(this);
+      }catch(e){
+        if(pStmt) capi.sqlite3_finalize(pStmt);
+        this.gotErr = e;
+        reject(e);
+      }finally{
+        capi.sqlite3_exec(db.handle,"rollback;",0,0,0);
+        wasm.scopedAllocPop(stack);
+      }
+    }.bind(this);
+    const p = new Promise(runner);
+    return p.catch(
+      (e)=>this.error("Error via execSql("+name+",...):",e.message)
+    ).finally(()=>{
+      endRun();
+    });
+  },
+
+  /**
+     Loads batch-runner.list and populates the selection list from
+     it. Returns a promise which resolves to nothing in particular
+     when it completes. Only intended to be run once at the start
+     of the app.
+  */
+  loadSqlList: async function(){
+    const infile = 'batch-runner.list';
+    this.log("Loading list of SQL files:", infile);
+    let txt;
+    try{
+      const r = await fetch(infile);
+      if(404 === r.status){
+        toss("Missing file '"+infile+"'.");
+      }
+      if(!r.ok) toss("Loading",infile,"failed:",r.statusText);
+      txt = await r.text();
+    }catch(e){
+      this.error(e.message);
+      throw e;
+    }
+    App.fileList = txt.split(/\n+/).filter(x=>!!x);
+    this.log("Loaded",infile);
+  },
+
+  /** Fetch ./fn and return its contents as a Uint8Array. */
+  fetchFile: async function(fn, cacheIt=false){
+    if(cacheIt && this.cache[fn]) return this.cache[fn];
+    this.log("Fetching",fn,"...");
+    let sql;
+    try {
+      const r = await fetch(fn);
+      if(!r.ok) toss("Fetch failed:",r.statusText);
+      sql = new Uint8Array(await r.arrayBuffer());
+    }catch(e){
+      this.error(e.message);
+      throw e;
+    }
+    this.log("Fetched",sql.length,"bytes from",fn);
+    if(cacheIt) this.cache[fn] = sql;
+    return sql;
+  }/*fetchFile()*/,
+
+  /**
+     Converts this.metrics() to a form which is suitable for easy conversion to
+     CSV. It returns an array of arrays. The first sub-array is the column names.
+     The 2nd and subsequent are the values, one per test file (only the most recent
+     metrics are kept for any given file).
+  */
+  metricsToArrays: function(){
+    const rc = [];
+    Object.keys(this.dbs).sort().forEach((k)=>{
+      const d = this.dbs[k];
+      const m = d.metrics;
+      delete m.evalSqlStart;
+      delete m.evalSqlEnd;
+      const mk = Object.keys(m).sort();
+      if(!rc.length){
+        rc.push(['db', ...mk]);
+      }
+      const row = [k.split('/').pop()/*remove dir prefix from filename*/];
+      rc.push(row);
+      row.push(...mk.map((kk)=>m[kk]));
+    });
+    return rc;
+  },
+
+  metricsToBlob: function(colSeparator='\t'){
+    const ar = [], ma = this.metricsToArrays();
+    if(!ma.length){
+      this.error("Metrics are empty. Run something.");
+      return;
+    }
+    ma.forEach(function(row){
+      ar.push(row.join(colSeparator),'\n');
+    });
+    return new Blob(ar);
+  },
+
+  /**
+     Fetch file fn and eval it as an SQL blob. This is an async
+     operation and returns a Promise which resolves to this
+     object on success.
+  */
+  evalFile: async function(fn){
+    const sql = await this.fetchFile(fn);
+    return this.execSql(fn,sql);
+  }/*evalFile()*/,
+
+  /**
+     Fetches the handle of the db associated with
+     this.e.selImpl.value, opening it if needed.
+  */
+  initDb: function(){
+    const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm;
+    const stack = wasm.scopedAllocPush();
+    let pDb = 0;
+    const d = Object.create(null);
+    d.filename = "/batch.db";
+    try{
+      const oFlags = capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE;
+      const ppDb = wasm.scopedAllocPtr();
+      const rc = capi.sqlite3_open_v2(d.filename, ppDb, oFlags, this.PoolUtil.vfsName);
+      pDb = wasm.peekPtr(ppDb)
+      if(rc) toss("sqlite3_open_v2() failed with code",rc);
+      capi.sqlite3_exec(pDb, "PRAGMA cache_size="+cacheSize, 0, 0, 0);
+      this.log("cache_size =",cacheSize);
+    }catch(e){
+      if(pDb) capi.sqlite3_close_v2(pDb);
+      throw e;
+    }finally{
+      wasm.scopedAllocPop(stack);
+    }
+    d.handle = pDb;
+    this.log("Opened db:",d.filename,'@',d.handle);
+    return d;
+  },
+
+  closeDb: function(){
+    if(this.db.handle){
+      this.sqlite3.capi.sqlite3_close_v2(this.db.handle);
+      this.db.handle = undefined;
+    }
+  },
+
+  run: async function(sqlite3){
+    delete this.run;
+    this.sqlite3 = sqlite3;
+    const capi = sqlite3.capi, wasm = sqlite3.wasm;
+    this.log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid());
+    this.log("WASM heap size =",wasm.heap8().length);
+    let timeStart;
+    sqlite3.installOpfsSAHPoolVfs({
+      clearOnInit: true, initialCapacity: 4,
+      name: 'batch-sahpool',
+      verbosity: 2
+    }).then(PoolUtil=>{
+      App.PoolUtil = PoolUtil;
+      App.db = App.initDb();
+    })
+      .then(async ()=>this.loadSqlList())
+      .then(async ()=>{
+        timeStart = performance.now();
+        for(let i = 0; i < App.fileList.length; ++i){
+          const fn = App.fileList[i];
+          await App.evalFile(fn);
+          if(App.gotErr) throw App.gotErr;
+        }
+      })
+      .then(()=>{
+        App.metrics.runTimeMs = performance.now() - timeStart;
+        App.log("total metrics:",JSON.stringify(App.metrics, undefined, ' '));
+        App.log("Reload the page to run this again.");
+        App.closeDb();
+        App.PoolUtil.removeVfs();
+      })
+      .catch(e=>this.error("ERROR:",e));
+  }/*run()*/
+}/*App*/;
+
+let sqlite3Js = 'sqlite3.js';
+if(urlParams.has('sqlite3.dir')){
+  sqlite3Js = urlParams.get('sqlite3.dir') + '/' + sqlite3Js;
+}
+importScripts(sqlite3Js);
+globalThis.sqlite3InitModule().then(async function(sqlite3_){
+  log("Done initializing. Running batch runner...");
+  sqlite3 = sqlite3_;
+  App.run(sqlite3_);
+});
@@ -72,7 +72,6 @@
    App.logHtml("reset db rc =",rc,db.id, db.filename);
  };

-  
  const E = (s)=>document.querySelector(s);
  const App = {
    e: {
@@ -91,6 +90,15 @@
    db: Object.create(null),
    dbs: Object.create(null),
    cache:{},
+    metrics: {
+      fileCount: 0,
+      runTimeMs: 0,
+      prepareTimeMs: 0,
+      stepTimeMs: 0,
+      stmtCount: 0,
+      strcpyMs: 0,
+      sqlBytes: 0
+    },
    log: console.log.bind(console),
    warn: console.warn.bind(console),
    cls: function(){this.e.output.innerHTML = ''},
@@ -117,7 +125,6 @@
                   "Running",name,'('+sql.length,'bytes) using',db.id);
      const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm;
      let pStmt = 0, pSqlBegin;
-      const stack = wasm.scopedAllocPush();
      const metrics = db.metrics = Object.create(null);
      metrics.prepTotal = metrics.stepTotal = 0;
      metrics.stmtCount = 0;
@@ -142,6 +149,11 @@
        this.logHtml("Overhead (time - prep - step):",
                     (metrics.evalTimeTotal - metrics.prepTotal - metrics.stepTotal)+"ms");
        this.logHtml(banner,"End of",name);
+        this.metrics.prepareTimeMs += metrics.prepTotal;
+        this.metrics.stepTimeMs += metrics.stepTotal;
+        this.metrics.stmtCount += metrics.stmtCount;
+        this.metrics.strcpyMs += metrics.strcpy;
+        this.metrics.sqlBytes += sql.length;
      };

      let runner;
@@ -214,7 +226,9 @@
        }.bind(this);
      }else{/*sqlite3 db...*/
        runner = function(resolve, reject){
+          ++this.metrics.fileCount;
          metrics.evalSqlStart = performance.now();
+          const stack = wasm.scopedAllocPush();
          try {
            let t;
            let sqlByteLen = sql.byteLength;
@@ -269,7 +283,7 @@
      let p;
      if(1){
        p = new Promise(function(res,rej){
-          setTimeout(()=>runner(res, rej), 50)/*give UI a chance to output the "running" banner*/;
+          setTimeout(()=>runner(res, rej), 0)/*give UI a chance to output the "running" banner*/;
        });
      }else{
        p = new Promise(runner);
@@ -401,7 +415,7 @@
      });
      return new Blob(ar);
    },
-    
+
    downloadMetrics: function(){
      const b = this.metricsToBlob();
      if(!b) return;
@@ -576,6 +590,8 @@
        const timeTotal = performance.now() - timeStart;
        who.logHtml("Run-remaining time:",timeTotal,"ms ("+(timeTotal/1000/60)+" minute(s))");
        who.clearStorage();
+        App.metrics.runTimeMs = timeTotal;
+        who.logHtml("Total metrics:",JSON.stringify(App.metrics,undefined,'  '));
      }, false);
    }/*run()*/
  }/*App*/;
@@ -20,7 +20,7 @@
     the main (UI) thread.
  */
  let logHtml;
-  if(self.window === self /* UI thread */){
+  if(globalThis.window === globalThis /* UI thread */){
    console.log("Running demo from main UI thread.");
    logHtml = function(cssClass,...args){
      const ln = document.createElement('div');
@@ -250,7 +250,7 @@
  }/*demo1()*/;

  log("Loading and initializing sqlite3 module...");
-  if(self.window!==self) /*worker thread*/{
+  if(globalThis.window!==globalThis) /*worker thread*/{
    /*
      If sqlite3.js is in a directory other than this script, in order
      to get sqlite3.js to resolve sqlite3.wasm properly, we have to
@@ -262,19 +262,20 @@
      that's not needed.

      URL arguments passed as part of the filename via importScripts()
-      are simply lost, and such scripts see the self.location of
+      are simply lost, and such scripts see the globalThis.location of
      _this_ script.
    */
    let sqlite3Js = 'sqlite3.js';
-    const urlParams = new URL(self.location.href).searchParams;
+    const urlParams = new URL(globalThis.location.href).searchParams;
    if(urlParams.has('sqlite3.dir')){
      sqlite3Js = urlParams.get('sqlite3.dir') + '/' + sqlite3Js;
    }
    importScripts(sqlite3Js);
  }
-  self.sqlite3InitModule({
-    // We can redirect any stdout/stderr from the module
-    // like so...
+  globalThis.sqlite3InitModule({
+    /* We can redirect any stdout/stderr from the module like so, but
+       note that doing so makes use of Emscripten-isms, not
+       well-defined sqlite APIs. */
    print: log,
    printErr: error
  }).then(function(sqlite3){
@@ -46,6 +46,9 @@
          file:// URLs.</li>
        <li>Any OPFS-related pages or tests require:
          <ul>
+            <li>An OPFS-capable browser released after February
+              2023. Some tests will work with Chromium-based browsers
+              going back to around v102.</li>
            <li>That the web server emit the so-called
              <a href='https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cross-Origin-Opener-Policy'>COOP</a>
              and
@@ -53,12 +56,6 @@
              headers. <a href='https://sqlite.org/althttpd'>althttpd</a> requires the
              <code>-enable-sab</code> flag for that.
            </li>
-            <li>A very recent version of a Chromium-based browser
-              (v102 at least, possibly newer). OPFS support in the
-              other major browsers is pending. Development and testing
-              is currently done against a dev-channel release of
-              Chrome (v111 as of 2023-02-10).
-            </li>
          </ul>
        </li>
      </ul>
@@ -31,6 +31,9 @@
          file:// URLs.</li>
        <li>Any OPFS-related pages or tests require:
          <ul>
+            <li>An OPFS-capable browser released after February
+              2023. Some tests will work with Chromium-based browsers
+              going back to around v102.</li>
            <li>That the web server emit the so-called
              <a href='https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cross-Origin-Opener-Policy'>COOP</a>
              and
@@ -38,12 +41,6 @@
              headers. <a href='https://sqlite.org/althttpd'>althttpd</a> requires the
              <code>-enable-sab</code> flag for that.
            </li>
-            <li>A very recent version of a
-              Chromium-based browser (v102 at least, possibly newer). OPFS
-              support in the other major browsers is pending. Development
-              and testing is currently done against a dev-channel release
-              of Chrome (v111 as of 2023-02-10).
-            </li>
          </ul>
        </li>
      </ul>
@@ -350,7 +350,7 @@
                eControls.classList.remove('hidden');
                break;
            case 'stdout': log(msg.data); break;
-            case 'stdout': logErr(msg.data); break;
+            case 'stderr': logErr(msg.data); break;
            case 'run-start':
                eControls.disabled = true;
                log("Running speedtest1 with argv =",msg.data.join(' '));
@@ -7,9 +7,9 @@
  }
  importScripts(speedtestJs);
  /**
-     If this environment contains OPFS, this function initializes it and
-     returns the name of the dir on which OPFS is mounted, else it returns
-     an empty string.
+     If this build includes WASMFS, this function initializes it and
+     returns the name of the dir on which OPFS is mounted, else it
+     returns an empty string.
  */
  const wasmfsDir = function f(wasmUtil){
    if(undefined !== f._) return f._;
@@ -47,6 +47,7 @@
  };
  const log = (...args)=>logMsg('stdout',args);
  const logErr = (...args)=>logMsg('stderr',args);
+  const realSahName = 'opfs-sahpool-speedtest1';

  const runSpeedtest = async function(cliFlagsArray){
    const scope = App.wasm.scopedAllocPush();
@@ -57,7 +58,6 @@
      ];
      App.logBuffer.length = 0;
      const ndxSahPool = argv.indexOf('opfs-sahpool');
-      const realSahName = 'opfs-sahpool-speedtest1';
      if(ndxSahPool>0){
        argv[ndxSahPool] = realSahName;
        log("Updated argv for opfs-sahpool: --vfs",realSahName);
@@ -73,7 +73,7 @@
          clearOnInit: true,
          verbosity: 2
        }).then(PoolUtil=>{
-          log("opfs-sahpool successfully installed as",realSahName);
+          log("opfs-sahpool successfully installed as",PoolUtil.vfsName);
          App.sqlite3.$SAHPoolUtil = PoolUtil;
          //console.log("sqlite3.oo1.OpfsSAHPoolDb =", App.sqlite3.oo1.OpfsSAHPoolDb);
        });
@@ -102,19 +102,6 @@
    }
  };

-  const sahpSanityChecks = function(sqlite3){
-    log("Attempting OpfsSAHPoolDb sanity checks...");
-    const db = new sqlite3.oo1.OpfsSAHPoolDb('opfs-sahpoool.db');
-    const fn = db.filename;
-    db.exec([
-      'create table t(a);',
-      'insert into t(a) values(1),(2),(3);'
-    ]);
-    db.close();
-    sqlite3.wasm.sqlite3_wasm_vfs_unlink(sqlite3_vfs_find("opfs-sahpool"), fn);
-    log("SAH sanity checks done.");
-  };
-
  const EmscriptenModule = {
    print: log,
    printErr: logErr,
@@ -22,7 +22,7 @@ const tryOpfsVfs = async function(sqlite3){
  const opfs = sqlite3.opfs;
  log("tryOpfsVfs()");
  if(!sqlite3.opfs){
-    const e = toss("OPFS is not available.");
+    const e = new Error("OPFS is not available.");
    error(e);
    throw e;
  }
@@ -1688,7 +1688,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
          wasm.sqlite3_wasm_vfs_unlink(0, filename);
        }
      }
-    }/*sqlite3_js_vfs_create_file()*/)
+    }/*sqlite3_js_posix_create_file()*/)

  ////////////////////////////////////////////////////////////////////
    .t({
@@ -2605,28 +2605,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
        }
      }
    }/*kvvfs sanity checks*/)
-    .t({
-      name: 'kvvfs sqlite3_js_vfs_create_file()',
-      predicate: ()=>"kvvfs does not currently support this",
-      test: function(sqlite3){
-        let db;
-        try {
-          db = new this.JDb(this.kvvfsDbFile);
-          const exp = capi.sqlite3_js_db_export(db);
-          db.close();
-          this.kvvfsUnlink();
-          capi.sqlite3_js_vfs_create_file("kvvfs", this.kvvfsDbFile, exp);
-          db = new this.JDb(filename);
-          T.assert(6 === db.selectValue('select count(*) from kvvfs'));
-        }finally{
-          db.close();
-          this.kvvfsUnlink();
-        }
-        delete this.kvvfsDbFile;
-        delete this.kvvfsUnlink;
-        delete this.JDb;
-      }
-   }/*kvvfs sqlite3_js_vfs_create_file()*/)
  ;/* end kvvfs tests */

  ////////////////////////////////////////////////////////////////////////
@@ -2644,13 +2622,17 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
        T.assert( 0 === rc /*void pointer*/ );

        // Commit hook...
+        T.assert( 0!=capi.sqlite3_get_autocommit(db) );
        db.exec("BEGIN; SELECT 1; COMMIT");
        T.assert(0 === countCommit,
                 "No-op transactions (mostly) do not trigger commit hook.");
        db.exec("BEGIN EXCLUSIVE; SELECT 1; COMMIT");
        T.assert(1 === countCommit,
                 "But EXCLUSIVE transactions do.");
-        db.transaction((d)=>{d.exec("create table t(a)");});
+        db.transaction((d)=>{
+          T.assert( 0==capi.sqlite3_get_autocommit(db) );
+          d.exec("create table t(a)");
+        });
        T.assert(2 === countCommit);

        // Rollback hook:
@@ -1,13 +1,13 @@
-C Version\s3.44.2
-D 2023-11-24T11:41:44.921
+C Add\sNEVER\sto\stwo\sunreachable\sbranches\sin\sJSON.
+D 2023-12-12T17:13:10.122
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
-F Makefile.in bddd493839ce057414b19cd405293363f7efb7a73a3d8f6d2ed1b59943ddfcb1
+F Makefile.in 890caf094636c308bc981032baf8f9208bf307755f9197ae4218a9fbcec2e449
 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6
 F Makefile.msc 59bb36dba001f0b38212be0794fb838f25371008b180929bcf08aa799694c168
-F README.md 963d30019abf0cc06b263cd2824bce022893f3f93a531758f6f04ff2194a16a8
-F VERSION 672a5ddf487d19856d8b1015797e14578fd8f8d59c14d9832ab0fe5938c6ab9b
+F README.md 6358805260a03ebead84e168bbf3740ddf3f683b477e478567186aa7afb490d3
+F VERSION 73573d4545343f001bf5dc5461173a7c78c203dd046cabcf99153878cf25d3a6
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
@@ -22,7 +22,7 @@ F autoconf/configure.ac ec7fa914c5e74ff212fe879f9bb6918e1234497e05facfb641f30c4d
 F autoconf/tea/Makefile.in 106a96f2f745d41a0f6193f1de98d7355830b65d45032c18cd7c90295ec24196
 F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873
 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43
-F autoconf/tea/configure.ac 0c358d924fe9cd5c624b9038e025bca35b572a15501014c69afe38b8ae4d7371
+F autoconf/tea/configure.ac 4c32b08691a5b296206b38422b53b92b65be3d3f6b3dd6552a50981a61f5acda
 F autoconf/tea/doc/sqlite3.n e1fe45d4f5286ee3d0ccc877aca2a0def488e9bb
 F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523
 F autoconf/tea/pkgIndex.tcl.in b9eb6dd37f64e08e637d576b3c83259814b9cddd78bec4af2e5abfc6c5c750ce
@@ -33,15 +33,15 @@ F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea0034
 F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63
 F config.guess 883205ddf25b46f10c181818bf42c09da9888884af96f79e1719264345053bd6
 F config.sub c2d0260f17f3e4bc0b6808fccf1b291cb5e9126c14fc5890efc77b9fd0175559
-F configure a56c8f24e118eff90e3870411bfc844e86c00577faf092ded131120a637ad2da x
-F configure.ac de31fea7d975bb7ebafbe0e2190a855cc80d48558bf0c9a6578a1836daf1cd3a
+F configure bcb1042e92775424a1021d2f4c89c78a699a6225df01fa8c593df7df0be6ad10 x
+F configure.ac f25bd7843120f2c2b8bc9db5a92b0502bbdd28e68907415c3d42fc8e57c657b9
 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
 F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd
-F doc/compile-for-windows.md 922ba580d210a1f1bd3ef9d0413121556f9b5714fb5c01e664d980f85fa4ac8c
+F doc/compile-for-windows.md 50b27d77be96195c66031a3181cb8684ed822327ea834e07f9c014213e5e3bcf
 F doc/json-enhancements.md e356fc834781f1f1aa22ee300027a270b2c960122468499bf347bb123ce1ea4f
 F doc/lemon.html 44a53a1d2b42d7751f7b2f478efb23c978e258d794bfd172442307a755b9fa44
 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710
-F doc/testrunner.md 2434864be2219d4f0b6ffc99d0a2172d531c4ca4345340776f67ad4edd90dc90
+F doc/testrunner.md 8d36ec692cf4994bb66d84a4645b9afa1ce9d47dc12cbf8d437c5a5fb6ddeedb
 F doc/trusted-schema.md 33625008620e879c7bcfbbfa079587612c434fa094d338b08242288d358c3e8a
 F doc/vdbesort-memory.md 4da2639c14cd24a31e0af694b1a8dd37eaf277aff3867e9a8cc14046bc49df56
 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a
@@ -50,8 +50,8 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 6f247666b495c477628dd19364d279c78ea48cd90c72d9f9b98ad1aff3294f94
 F ext/async/sqlite3async.h 46b47c79357b97ad85d20d2795942c0020dc20c532114a49808287f04aa5309a
-F ext/consio/console_io.c e048bd69f3a357a4b6b5da01c504728631d944ad5735ad83c44800c80a91e978 x
-F ext/consio/console_io.h d18bcc50f4e4d42b65be8fecba72918364ae77182c86360aca842e44b823384e
+F ext/consio/console_io.c 097323f60037d70523421f0248958d2280851f8ff2f9a443d4ee1474c4769118 x
+F ext/consio/console_io.h 3228dff1717481202a24f6dcf45ce0b75e7a778bf6877089518a44e1473b76a3
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4
 F ext/expert/expert1.test 0dd5cb096d66bed593e33053a3b364f6ef52ed72064bf5cf298364636dbf3cd6
@@ -88,28 +88,28 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
-F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
-F ext/fts5/fts5.h 05501612cc655504c5dce8ba765ab621d50fc478490089beaa0d75e00b23e520
-F ext/fts5/fts5Int.h 78a63cc0795186cde5384816a9403a68c65774b35d952e05b81a1b4b158e07c8
-F ext/fts5/fts5_aux.c 35c4101613eff86902877a4dedd9400b07922e412cbdd637b45041dce2fd5388
+F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e
+F ext/fts5/fts5.h 9d7867cc63631296462c90bafe57c4881b56e480fa510ffaee6a77049258c714
+F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880e9834
+F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
-F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081
-F ext/fts5/fts5_expr.c bd3b81ce669c4104e34ffe66570af1999a317b142c15fccb112de9fb0caa57a6
-F ext/fts5/fts5_hash.c 076058f93327051952a752dc765df1acfe783eb11b419b30652aa1fc1f987902
-F ext/fts5/fts5_index.c 458cbed8a3e17617cbf7e80cdfb7612000b9bb3781f286b345fb9655858658cf
-F ext/fts5/fts5_main.c a07ed863b8bd9e6fefb62db2fd40a3518eb30a5f7dcfda5be915dd2db45efa2f
+F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf
+F ext/fts5/fts5_expr.c b1ec526371b9ffde82341423a5b9753c42cbea629a41b69f26fa377d13b95a8e
+F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
+F ext/fts5/fts5_index.c ed206045ff0f2226d870fa41fba45f738c0cc953ab74ba68477091b9a574ccd3
+F ext/fts5/fts5_main.c b908696c52410e8383019ac0657c8a5cd0c8f60e78edc169e9b3c4b93f24c933
 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
-F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
+F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b
-F ext/fts5/fts5_tokenize.c 5e251efb0f1af99a25ed50010ba6b1ad1250aca5921af1988fdcabe5ebc3cb43
+F ext/fts5/fts5_tokenize.c 83cfcede3898001cab84432a36ce1503e3080cf9b1c682b022ec82e267ea4c13
 F ext/fts5/fts5_unicode2.c eca63dbc797f8ff0572e97caf4631389c0ab900d6364861b915bdd4735973f00
 F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80
-F ext/fts5/fts5_vocab.c aed56169ae5c1aa9b8189c779ffeef04ed516d3c712c06914e6d91a6759f4e4a
+F ext/fts5/fts5_vocab.c 209e0c151e108d5f3621fa24b91e9b02f3750ee6c3f9ccec312df39481b68a09
 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
-F ext/fts5/test/fts5_common.tcl a9de9c2209cc4e7ae3c753e783504e67206c6c1467d08f209cd0c5923d3e8d8b
-F ext/fts5/test/fts5aa.test ba5158eba7d61359becdfca895ef471072c7bf7b20e5e60dcb4d024c8419c926
+F ext/fts5/test/fts5_common.tcl 8b1848ac2baad10e444e4183034a52050b52d20b3796d9d30e78f01ab0d05583
+F ext/fts5/test/fts5aa.test 4db81519863244a3cab35795fe65ab6b592e7970c7409eba098b23ebbfc08d95
 F ext/fts5/test/fts5ab.test bd932720c748383277456b81f91bc00453de2174f9762cd05f95d0495dc50390
 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d
 F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de
@@ -145,7 +145,7 @@ F ext/fts5/test/fts5corrupt.test b6d4034b682bb3387bc44c510c71b3c67d4349e4df13949
 F ext/fts5/test/fts5corrupt2.test 99e7e23a58b4d89eb7167c6de1669cbc595cd3c79ab333e0eb56405473319e77
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
 F ext/fts5/test/fts5corrupt4.test f4c08e2182a48d8b70975fd869ee5391855c06d8a0ff87b6a2529e7c5a88a1d3
-F ext/fts5/test/fts5corrupt5.test 38a238df26c4de471e1c4b98f8de6c902bc692577a1c08d0ff4f2251f3559644
+F ext/fts5/test/fts5corrupt5.test f9dbed6785c38d891c0b947fd6e973e9c4eb44991e1322f8d7e5281a019276f2
 F ext/fts5/test/fts5corrupt6.test bf8eeae07825b088b9665d9d8e4accbd8dc9bf3cb85b6c64cf6c9e18ccc420a4
 F ext/fts5/test/fts5corrupt7.test 80ad7f683a8bda2404731bb77e8c3dbbb620c1f6cc583cca8239f6accd6338c0
 F ext/fts5/test/fts5delete.test 619295b20dbc1d840b403ee07c878f52378849c3c02e44f2ee143b3e978a0aa7
@@ -170,6 +170,7 @@ F ext/fts5/test/fts5faultD.test e7ed7895abfe6bc98a5e853826f6b74956e7ba7f594f1860
 F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e4710c77eb8ce7075
 F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1
 F ext/fts5/test/fts5faultG.test d2e5a4d9a34e08dcaadcaeafef74d10cbc2abdd11aa2659a18af0294bf2812d3
+F ext/fts5/test/fts5faultH.test 57f53c87ffd59be0265840f2b54a16811f9cb9012db86aad9b41d0d14d85dfe3
 F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b996159f6909dc8079
 F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717
 F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e
@@ -182,7 +183,7 @@ F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fc
 F ext/fts5/test/fts5matchinfo.test 10c9a6f7fe61fb132299c4183c012770b10c4d5c2f2edb6df0b6607f683d737a
 F ext/fts5/test/fts5merge.test e92a8db28b45931e7a9c7b1bbd36101692759d00274df74d83fd29d25d53b3a6
 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2
-F ext/fts5/test/fts5misc.test 5ca82f2a5ee016b0842043155d1382f98a34d0d86b2791165a44d7807f6e0f54
+F ext/fts5/test/fts5misc.test dd97c86c9cbc3e587067e640f6ce88842cfbf5d23bb0e0fbb7f6707623b2d505
 F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581
 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45
 F ext/fts5/test/fts5near.test 211477940142d733ac04fad97cb24095513ab2507073a99c2765c3ddd2ef58bd
@@ -190,6 +191,11 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618
 F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1
 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785
 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca
+F ext/fts5/test/fts5origintext.test d2796fa08ee7aecfabdc0c45bb8a2fb16a00ea8757e63fbc153b718dbe430a39
+F ext/fts5/test/fts5origintext2.test f3b9436de540828d01f0672df855b09ebc0863e126d5b56234701d71dfa73634
+F ext/fts5/test/fts5origintext3.test 0d25933506600452a5ab3873cbb418ed5f2de2446c3672b9997b1ea104b0e7f0
+F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871
+F ext/fts5/test/fts5origintext5.test a037bdf7235a22033c4663837bdb12d9738245464a3ac2f60c71fc40d07ede7d
 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b
 F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a
 F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15
@@ -212,14 +218,16 @@ F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114
 F ext/fts5/test/fts5secure8.test eb3579e9d58b0acad97e8082dee1f99b2d393198f03500b453c2b25761c0c298
 F ext/fts5/test/fts5securefault.test dbca2b6a1c16700017f5051138991b705410889933f2a37c57ae8a23b296b10b
 F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b
-F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0
+F ext/fts5/test/fts5simple2.test 8dd2389ee75e21a1429fe87e5f8c7d9a97ad1470304a8a2d3ba4b8c3c345fecd
 F ext/fts5/test/fts5simple3.test d5c74a9d3ca71bd5dd5cacb7c55b86ea12cdddfc8b1910e3de2995206898380f
 F ext/fts5/test/fts5synonym.test 1651815b8008de170e8e600dcacc17521d765482ea8f074ae82cfa870d8bb7fb
 F ext/fts5/test/fts5synonym2.test 8f891fc49cc1e8daed727051e77e1f42849c784a6a54bef82564761b2cb3e016
 F ext/fts5/test/fts5tok1.test 1f7817499f5971450d8c4a652114b3d833393c8134e32422d0af27884ffe9cef
 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
 F ext/fts5/test/fts5tokenizer.test ac3c9112b263a639fb0508ae73a3ee886bf4866d2153771a8e8a20c721305a43
+F ext/fts5/test/fts5tokenizer2.test cb5428c7cfb3b6a74b7adfcde65506e329112003e8dffa7501d01c2d18d02569
 F ext/fts5/test/fts5trigram.test 6c4e37864f3e7d90673db5563d9736d7e40080ab94d10ebdffa94c1b77941da0
+F ext/fts5/test/fts5trigram2.test 9fe4207f8a4241747aff1005258b564958588d21bfd240d6cd4c2e955d31c156
 F ext/fts5/test/fts5ubsan.test 783d5a8d13ebfa169e634940228db54540780e3ba7a87ad1e4510e61440bf64b
 F ext/fts5/test/fts5umlaut.test a42fe2fe6387c40c49ab27ccbd070e1ae38e07f38d05926482cc0bccac9ad602
 F ext/fts5/test/fts5unicode.test 17056f4efe6b0a5d4f41fdf7a7dc9af2873004562eaa899d40633b93dc95f5a9
@@ -230,7 +238,7 @@ F ext/fts5/test/fts5unindexed.test 9021af86a0fb9fc616f7a69a996db0116e7936d0db638
 F ext/fts5/test/fts5update.test b8affd796e45c94a4d19ad5c26606ea06065a0f162a9562d9f005b5a80ccf0bc
 F ext/fts5/test/fts5version.test d6e5a5897550afeccc2f8531d87404dc1c289ee89385dd4318dbdd75e71d7a67
 F ext/fts5/test/fts5vocab.test 7ed80d9af1ddaaa1637da05e406327b5aac250848bc604c1c1cc667908b87760
-F ext/fts5/test/fts5vocab2.test 681980e92e031c9f3fe8d9c149189e876c108da2fb0fb3a25bd8a9b94bff8f68
+F ext/fts5/test/fts5vocab2.test 1b1f0059f762ffb404213d35dac013e010621f08128589b6ec7bec59d9a710f3
 F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85
 F ext/fts5/tool/fts5txt2db.tcl c0d43c8590656f8240e622b00957b3a0facc49482411a9fdc2870b45c0c82f9f
 F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093
@@ -239,49 +247,50 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c
 F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9
 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282
 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8
-F ext/jni/GNUmakefile 5c3ac326bf3853486ebe0d70819abc790cc65c412182ce4ebd5012b008d9b059
-F ext/jni/README.md ef9ac115e97704ea995d743b4a8334e23c659e5534c3b64065a5405256d5f2f4
+F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2
+F ext/jni/README.md d899789a9082a07b99bf30b1bbb6204ae57c060efcaa634536fa669323918f42
 F ext/jni/jar-dist.make 030aaa4ae71dd86e4ec5e7c1e6cd86f9dfa47c4592c070d2e35157e42498e1fa
-F ext/jni/src/c/sqlite3-jni.c 6f6df9657989e9ca2cfdcc2fe9a71c279de56d5c941adfd09a0f24256de35c8f
-F ext/jni/src/c/sqlite3-jni.h b4c413a0d0c734683da1049cfcf89e35ae2719759d0656ec0f8c57188f18cab8
-F ext/jni/src/org/sqlite/jni/annotation/NotNull.java a99341e88154e70447596b1af6a27c586317df41a7e0f246fd41370cd7b723b2
-F ext/jni/src/org/sqlite/jni/annotation/Nullable.java 0b1879852707f752512d4db9d7edd0d8db2f0c2612316ce1c832715e012ff6ba
+F ext/jni/src/c/sqlite3-jni.c 6040c0de97644a1fb14bb589ee9f2f4208f6e6b165d14a0e33ed24945b118838
+F ext/jni/src/c/sqlite3-jni.h 913ab8e8fee432ae40f0e387c8231118d17053714703f5ded18202912a8a3fbf
+F ext/jni/src/org/sqlite/jni/annotation/Experimental.java 8603498634e41d0f7c70f661f64e05df64376562ea8f126829fd1e0cdd47e82b
+F ext/jni/src/org/sqlite/jni/annotation/NotNull.java 38e7e58a69b26dc100e458b31dfa3b2a7d67bc36d051325526ef1987d5bc8a24
+F ext/jni/src/org/sqlite/jni/annotation/Nullable.java 56e3dee1f3f703a545dfdeddc1c3d64d1581172b1ad01ffcae95c18547fafd90
 F ext/jni/src/org/sqlite/jni/annotation/package-info.java 977b374aed9d5853cbf3438ba3b0940abfa2ea4574f702a2448ee143b98ac3ca
 F ext/jni/src/org/sqlite/jni/capi/AbstractCollationCallback.java 1afa90d3f236f79cc7fcd2497e111992644f7596fbc8e8bcf7f1908ae00acd6c
-F ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java bc29e986c866c2ddbbb9f935f5b7264c1c1026864e50a4a735192864f75e37c0
-F ext/jni/src/org/sqlite/jni/capi/AuthorizerCallback.java 7ed409d5449684616cc924534e22ff6b07d361f12ad904b69ecb10e0568a8013
+F ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java 0b72cdff61533b564d65b63418129656daa9a9f30e7e7be982bd5ab394b1dbd0
+F ext/jni/src/org/sqlite/jni/capi/AuthorizerCallback.java c045a5b47e02bb5f1af91973814a905f12048c428a3504fbc5266d1c1be3de5a
 F ext/jni/src/org/sqlite/jni/capi/AutoExtensionCallback.java 74cc4998a73d6563542ecb90804a3c4f4e828cb4bd69e61226d1a51f4646e759
 F ext/jni/src/org/sqlite/jni/capi/BusyHandlerCallback.java 7b8e19810c42b0ad21a04b5d8c804b32ee5905d137148703f16a75b612c380ca
-F ext/jni/src/org/sqlite/jni/capi/CApi.java bccb442ca81cd4decb1adae99006a60b7a9f54e5153842e738c01104e97d1de0
-F ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java 0bfd6e56e8265c2f05c9207665707285534d78f8466ef0e0430c65677f00943d
+F ext/jni/src/org/sqlite/jni/capi/CApi.java d428a1fd3b827f01c55d10d21ff35e33e7dac9e8a1d92a8b5c7d7255e67407d8
+F ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java 57e2d275dcebe690b1fc1f3d34eb96879b2d7039bce30b563aee547bf45d8a8b
 F ext/jni/src/org/sqlite/jni/capi/CollationCallback.java e29bcfc540fdd343e2f5cca4d27235113f2886acb13380686756d5cabdfd065a
-F ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java f81cf10b79c52f9b2e9247d523d29ae48863935f60420eae35f257c38c80ce95
-F ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java 29c002f3c638cc80f7db1594564a262d1beb32637824c3dca2d60a224d1f71d7
+F ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java 5bfa226a8e7a92e804fd52d6e42b4c7b875fa7a94f8e2c330af8cc244a8920ab
+F ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java 482f53dfec9e3ac2a9070d3fceebd56250932aaaf7c4f5bc8de29fc011416e0c
 F ext/jni/src/org/sqlite/jni/capi/ConfigLogCallback.java b995ca412f59b631803b93aa5b3684fce62e335d1e123207084c054abfd488d4
-F ext/jni/src/org/sqlite/jni/capi/ConfigSqllogCallback.java 701f2e4d8bdeb27cfbeeb56315d15b13d8752b0fdbca705f31bd4366c58d8a33
+F ext/jni/src/org/sqlite/jni/capi/ConfigSqlLogCallback.java e5723900b6458bc6288f52187090a78ebe0a20f403ac7c887ec9061dfe51aba7
 F ext/jni/src/org/sqlite/jni/capi/NativePointerHolder.java b7036dcb1ef1b39f1f36ac605dde0ff1a24a9a01ade6aa1a605039443e089a61
-F ext/jni/src/org/sqlite/jni/capi/OutputPointer.java 68f60aec7aeb5cd4e5fb83449037f668c63cb99f682ee1036cc226d0cbd909b9
-F ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java aca8f9fa72e3b6602bc9a7dd3ae9f5b2808103fbbee9b2749dc96c19cdc261a1
-F ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java 819d938e26208adde17ca4b7ddde1d8cd6915b6ab7b708249a9787beca6bd6b6
+F ext/jni/src/org/sqlite/jni/capi/OutputPointer.java 246b0e66c4603f41c567105a21189d138aaf8c58203ecd4928802333da553e7c
+F ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java 97352091abd7556167f4799076396279a51749fdae2b72a6ba61cd39b3df0359
+F ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java efcf57545c5e282d1dd332fa63329b3b218d98f356ef107a9dbe3979be82213a
 F ext/jni/src/org/sqlite/jni/capi/ProgressHandlerCallback.java 01bc0c238eed2d5f93c73522cb7849a445cc9098c2ed1e78248fa20ed1cfde5b
 F ext/jni/src/org/sqlite/jni/capi/ResultCode.java 8141171f1bcf9f46eef303b9d3c5dc2537a25ad1628f3638398d8a60cacefa7f
-F ext/jni/src/org/sqlite/jni/capi/RollbackHookCallback.java 105e324d09c207100485e7667ad172e64322c62426bb49b547e9b0dc9c33f5f0
-F ext/jni/src/org/sqlite/jni/capi/SQLFunction.java fef556adbc3624292423083a648bdf97fa8a4f6b3b6577c9660dd7bd6a6d3c4a
+F ext/jni/src/org/sqlite/jni/capi/RollbackHookCallback.java e172210a2080e851ebb694c70e9f0bf89284237795e38710a7f5f1b61e3f6787
+F ext/jni/src/org/sqlite/jni/capi/SQLFunction.java 0d1e9afc9ff8a2adb94a155b72385155fa3b8011a5cca0bb3c28468c7131c1a5
 F ext/jni/src/org/sqlite/jni/capi/SQLTester.java 09bee15aa0eedac68d767ae21d9a6a62a31ade59182a3ccbf036d6463d9e30b1
 F ext/jni/src/org/sqlite/jni/capi/ScalarFunction.java 93b9700fca4c68075ccab12fe0fbbc76c91cafc9f368e835b9bd7cd7732c8615
 F ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java addf120e0e76e5be1ff2260daa7ce305ff9b5fafd64153a7a28e9d8f000a815f
-F ext/jni/src/org/sqlite/jni/capi/Tester1.java ca195521b6bda3e0cd00e76bb71ec8060d1fab76a2f13b1af9feea40789f44bb
+F ext/jni/src/org/sqlite/jni/capi/Tester1.java e5fa17301b7266c1cbe4bcce67788e08e45871c7c72c153d515abb37e501de0a
 F ext/jni/src/org/sqlite/jni/capi/TraceV2Callback.java 0a25e117a0daae3394a77f24713e36d7b44c67d6e6d30e9e1d56a63442eef723
-F ext/jni/src/org/sqlite/jni/capi/UpdateHookCallback.java 2766b8526bbffc4f1045f70e79f1bc1b1efe1c3e95ca06cdb8a7391032dda3b4
-F ext/jni/src/org/sqlite/jni/capi/ValueHolder.java 9f9e151f1da017b706c0ee5f40f4c86b54e773d6ae4339723e0cc85a456251ab
+F ext/jni/src/org/sqlite/jni/capi/UpdateHookCallback.java c8bdf7848e6599115d601bcc9427ff902cb33129b9be32870ac6808e04b6ae56
+F ext/jni/src/org/sqlite/jni/capi/ValueHolder.java 2ce069f3e007fdbbe1f4e507a5a407fc9679da31a0aa40985e6317ed4d5ec7b5
 F ext/jni/src/org/sqlite/jni/capi/WindowFunction.java caf4396f91b2567904cf94bc538a069fd62260d975bd037d15a02a890ed1ef9e
 F ext/jni/src/org/sqlite/jni/capi/XDestroyCallback.java f3abb8dd7381f53ebba909437090caf68200f06717b8a7d6aa96fa3e8133117d
 F ext/jni/src/org/sqlite/jni/capi/package-info.java 08ff986a65d2be9162442c82d28a65ce431d826f188520717c2ecb1484d0a50e
-F ext/jni/src/org/sqlite/jni/capi/sqlite3.java 4010bbebc5bf44e2044e610786088cdee7dc155da2b333c0551492ff1cedf33b
+F ext/jni/src/org/sqlite/jni/capi/sqlite3.java c6a5c555d163d76663534f2b2cce7cab15325b9852d0f58c6688a85e73ae52f0
 F ext/jni/src/org/sqlite/jni/capi/sqlite3_backup.java 6742b431cd4d77e8000c1f92ec66265a58414c86bf3b0b5fbcb1164e08477227
-F ext/jni/src/org/sqlite/jni/capi/sqlite3_blob.java f204ab6ab1263e119fe43730141a00662d80972129a5351dfb11aae5d282df36
+F ext/jni/src/org/sqlite/jni/capi/sqlite3_blob.java 59e26ca5254cd4771f467237bcfe2d8deed30a77152fabcd4574fd406c301d63
 F ext/jni/src/org/sqlite/jni/capi/sqlite3_context.java f0ef982009c335c4393ffcb68051809ca1711e4f47bcb8d1d46952f22c01bc22
-F ext/jni/src/org/sqlite/jni/capi/sqlite3_stmt.java ff579621e9bd5ffbc6b2ef9f996c12db4df6e0c8cc5697c91273e5fca279fcf8
+F ext/jni/src/org/sqlite/jni/capi/sqlite3_stmt.java 293b5fa7d5b5724c87de544654aca1103d76f3092bc2c8f4360102a65ba25dff
 F ext/jni/src/org/sqlite/jni/capi/sqlite3_value.java e1d62a257c13504b46d39d5c21c49cf157ad73fda00cc5f34c931aa008c37049
 F ext/jni/src/org/sqlite/jni/fts5/Fts5.java e94681023785f1eff5399f0ddc82f46b035977d350f14838db659236ebdf6b41
 F ext/jni/src/org/sqlite/jni/fts5/Fts5Context.java 338637e6e5a2cc385d962b220f3c1f475cc371d12ae43d18ef27327b6e6225f7
@@ -294,13 +303,14 @@ F ext/jni/src/org/sqlite/jni/fts5/fts5_api.java a8e88c3783d21cec51b0748568a96653
 F ext/jni/src/org/sqlite/jni/fts5/fts5_extension_function.java 9e2b954d210d572552b28aca523b272fae14bd41e318921b22f65b728d5bf978
 F ext/jni/src/org/sqlite/jni/fts5/fts5_tokenizer.java 92bdaa3893bd684533004d64ade23d329843f809cd0d0f4f1a2856da6e6b4d90
 F ext/jni/src/org/sqlite/jni/test-script-interpreter.md f9f25126127045d051e918fe59004a1485311c50a13edbf18c79a6ff9160030e
-F ext/jni/src/org/sqlite/jni/wrapper1/AggregateFunction.java 5ad99bd74c85f56bbef324d9ec29b4048f4620547c9a80093d8586c3557f9f9a
+F ext/jni/src/org/sqlite/jni/wrapper1/AggregateFunction.java d5c108b02afd3c63c9e5e53f71f85273c1bfdc461ae526e0a0bb2b25e4df6483
 F ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java 43c43adfb7866098aadaaca1620028a6ec82d5193149970019b1cce9eb59fb03
-F ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java 004394eeb944baa56e36cd7ae69ba6d4a52b52db3c49439db16e98270b861421
-F ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java a9ddc6a9e8c113168cc67592ae24c0e56d30dd06226eeab012f2761a0889d7bb
-F ext/jni/src/org/sqlite/jni/wrapper1/SqliteException.java 1386f7b753134fc12253ce2fbbc448ba8c970567fac01a3356cb672e14408d73
-F ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java c24b510ebe801c30533cc62efdf69a4a5e2da9ec4b49f8d403f2060693f060a0
-F ext/jni/src/org/sqlite/jni/wrapper1/ValueHolder.java 7b89a7391f771692c5b83b0a5b86266abe8d59f1c77d7a0eccc9b79f259d79af
+F ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java 27b141f5914c7cb0e40e90a301d5e05b77f3bd42236834a68031b7086381fafd
+F ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java ada39f18e4e3e9d4868dadbc3f7bfe1c6c7fde74fb1fb2954c3f0f70120b805c
+F ext/jni/src/org/sqlite/jni/wrapper1/SqliteException.java 982538ddb4c0719ef87dfa664cd137b09890b546029a7477810bd64d4c47ee35
+F ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java ce45f2ec85facbb73690096547ed166e7be82299e3d92eaa206f82b60a6ec969
+F ext/jni/src/org/sqlite/jni/wrapper1/ValueHolder.java a84e90c43724a69c2ecebd601bc8e5139f869b7d08cb705c77ef757dacdd0593
+F ext/jni/src/org/sqlite/jni/wrapper1/WindowFunction.java c7d1452f9ff26175b3c19bbf273116cc2846610af68e01756d755f037fe7319f
 F ext/jni/src/tests/000-000-sanity.test c3427a0e0ac84d7cbe4c95fdc1cd4b61f9ddcf43443408f3000139478c4dc745
 F ext/jni/src/tests/000-001-ignored.test e17e874c6ab3c437f1293d88093cf06286083b65bf162317f91bbfd92f961b70
 F ext/jni/src/tests/900-001-fts.test bf0ce17a8d082773450e91f2388f5bbb2dfa316d0b676c313c637a91198090f0
@@ -456,7 +466,7 @@ F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69
 F ext/rbu/sqlite3rbu.c d4ddf8f0e93772556e452a6c2814063cf47efb760a0834391a9d0cd9859fa4b9
 F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304
 F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055
-F ext/recover/dbdata.c fc7147a68422cbbbaa481ee92ae1752cc25f5a24302bece1c70dcb76345bd736
+F ext/recover/dbdata.c b7746c2ec801b453840831311be4b31f8c8f9dd97791060a69bbf12392c78949
 F ext/recover/recover1.test c484d01502239f11b61f23c1cee9f5dd19fa17617f8974e42e74d64639c524cf
 F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a
 F ext/recover/recoverbuild.test c74170e0f7b02456af41838afeb5353fdb985a48cc2331d661bbabbca7c6b8e3
@@ -564,7 +574,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb
 F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c
-F ext/wasm/GNUmakefile 0e362f3fc04eab6628cbe4f1e35f4ab4a200881f6b5f753b27fb45eabeddd9d2
+F ext/wasm/GNUmakefile 57439eec2b8b4d4074e5d861e93aab3f32bb0f44339a2b472c23f4e638b7e8a3
 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576
 F ext/wasm/README.md a8a2962c3aebdf8d2104a9102e336c5554e78fc6072746e5daf9c61514e7d193
 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff
@@ -572,7 +582,7 @@ F ext/wasm/SQLTester/SQLTester.mjs ec2f6ba63a0f2f0562941a0fb8e46b7dc55589711513f
 F ext/wasm/SQLTester/SQLTester.run.mjs c72b7fe2072d05992f7a3d8c6a1d34e95712513ceabe40849784e24e41c84638
 F ext/wasm/SQLTester/index.html 3f8a016df0776be76605abf20e815ecaafbe055abac0e1fe5ea080e7846b760d
 F ext/wasm/SQLTester/touint8array.c 2d5ece04ec1393a6a60c4bf96385bda5e1a10ad49f3038b96460fc5e5aa7e536
-F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab
+F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api f442460ed9a109e637dd3ea1caa4489553ad9414e8988118b208bb7a4bbece6b
 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b
 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287
 F ext/wasm/api/README.md 5eb44fa02e9c693a1884a3692428647894b0380b24bca120866b7a24c8786134
@@ -582,10 +592,10 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08
 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62
 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219
 F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e
-F ext/wasm/api/sqlite3-api-glue.js 26aedfb27915f4f316f6eac84078443e4f0d2dfe5f012310014923ed4b77b2b6
+F ext/wasm/api/sqlite3-api-glue.js 119b91c8a7ce6648679eb66fcdd1ed07ef7fd892eb501d658fbfefcc962012d9
 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8
 F ext/wasm/api/sqlite3-api-prologue.js 9aeba7b45cf41b3a26d34d7fb2525633cd1adfc544888c1ea8dbb077496f4ce9
-F ext/wasm/api/sqlite3-api-worker1.js f941382f21006b4a817754184e2661b0a63ce650201f3419cd60f4758b6fd60e
+F ext/wasm/api/sqlite3-api-worker1.js 88770ac01fc756f89a3e060eec17111d6c1688e89ebfd34cb9d9e54d25affbb9
 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379
 F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25
@@ -594,8 +604,10 @@ F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 46c4afa6c50d7369252c104f274ad977a97e91cc
 F ext/wasm/api/sqlite3-wasm.c d0e09eb5ed3743c00294e30019e591c3aa150572ae7ffe8a8994568a7377589f
 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 569d4e859968e65f55dec5fac0b879828a23c8b179162cc7812fec19f844dd21
 F ext/wasm/api/sqlite3-worker1.c-pp.js a541112aa51e16705f13a99bb943c64efe178aa28c86704a955f8fd9afe4ba37
+F ext/wasm/batch-runner-sahpool.html e9a38fdeb36a13eac7b50241dfe7ae066fe3f51f5c0b0151e7baee5fce0d07a7
+F ext/wasm/batch-runner-sahpool.js 54a3ac228e6c4703fe72fb65c897e19156263a51fe9b7e21d2834a45e876aabd
 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8
-F ext/wasm/batch-runner.js 0dad6a02ad796f1003d3b7048947d275c4d6277f63767b8e685c27df8fdac93e
+F ext/wasm/batch-runner.js 05ec254f5dbfe605146d9640b3db17d6ef8c3fbef6aa8396051ca72bb5884e3f
 F ext/wasm/c-pp.c 6d80d8569d85713effe8b0818a3cf51dc779e3f0bf8dc88771b8998552ee25b4
 F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51
 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15
@@ -603,7 +615,7 @@ F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a7
 F ext/wasm/common/whwasmutil.js 4c64594eecc7af4ae64259e95a71ba2a7edf118881aaff0bba86d0c7164e78e4
 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed
 F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508
-F ext/wasm/demo-123.js 38aa8faec4d0ace1c973bc8a7a1533584463ebeecd4c420daa7d9687beeb9cb5
+F ext/wasm/demo-123.js c7b3cca50c55841c381a9ca4f9396e5bbdc6114273d0b10a43e378e32e7be5bf
 F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e
 F ext/wasm/demo-jsstorage.js 44e3ae7ec2483b6c511384c3c290beb6f305c721186bcf5398ca4e00004a06b8
 F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98ab22f5786620b3354ed15f
@@ -617,8 +629,8 @@ F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d695
 F ext/wasm/fiddle/fiddle-worker.js e0153f9af6500805c6f09c0b3cfdb7d857e9d6863dbee9d50d1628fccf5f4b4d
 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715
 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2
-F ext/wasm/index-dist.html 22379774f0ad4edcaaa8cf9c674c82e794cc557719a8addabed74eb8069d412e
-F ext/wasm/index.html 4e7847b909f4ae0da8c829b150b79454050e53b3658431f138636257729cd42b
+F ext/wasm/index-dist.html e91d76e4581185238fd3d42ed86ec600f7023ed3e3a944c5c356f25304bf1263
+F ext/wasm/index.html b31ce41c0da476d5ffcef23069b9d3415b419d65af5779096ebcfbcbade453a9
 F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54
 F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb
 F ext/wasm/module-symbols.html dc476b403369b26a1a23773e13b80f41b9a49f0825e81435fe3600a7cfbbe337
@@ -626,17 +638,17 @@ F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d
 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63
 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d
 F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe
-F ext/wasm/speedtest1-worker.html e33e2064bda572c0c3ebaec7306c35aa758d9d27e245d67e807f8cc4a9351cc5
-F ext/wasm/speedtest1-worker.js 315d26198c46be7c85e26fda15d80ef882424276abde25ffd8b026fb02a35d8c
+F ext/wasm/speedtest1-worker.html 864b65ed78ce24847a348c180e7f267621a02ca027068a1863ec1c90187c1852
+F ext/wasm/speedtest1-worker.js 4d2ea70a3c24e05bdca78025202841f33d298c4fa9541a0070c3228661f89ecd
 F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da
 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
 F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5
 F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555e685bce3da8c3f
-F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b
+F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c
 F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c
 F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2
-F ext/wasm/tester1.c-pp.js fb20d9e1c308ea34a29d8afdda1a6c5edc406241b5560fa23c19c14747ee41bc
+F ext/wasm/tester1.c-pp.js a92dc256738dbd1b50f142d1fd0c835294ba09b7bb6526650360e942f88cb63f
 F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1
 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d
 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2
@@ -665,29 +677,29 @@ F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645
 F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522
 F src/btree.c f3b09c5414de3a11db73e11e1d66f4c5e53c9e89876ff3b531a887ab656ca303 x
 F src/btree.h 03e3356f5208bcab8eed4e094240fdac4a7f9f5ddf5e91045ce589f67d47c240
-F src/btreeInt.h ef12a72b708677e48d6bc8dcd66fed25434740568b89e2cfa368093cfc5b9d15
-F src/build.c 189e4517d67f09f0a3e0d8e1faa6e2ef0c2e95f6ac82e33c912cb7efa2a359cc
+F src/btreeInt.h 3e2589726c4f105e653461814f65857465da68be1fac688de340c43b873f4062
+F src/build.c 386eadecabe2e99a3783eb802ca01e665f8e0c2af0e0aab28161fd7def219a9d
 F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 23331529e654be40ca97d171cbbffe9b3d4c71cc53b78fe5501230675952da8b
-F src/date.c eebc54a00e888d3c56147779e9f361b77d62fd69ff2008c5373946aa1ba1d574
+F src/date.c 3b8d02977d160e128469de38493b4085f7c5cf4073193459909a6af3cf6d7c91
 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782
 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43
 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500
-F src/expr.c cacf230b124891ea830d80395ea2119c2d7ee52c134a010f3950de9391ab79d1
+F src/expr.c 05278def9c186b5875d6903ea26148c7461b9ce0344f0fd7be9a0dfea0a4538a
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00
 F src/func.c 472f6dcfa39cf54f89a6aec76c79c225fb880a6c14469c15d361331662b9bf43
-F src/global.c 29f56a330ed9d1b5cd9b79ac0ca36f97ac3afc730ff8bfa987b0db9e559d684d
+F src/global.c 765a0656d6cbf043cb272ff0ae38f39cc46713539ffe6793258ed3eb4b188b52
 F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220
 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c b2045aafc52f0475dc5d723d11cc0aaa9e1d233a28031b08c8174a3e407ed50c
+F src/json.c cd270629b4b99d065522b45c3d2b5185491b1ab3544843c6f780aa50755d54dc
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36
-F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9
+F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4
 F src/malloc.c f016922435dc7d1f1f5083a03338a3e91f8c67ce2c5bdcfa4cdef62e612f5fcc
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2
@@ -708,7 +720,7 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63
 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06
 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a
 F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107
-F src/os_unix.c cb116fde9e3ca3c1bbfdf89d6928f776a2a34da168e2667426523a4db353b271
+F src/os_unix.c 97bdcd43315da7aaec9fea2da1ff7c9de458f93dd363e073f2742403a7f2e011
 F src/os_win.c 4a50a154aeebc66a1f8fb79c1ff6dd5fe3d005556533361e0d460d41cb6a45a8
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 987ab3a2cd9065d62e9955474470ff733445e2357432a67e3d0f5a8f9313e334
@@ -717,19 +729,19 @@ F src/parse.y 020d80386eb216ec9520549106353c517d2bbc89be28752ffdca649a9eaf56ec
 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75
 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5
 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00
-F src/pragma.c b3b4ad9c0298d63098a067acca613c21a5f56b4d176d5842922bcd0b07b7164e
+F src/pragma.c b5b4cff830575e6188cd56a295a57448d2b9dbc53f0dae58e22b97354cda3781
 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
-F src/prepare.c bde74add20fc0e8ce0c4e937a1f70a36d17413afe4f71d3e103f5cb74b17c8d9
-F src/printf.c 9da63b9ae1c14789bcae12840f5d800fd9302500cd2d62733fac77f0041b4750
+F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c
+F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
-F src/resolve.c 31229276a8eb5b5de1428cd2d80f6f1cf8ffc5248be25e47cf575df12f1b8f23
+F src/resolve.c d017bad7ba8e778617701a0e986fdeb393d67d6afa84fb28ef4e8b8ad2acf916
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
-F src/select.c 85857bedd2913d888aa571755b48c54cd2e6e7fcb0087e19b226ee0368cfda1e
-F src/shell.c.in cbe850737d4166347bfe5fc7d695229379347509bf114cf6abeb1e384f2d7a70
-F src/sqlite.h.in 4f841d3d117b830ee5ee45e8d89ceff1195f3ebb72d041ace8d116ba4c103b35
+F src/select.c 9f55c9f3307b9888f62abe709eec245e98ff217bd14c044f93d72810bb7dc445
+F src/shell.c.in 0cd2ef4b3c814dded5436625ab664d9a973cbc4266a1768e3aa4cbf11bb41eca
+F src/sqlite.h.in adcc7dbfeea1e69d6d487139a7e90db8a48fe998f3f5bb0f85c683e6a6fa68ca
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
-F src/sqliteInt.h 08efc0ad9ddbbc5ea859111521dff2c4cecc589c0c9a3e4840c927663b31445b
+F src/sqliteInt.h 615524fff083abf9353aa4a3a8adca2d32b3e61e014e62b1a5335c454a386783
 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6
 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -787,18 +799,18 @@ F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 23d9f4539880b40226254ad9072f4ecf12eb1902e62aea47aac29928afafcfd5
-F src/treeview.c 62fafcd31eea60b718f8daf448116b7b19f90134ebc6c20777ddbb07f56a3d28
+F src/treeview.c c6fc972683fd00f975d8b32a81c1f25d2fb7d4035366bf45c9f5622d3ccd70ee
 F src/trigger.c 0905b96b04bb6658509f711a8207287f1315cdbc3df1a1b13ba6483c8e341c81
 F src/update.c 6904814dd62a7a93bbb86d9f1419c7f134a9119582645854ab02b36b676d9f92
 F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242
 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
 F src/util.c b22cc9f203a8c0b9ee5338a67f8860347d14845864c10248bebe84518a781677
 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104
-F src/vdbe.c 1cfb61c58193dcb74e8e0f90dc1405ff7fe39f43a1e9bdd38329e174f2102ed4
-F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0
+F src/vdbe.c f73bead140670fac1aa4227188827ada52387a5fe0ccff0dd5af2a906754d904
+F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb
 F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c
-F src/vdbeapi.c dd812ddd2f7806519e9d492c820456f86628bb6014b559034506e7f9d7f683c5
-F src/vdbeaux.c 16f27624831aa6663fe5927b647a0a54dfda0088da9b170ebc14bce98c5ac414
+F src/vdbeapi.c 8f57d60c89da0b60e6d4e272358c511f6bae4e24330bdb11f8b42f986d1bf21b
+F src/vdbeaux.c c5a471b34e9c4cfc0295a3e10734fd197670ffaebcb742f284c8e17e8026ceea
 F src/vdbeblob.c 13f9287b55b6356b4b1845410382d6bede203ceb29ef69388a4a3d007ffacbe5
 F src/vdbemem.c 0012d5f01cc866833847c2f3ae4c318ac53a1cb3d28acad9c35e688039464cf0
 F src/vdbesort.c 237840ca1947511fa59bd4e18b9eeae93f2af2468c34d2427b059f896230a547
@@ -806,10 +818,10 @@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf8
 F src/vdbevtab.c 2143db7db0ceed69b21422581f434baffc507a08d831565193a7a02882a1b6a7
 F src/vtab.c 154725ebecd3bc02f7fbd7ad3974334f73fff76e02a964e828e48a7c5fb7efff
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 01e051a1e713d9eabdb25df38602837cec8f4c2cae448ce2cf6accc87af903e9
+F src/wal.c e5247a3406531b705b44630e9ccf9ca0e5c74955ef19c06fbb146d765c500c20
 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452
 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2
-F src/where.c 45b2239e127beaaae2367e503ca4c82868d8fef707c7f7f4a6c0528e7d5f65ff
+F src/where.c dad4b472b8dc886c0962409e70978568c06b5108bcf3dcd4938a9305a3540440
 F src/whereInt.h 4b38c5889514e3aead3f27d0ee9a26e47c3f150efc59e2a8b4e3bc8835e4d7a1
 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1
 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00
@@ -819,7 +831,7 @@ F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867d
 F test/affinity3.test f094773025eddf31135c7ad4cde722b7696f8eb07b97511f98585addf2a510a9
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/aggfault.test 777f269d0da5b0c2524c7ff6d99ae9a93db4f1b1839a914dd2a12e3035c29829
-F test/aggnested.test 2e738bfe2980df301a782f6e7bbf9459266f64f7e72f58f3b5c843bf897c568c
+F test/aggnested.test ce85a6af7d59c3109e35c5f03b2cd11da1a9b1417371e2f942102d0f0d77fd62
 F test/aggorderby.test e6b98dbbf3ababa96892435d387de2dcf602ef02c2b848d2d817473066f154ba
 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87
 F test/all.test 2ecb8bbd52416642e41c9081182a8df05d42c75637afd4488aace78cc4b69e13
@@ -1016,7 +1028,7 @@ F test/ctime.test 340f362f41f92972bbd71f44e10569a5cc694062b692231bd08aa6fe6c1c47
 F test/cursorhint.test 05cf0febe5c5f8a31f199401fd1c9322249e753950d55f26f9d5aca61408a270
 F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee9ae9c42f
 F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8
-F test/date.test c0d17cdfd89395bc78087b131e3538d96f864b5029c335318011accc7c0d0934
+F test/date.test ff2341a1ef71b9a27979494d299222f9a293aa22cb9ff6e9c38d88a895317ebf
 F test/date2.test 7e12ec14aaf4d5e6294b4ba140445b0eca06ea50062a9c3a69c4ee13d0b6f8b1
 F test/date3.test a1b77abf05c6772fe5ca2337cac1398892f2a41e62bce7e6be0f4a08a0e64ae5
 F test/date4.test 8aeb3de5b5e9fda968baa9357e4c0fae573724b7904943410195a19e96e31b6a
@@ -1217,7 +1229,7 @@ F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c
 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634
 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830
 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2
-F test/fuzzcheck.c 69b8549e112fb815931a8c14c7955a0c407ae91a79356eecb82458384f2cb989
+F test/fuzzcheck.c 442b0b02146bb905c47c4ee1388949e7fd9d27acf22d9535e9abf216e582b956
 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517
 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f
 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba
@@ -1318,15 +1330,17 @@ F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd7
 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa
 F test/json/README.md 63e3e589e1df8fd3cc1588ba1faaff659214003f8b77a15af5c6452b35e30ee2
 F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f
+F test/json/json-q1-b.txt 1e180fe6491efab307e318b22879e3a736ac9a96539bbde7911a13ee5b33abc7
 F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307
-F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x
-F test/json101.test bc05d2476fd6f7ead31ec05b43d1b24b2b193ae112fd8f0d2ed56d9a904f9fa5
-F test/json102.test 4c69694773a470f1fda34e5f4ba24920b35184fb66050b450fc2ef9ab5ad310b
+F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x
+F test/json101.test 70587d7d35ef9e2126364ba70f0c951f70827cfbd28649d779ff3df7e8f87547
+F test/json102.test 557a46e16df1aa9bdbc4076a71a45814ea0e7503d6621d87d42a8c04cbc2b0ef
 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe
 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1
-F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d
-F test/json501.test c419deb835b70c1a2c8532936927bcc1146730328edd2052276715bfd209724d
-F test/json502.test 98c38e3c4573841028a1381dfb81d4c3f9b105d39668167da10d055e503f6d0b
+F test/json105.test e64a8d73677fbae67886642cd5076e2ef3efe89f8483b87595cf9c030216c9bd
+F test/json501.test ab168a12eb6eb14d479f8c1cdae3ac062fd5a4679f17f976e96f1af518408330
+F test/json502.test 3c697e506fc38ccb455b49660b21b6e62e08ede0f2d0c869a7d171e17809093c
+F test/jsonb01.test cace70765b36a36aec9a85a41ea65667d3bbf647d4400ddc3ac76f8fe7d94f90
 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff
 F test/kvtest.c 6e0228409ea7ca0497dad503fbd109badb5e59545d131014b6aaac68b56f484a
 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63
@@ -1487,7 +1501,6 @@ F test/recover.test fd5199f928757cb308661b5fdca1abc19398a798ff7f24b57c3071e9f8e0
 F test/regexp1.test 8f2a8bc1569666e29a4cee6c1a666cd224eb6d50e2470d1dc1df995170f3e0f1
 F test/regexp2.test 55ed41da802b0e284ac7e2fe944be3948f93ff25abbca0361a609acfed1368b5
 F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2118d
-F test/releasetest_data.tcl 80ef3941bf7ad136f4dc3d8960786f10a4f488797238171bdd757cc6eb4c3efa
 F test/resetdb.test 54c06f18bc832ac6d6319e5ab23d5c8dd49fdbeec7c696d791682a8006bd5fc3
 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb
 F test/returning1.test db532cde29d6aebbc48c6ddc3149b30476f8e69ca7a2c4b53986c7635e6fd8ec
@@ -1587,7 +1600,7 @@ F test/snapshot2.test 8d6ff5dd9cc503f6e12d408a30409c3f9c653507b24408d9cd7195931c
 F test/snapshot3.test 8744313270c55f6e18574283553d3c5c5fe4c5970585663613a0e75c151e599b
 F test/snapshot4.test d4e9347ef2fcabc491fc893506c7bbaf334da3be111d6eb4f3a97cc623b78322
 F test/snapshot_fault.test 129234ceb9b26a0e1000e8563a16e790f5c1412354e70749cbd78c3d5d07d60a
-F test/snapshot_up.test a0a29c4cf33475fcef07c3f8e64af795e24ab91b4cc68295863402a393cdd41c
+F test/snapshot_up.test 77dc7853bfb2b4fa249f76e1714cfa1e596826165d9ef22c06ac3a0b7b778d9a
 F test/soak.test 18944cf21b94a7fe0df02016a6ee1e9632bc4e8d095a0cb49d95e15d5cca2d5c
 F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087
 F test/sort.test f86751134159abb5e5fd4381a0d7038c91013638cd1e3fa1d7850901f6df6196
@@ -1650,8 +1663,8 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163
 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc
 F test/tester.tcl 68454ef88508c196d19e8694daa27bff7107a91857799eaa12f417188ae53ede
-F test/testrunner.tcl 14c8b8ece841b1dd17516a0dc9c7ad9b5f4d4db7987974d3fdf66ae56b2a71fa
-F test/testrunner_data.tcl 7f73f93634d32dafc857ed491b840f371113d09fde6a8bfb9e47b938d47b8c85
+F test/testrunner.tcl e18d71f2e797da808ba6d31335e504ed6b2791581b89287a72b697a2f31b1ea1
+F test/testrunner_data.tcl e4d5017290a6d5c11785e36cc94c67d8bb950c8cdc2dbe4c1db2a3a583812560
 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899
 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502
 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7
@@ -1816,7 +1829,7 @@ F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7
 F test/tpch01.test 4479008f85f6f8f25f7ab2cb305d665752b4727fa28a8df3d8e0ad46520c62ff
 F test/trace.test a659a9862957f4789e37a92b3bf6d2caf5c86b02cdeefc41e850ae53acf6992a
 F test/trace2.test f5cb67ad3bc09e0c58e8cca78dfd0b5639259983
-F test/trace3.test ae2004df24b585fed9046cc0bae4601762bc6fc4aa321d475f1350bba5047f31
+F test/trace3.test 4f418ed30d15d9d17dcf13a17f0bd99a92e3038e038798e35db7525f82f4c281
 F test/trans.test 45f6f9ab6f66a7b5744f1caac06b558f95da62501916906cf55586a896f9f439
 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76
 F test/trans3.test 91a100e5412b488e22a655fe423a14c26403ab94
@@ -1952,8 +1965,6 @@ F test/walshared.test 42e3808582504878af237ea02c42ca793e8a0efaa19df7df26ac573370
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
 F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2af51747
 F test/walvfs.test e1a6ad0f3c78e98b55c3d5f0889cf366cc0d0a1cb2bccb44ac9ec67384adc4a1
-F test/wapp.tcl b440cd8cf57953d3a49e7ee81e6a18f18efdaf113b69f7d8482b0710a64566ec
-F test/wapptest.tcl 1bea58a6a8e68a73f542ee4fca28b771b84ed803bd0c9e385087070b3d747b3c x
 F test/where.test 59abb854eee24f166b5f7ba9d17eb250abc59ce0a66c48912ffb10763648196d
 F test/where2.test 03c21a11e7b90e2845fc3c8b4002fc44cc2797fa74c86ee47d70bd7ea4f29ed6
 F test/where3.test 5b4ffc0ac2ea0fe92f02b1244b7531522fe4d7bccf6fa8741d54e82c10e67753
@@ -1985,7 +1996,7 @@ F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2
 F test/win32lock.test e0924eb8daac02bf80e9da88930747bd44dd9b230b7759fed927b1655b467c9c
 F test/win32longpath.test 4baffc3acb2e5188a5e3a895b2b543ed09e62f7c72d713c1feebf76222fe9976
 F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc
-F test/window1.test ccfeaf116afc6a8f748a8122a4f1ee6b69e6bbc5acee61197d3c17167338b100
+F test/window1.test 5e8abe56a7d667eeddbba6de180086dcf69ed528d046447a25464f945ece101f
 F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476
 F test/window2.test e466a88bd626d66edc3d352d7d7e1d5531e0079b549ba44efb029d1fbff9fd3c
 F test/window3.tcl acea6e86a4324a210fd608d06741010ca83ded9fde438341cb978c49928faf03
@@ -2109,7 +2120,7 @@ F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaa
 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848
 F tool/src-verify.c 41c586dee84d0b190ad13e0282ed83d4a65ec9fefde9adf4943efdf6558eea7f
 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f
-F tool/srctree-check.tcl 9f1098d439159692ece45cc9e701b35df4956269399fe0c770e41f235d1ce5e6
+F tool/srctree-check.tcl c15f860a3c97d5f7b4c14b60392d9466af29dd006c4ef18127f502641e2977a8
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/stripccomments.c 20b8aabc4694d0d4af5566e42da1f1a03aff057689370326e9269a9ddcffdc37
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
@@ -2142,10 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f796da626b5c3b099cc246a8b46d92839922e34f398d673b0b03d7ee33a28a60
-R 1ffc537351df4c25512f7c1fd5cd9511
-T +sym-release *
-T +sym-version-3.44.2 *
+P be1864eac4eb75cc30bf98f73092c8608467f4bd956240df6a0cbea9f1e09e85
+R 6e221a94cd5525fe9f8a7e94ea467b60
 U drh
-Z abd52f0184ebdbba1837e841b542823e
+Z 2ea9e875faf360824d045f495f8867aa
 # Remove this line to create a well-formed Fossil manifest.
@@ -1 +1 @@
-ebead0e7230cd33bcec9f95d2183069565b9e709bf745c9b5db65cc0cbf92c0f
+c96ebb086feb89341565cc52b970ae7799ce1327fe1ad4fc790f1b0dcaa6e229
@@ -64,7 +64,7 @@
 **     22       1     Min embedded payload fraction (must be 32)
 **     23       1     Min leaf payload fraction (must be 32)
 **     24       4     File change counter
-**     28       4     Reserved for future use
+**     28       4     The size of the database in pages
 **     32       4     First freelist page
 **     36       4     Number of freelist pages in the file
 **     40      60     15 4-byte meta values passed to higher layers
@@ -720,7 +720,7 @@ void sqlite3ColumnSetExpr(
 */
 Expr *sqlite3ColumnExpr(Table *pTab, Column *pCol){
  if( pCol->iDflt==0 ) return 0;
-  if( NEVER(!IsOrdinaryTable(pTab)) ) return 0;
+  if( !IsOrdinaryTable(pTab) ) return 0;
  if( NEVER(pTab->u.tab.pDfltList==0) ) return 0;
  if( NEVER(pTab->u.tab.pDfltList->nExpr<pCol->iDflt) ) return 0;
  return pTab->u.tab.pDfltList->a[pCol->iDflt-1].pExpr;
@@ -873,6 +873,9 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){
  if( db->pnBytesFreed==0 && (--pTable->nTabRef)>0 ) return;
  deleteTable(db, pTable);
 }
+void sqlite3DeleteTableGeneric(sqlite3 *db, void *pTable){
+  sqlite3DeleteTable(db, (Table*)pTable);
+}


 /*
@@ -1410,7 +1413,8 @@ void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){
 /*
 ** Clean up the data structures associated with the RETURNING clause.
 */
-static void sqlite3DeleteReturning(sqlite3 *db, Returning *pRet){
+static void sqlite3DeleteReturning(sqlite3 *db, void *pArg){
+  Returning *pRet = (Returning*)pArg;
  Hash *pHash;
  pHash = &(db->aDb[1].pSchema->trigHash);
  sqlite3HashInsert(pHash, pRet->zName, 0);
@@ -1452,8 +1456,7 @@ void sqlite3AddReturning(Parse *pParse, ExprList *pList){
  pParse->u1.pReturning = pRet;
  pRet->pParse = pParse;
  pRet->pReturnEL = pList;
-  sqlite3ParserAddCleanup(pParse,
-     (void(*)(sqlite3*,void*))sqlite3DeleteReturning, pRet);
+  sqlite3ParserAddCleanup(pParse, sqlite3DeleteReturning, pRet);
  testcase( pParse->earlyCleanup );
  if( db->mallocFailed ) return;
  sqlite3_snprintf(sizeof(pRet->zName), pRet->zName,
@@ -5692,4 +5695,7 @@ void sqlite3WithDelete(sqlite3 *db, With *pWith){
    sqlite3DbFree(db, pWith);
  }
 }
+void sqlite3WithDeleteGeneric(sqlite3 *db, void *pWith){
+  sqlite3WithDelete(db, (With*)pWith);
+}
 #endif /* !defined(SQLITE_OMIT_CTE) */
@@ -1043,6 +1043,12 @@ static int isDate(
  }
  computeJD(p);
  if( p->isError || !validJulianDay(p->iJD) ) return 1;
+  if( argc==1 && p->validYMD && p->D>28 ){
+    /* Make sure a YYYY-MM-DD is normalized.
+    ** Example: 2023-02-31 -> 2023-03-03 */
+    assert( p->validJD );
+    p->validYMD = 0;  
+  }
  return 0;
 }

@@ -1223,9 +1223,7 @@ void sqlite3ExprAddFunctionOrderBy(
  assert( ExprUseXList(pExpr) );
  if( pExpr->x.pList==0 || NEVER(pExpr->x.pList->nExpr==0) ){
    /* Ignore ORDER BY on zero-argument aggregates */
-    sqlite3ParserAddCleanup(pParse,
-        (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
-        pOrderBy);
+    sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, pOrderBy);
    return;
  }
  if( IsWindowFunc(pExpr) ){
@@ -1406,6 +1404,9 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){
 void sqlite3ExprDelete(sqlite3 *db, Expr *p){
  if( p ) sqlite3ExprDeleteNN(db, p);
 }
+void sqlite3ExprDeleteGeneric(sqlite3 *db, void *p){
+  if( ALWAYS(p) ) sqlite3ExprDeleteNN(db, (Expr*)p);
+}

 /*
 ** Clear both elements of an OnOrUsing object
@@ -1431,9 +1432,7 @@ void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){
 ** pExpr to the pParse->pConstExpr list with a register number of 0.
 */
 void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){
-  sqlite3ParserAddCleanup(pParse,
-    (void(*)(sqlite3*,void*))sqlite3ExprDelete,
-    pExpr);
+  sqlite3ParserAddCleanup(pParse, sqlite3ExprDeleteGeneric, pExpr);
 }

 /* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the
@@ -2239,6 +2238,9 @@ static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){
 void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){
  if( pList ) exprListDeleteNN(db, pList);
 }
+void sqlite3ExprListDeleteGeneric(sqlite3 *db, void *pList){
+  if( ALWAYS(pList) ) exprListDeleteNN(db, (ExprList*)pList);
+}

 /*
 ** Return the bitwise-OR of all Expr.flags fields in the given
@@ -6769,13 +6771,14 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
    case TK_AGG_FUNCTION: {
      if( (pNC->ncFlags & NC_InAggFunc)==0
       && pWalker->walkerDepth==pExpr->op2
+       && pExpr->pAggInfo==0
      ){
        /* Check to see if pExpr is a duplicate of another aggregate
        ** function that is already in the pAggInfo structure
        */
        struct AggInfo_func *pItem = pAggInfo->aFunc;
        for(i=0; i<pAggInfo->nFunc; i++, pItem++){
-          if( pItem->pFExpr==pExpr ) break;
+          if( NEVER(pItem->pFExpr==pExpr) ) break;
          if( sqlite3ExprCompare(0, pItem->pFExpr, pExpr, -1)==0 ){
            break;
          }
@@ -244,6 +244,9 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = {
   0,                         /* bSmallMalloc */
   1,                         /* bExtraSchemaChecks */
   sizeof(LONGDOUBLE_TYPE)>8, /* bUseLongDouble */
+#ifdef SQLITE_DEBUG
+   0,                         /* bJsonSelfcheck */
+#endif
   0x7ffffffe,                /* mxStrlen */
   0,                         /* neverCorrupt */
   SQLITE_DEFAULT_LOOKASIDE,  /* szLookaside, nLookaside */
@@ -4659,6 +4659,28 @@ int sqlite3_test_control(int op, ...){
      break;
    }
 #endif
+
+    /* sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &onOff);
+    **
+    ** Activate or deactivate validation of JSONB that is generated from
+    ** text.  Off by default, as the validation is slow.  Validation is
+    ** only available if compiled using SQLITE_DEBUG.
+    **
+    ** If onOff is initially 1, then turn it on.  If onOff is initially
+    ** off, turn it off.  If onOff is initially -1, then change onOff
+    ** to be the current setting.
+    */
+    case SQLITE_TESTCTRL_JSON_SELFCHECK: {
+#if defined(SQLITE_DEBUG)
+      int *pOnOff = va_arg(ap, int*);
+      if( *pOnOff<0 ){
+        *pOnOff = sqlite3Config.bJsonSelfcheck;
+      }else{
+        sqlite3Config.bJsonSelfcheck = (u8)((*pOnOff)&0xff);
+      }
+#endif
+      break;
+    }
  }
  va_end(ap);
 #endif /* SQLITE_UNTESTABLE */
@@ -4054,7 +4054,13 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){
 #ifdef SQLITE_ENABLE_SETLK_TIMEOUT
    case SQLITE_FCNTL_LOCK_TIMEOUT: {
      int iOld = pFile->iBusyTimeout;
+#if SQLITE_ENABLE_SETLK_TIMEOUT==1
      pFile->iBusyTimeout = *(int*)pArg;
+#elif SQLITE_ENABLE_SETLK_TIMEOUT==2
+      pFile->iBusyTimeout = !!(*(int*)pArg);
+#else
+# error "SQLITE_ENABLE_SETLK_TIMEOUT must be set to 1 or 2"
+#endif
      *(int*)pArg = iOld;
      return SQLITE_OK;
    }
@@ -4307,6 +4313,25 @@ static int unixGetpagesize(void){
 ** Either unixShmNode.pShmMutex must be held or unixShmNode.nRef==0 and
 ** unixMutexHeld() is true when reading or writing any other field
 ** in this structure.
+**
+** aLock[SQLITE_SHM_NLOCK]:
+**   This array records the various locks held by clients on each of the
+**   SQLITE_SHM_NLOCK slots. If the aLock[] entry is set to 0, then no
+**   locks are held by the process on this slot. If it is set to -1, then
+**   some client holds an EXCLUSIVE lock on the locking slot. If the aLock[]
+**   value is set to a positive value, then it is the number of shared 
+**   locks currently held on the slot.
+**
+** aMutex[SQLITE_SHM_NLOCK]:
+**   Normally, when SQLITE_ENABLE_SETLK_TIMEOUT is not defined, mutex 
+**   pShmMutex is used to protect the aLock[] array and the right to
+**   call fcntl() on unixShmNode.hShm to obtain or release locks.
+**
+**   If SQLITE_ENABLE_SETLK_TIMEOUT is defined though, we use an array
+**   of mutexes - one for each locking slot. To read or write locking
+**   slot aLock[iSlot], the caller must hold the corresponding mutex
+**   aMutex[iSlot]. Similarly, to call fcntl() to obtain or release a
+**   lock corresponding to slot iSlot, mutex aMutex[iSlot] must be held.
 */
 struct unixShmNode {
  unixInodeInfo *pInode;     /* unixInodeInfo that owns this SHM node */
@@ -4320,10 +4345,11 @@ struct unixShmNode {
  char **apRegion;           /* Array of mapped shared-memory regions */
  int nRef;                  /* Number of unixShm objects pointing to this */
  unixShm *pFirst;           /* All unixShm objects pointing to this */
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+  sqlite3_mutex *aMutex[SQLITE_SHM_NLOCK];
+#endif
  int aLock[SQLITE_SHM_NLOCK];  /* # shared locks on slot, -1==excl lock */
 #ifdef SQLITE_DEBUG
-  u8 exclMask;               /* Mask of exclusive locks held */
-  u8 sharedMask;             /* Mask of shared locks held */
  u8 nextShmId;              /* Next available unixShm.id value */
 #endif
 };
@@ -4406,16 +4432,29 @@ static int unixShmSystemLock(
  struct flock f;        /* The posix advisory locking structure */
  int rc = SQLITE_OK;    /* Result code form fcntl() */

-  /* Access to the unixShmNode object is serialized by the caller */
  pShmNode = pFile->pInode->pShmNode;
-  assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->pShmMutex) );
-  assert( pShmNode->nRef>0 || unixMutexHeld() );
+
+  /* Assert that the correct mutex or mutexes are held. */
+  if( pShmNode->nRef==0 ){
+    assert( ofst==UNIX_SHM_DMS && n==1 && unixMutexHeld() );
+  }else{
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+    int ii;
+    for(ii=ofst-UNIX_SHM_BASE; ii<ofst-UNIX_SHM_BASE+n; ii++){
+      assert( sqlite3_mutex_held(pShmNode->aMutex[ii]) );
+    }
+#else
+    assert( sqlite3_mutex_held(pShmNode->pShmMutex) );
+    assert( pShmNode->nRef>0 );
+#endif
+  }

  /* Shared locks never span more than one byte */
  assert( n==1 || lockType!=F_RDLCK );

  /* Locks are within range */
  assert( n>=1 && n<=SQLITE_SHM_NLOCK );
+  assert( ofst>=UNIX_SHM_BASE && ofst<=(UNIX_SHM_DMS+SQLITE_SHM_NLOCK) );

  if( pShmNode->hShm>=0 ){
    int res;
@@ -4426,7 +4465,7 @@ static int unixShmSystemLock(
    f.l_len = n;
    res = osSetPosixAdvisoryLock(pShmNode->hShm, &f, pFile);
    if( res==-1 ){
-#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+#if defined(SQLITE_ENABLE_SETLK_TIMEOUT) && SQLITE_ENABLE_SETLK_TIMEOUT==1
      rc = (pFile->iBusyTimeout ? SQLITE_BUSY_TIMEOUT : SQLITE_BUSY);
 #else
      rc = SQLITE_BUSY;
@@ -4434,39 +4473,28 @@ static int unixShmSystemLock(
    }
  }

-  /* Update the global lock state and do debug tracing */
+  /* Do debug tracing */
 #ifdef SQLITE_DEBUG
-  { u16 mask;
  OSTRACE(("SHM-LOCK "));
-  mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<<ofst);
  if( rc==SQLITE_OK ){
    if( lockType==F_UNLCK ){
-      OSTRACE(("unlock %d ok", ofst));
-      pShmNode->exclMask &= ~mask;
-      pShmNode->sharedMask &= ~mask;
+      OSTRACE(("unlock %d..%d ok\n", ofst, ofst+n-1));
    }else if( lockType==F_RDLCK ){
-      OSTRACE(("read-lock %d ok", ofst));
-      pShmNode->exclMask &= ~mask;
-      pShmNode->sharedMask |= mask;
+      OSTRACE(("read-lock %d..%d ok\n", ofst, ofst+n-1));
    }else{
      assert( lockType==F_WRLCK );
-      OSTRACE(("write-lock %d ok", ofst));
-      pShmNode->exclMask |= mask;
-      pShmNode->sharedMask &= ~mask;
+      OSTRACE(("write-lock %d..%d ok\n", ofst, ofst+n-1));
    }
  }else{
    if( lockType==F_UNLCK ){
-      OSTRACE(("unlock %d failed", ofst));
+      OSTRACE(("unlock %d..%d failed\n", ofst, ofst+n-1));
    }else if( lockType==F_RDLCK ){
-      OSTRACE(("read-lock failed"));
+      OSTRACE(("read-lock %d..%d failed\n", ofst, ofst+n-1));
    }else{
      assert( lockType==F_WRLCK );
-      OSTRACE(("write-lock %d failed", ofst));
+      OSTRACE(("write-lock %d..%d failed\n", ofst, ofst+n-1));
    }
  }
-  OSTRACE((" - afterwards %03x,%03x\n",
-           pShmNode->sharedMask, pShmNode->exclMask));
-  }
 #endif

  return rc;      
@@ -4503,6 +4531,11 @@ static void unixShmPurge(unixFile *pFd){
    int i;
    assert( p->pInode==pFd->pInode );
    sqlite3_mutex_free(p->pShmMutex);
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+    for(i=0; i<SQLITE_SHM_NLOCK; i++){
+      sqlite3_mutex_free(p->aMutex[i]);
+    }
+#endif
    for(i=0; i<p->nRegion; i+=nShmPerMap){
      if( p->hShm>=0 ){
        osMunmap(p->apRegion[i], p->szRegion);
@@ -4562,7 +4595,20 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){
      pShmNode->isUnlocked = 1;
      rc = SQLITE_READONLY_CANTINIT;
    }else{
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+      /* Do not use a blocking lock here. If the lock cannot be obtained
+      ** immediately, it means some other connection is truncating the
+      ** *-shm file. And after it has done so, it will not release its
+      ** lock, but only downgrade it to a shared lock. So no point in
+      ** blocking here. The call below to obtain the shared DMS lock may 
+      ** use a blocking lock. */
+      int iSaveTimeout = pDbFd->iBusyTimeout;
+      pDbFd->iBusyTimeout = 0;
+#endif
      rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1);
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+      pDbFd->iBusyTimeout = iSaveTimeout;
+#endif
      /* The first connection to attach must truncate the -shm file.  We
      ** truncate to 3 bytes (an arbitrary small number, less than the
      ** -shm header size) rather than 0 as a system debugging aid, to
@@ -4683,6 +4729,18 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
        rc = SQLITE_NOMEM_BKPT;
        goto shm_open_err;
      }
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+      {
+        int ii;
+        for(ii=0; ii<SQLITE_SHM_NLOCK; ii++){
+          pShmNode->aMutex[ii] = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST);
+          if( pShmNode->aMutex[ii]==0 ){
+            rc = SQLITE_NOMEM_BKPT;
+            goto shm_open_err;
+          }
+        }
+      }
+#endif
    }

    if( pInode->bProcessLock==0 ){
@@ -4904,9 +4962,11 @@ shmpage_out:
 */
 #ifdef SQLITE_DEBUG
 static int assertLockingArrayOk(unixShmNode *pShmNode){
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+  return 1;
+#else
  unixShm *pX;
  int aLock[SQLITE_SHM_NLOCK];
-  assert( sqlite3_mutex_held(pShmNode->pShmMutex) );

  memset(aLock, 0, sizeof(aLock));
  for(pX=pShmNode->pFirst; pX; pX=pX->pNext){
@@ -4924,13 +4984,14 @@ static int assertLockingArrayOk(unixShmNode *pShmNode){

  assert( 0==memcmp(pShmNode->aLock, aLock, sizeof(aLock)) );
  return (memcmp(pShmNode->aLock, aLock, sizeof(aLock))==0);
+#endif
 }
 #endif

 /*
 ** Change the lock state for a shared-memory segment.
 **
-** Note that the relationship between SHAREd and EXCLUSIVE locks is a little
+** Note that the relationship between SHARED and EXCLUSIVE locks is a little
 ** different here than in posix.  In xShmLock(), one can go from unlocked
 ** to shared and back or from unlocked to exclusive and back.  But one may
 ** not go from shared to exclusive or from exclusive to shared.
@@ -4945,7 +5006,7 @@ static int unixShmLock(
  unixShm *p;                           /* The shared memory being locked */
  unixShmNode *pShmNode;                /* The underlying file iNode */
  int rc = SQLITE_OK;                   /* Result code */
-  u16 mask;                             /* Mask of locks to take or release */
+  u16 mask = (1<<(ofst+n)) - (1<<ofst); /* Mask of locks to take or release */
  int *aLock;

  p = pDbFd->pShm;
@@ -4980,88 +5041,150 @@ static int unixShmLock(
  ** It is not permitted to block on the RECOVER lock.
  */
 #ifdef SQLITE_ENABLE_SETLK_TIMEOUT
-  assert( (flags & SQLITE_SHM_UNLOCK) || pDbFd->iBusyTimeout==0 || (
-         (ofst!=2)                                   /* not RECOVER */
-      && (ofst!=1 || (p->exclMask|p->sharedMask)==0)
-      && (ofst!=0 || (p->exclMask|p->sharedMask)<3)
-      && (ofst<3  || (p->exclMask|p->sharedMask)<(1<<ofst))
-  ));
+  {
+    u16 lockMask = (p->exclMask|p->sharedMask);
+    assert( (flags & SQLITE_SHM_UNLOCK) || pDbFd->iBusyTimeout==0 || (
+          (ofst!=2)                                   /* not RECOVER */
+       && (ofst!=1 || lockMask==0 || lockMask==2)
+       && (ofst!=0 || lockMask<3)
+       && (ofst<3  || lockMask<(1<<ofst))
+    ));
+  }
 #endif

-  mask = (1<<(ofst+n)) - (1<<ofst);
-  assert( n>1 || mask==(1<<ofst) );
-  sqlite3_mutex_enter(pShmNode->pShmMutex);
-  assert( assertLockingArrayOk(pShmNode) );
-  if( flags & SQLITE_SHM_UNLOCK ){
-    if( (p->exclMask|p->sharedMask) & mask ){
-      int ii;
-      int bUnlock = 1;
+  /* Check if there is any work to do. There are three cases:
+  **
+  **    a) An unlock operation where there are locks to unlock,
+  **    b) An shared lock where the requested lock is not already held
+  **    c) An exclusive lock where the requested lock is not already held
+  **
+  ** The SQLite core never requests an exclusive lock that it already holds.
+  ** This is assert()ed below.
+  */
+  assert( flags!=(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK) 
+       || 0==(p->exclMask & mask) 
+  );
+  if( ((flags & SQLITE_SHM_UNLOCK) && ((p->exclMask|p->sharedMask) & mask))
+   || (flags==(SQLITE_SHM_SHARED|SQLITE_SHM_LOCK) && 0==(p->sharedMask & mask))
+   || (flags==(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK))
+  ){

-      for(ii=ofst; ii<ofst+n; ii++){
-        if( aLock[ii]>((p->sharedMask & (1<<ii)) ? 1 : 0) ){
-          bUnlock = 0;
-        }
-      }
-
-      if( bUnlock ){
-        rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n);
-        if( rc==SQLITE_OK ){
-          memset(&aLock[ofst], 0, sizeof(int)*n);
-        }
-      }else if( ALWAYS(p->sharedMask & (1<<ofst)) ){
-        assert( n==1 && aLock[ofst]>1 );
-        aLock[ofst]--;
-      }
-
-      /* Undo the local locks */
-      if( rc==SQLITE_OK ){
-        p->exclMask &= ~mask;
-        p->sharedMask &= ~mask;
+    /* Take the required mutexes. In SETLK_TIMEOUT mode (blocking locks), if
+    ** this is an attempt on an exclusive lock use sqlite3_mutex_try(). If any
+    ** other thread is holding this mutex, then it is either holding or about
+    ** to hold a lock exclusive to the one being requested, and we may
+    ** therefore return SQLITE_BUSY to the caller.
+    **
+    ** Doing this prevents some deadlock scenarios. For example, thread 1 may
+    ** be a checkpointer blocked waiting on the WRITER lock. And thread 2
+    ** may be a normal SQL client upgrading to a write transaction. In this
+    ** case thread 2 does a non-blocking request for the WRITER lock. But -
+    ** if it were to use sqlite3_mutex_enter() then it would effectively
+    ** become a (doomed) blocking request, as thread 2 would block until thread
+    ** 1 obtained WRITER and released the mutex. Since thread 2 already holds
+    ** a lock on a read-locking slot at this point, this breaks the
+    ** anti-deadlock rules (see above).  */
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+    int iMutex;
+    for(iMutex=ofst; iMutex<ofst+n; iMutex++){
+      if( flags==(SQLITE_SHM_LOCK|SQLITE_SHM_EXCLUSIVE) ){
+        rc = sqlite3_mutex_try(pShmNode->aMutex[iMutex]);
+        if( rc!=SQLITE_OK ) break;
+      }else{
+        sqlite3_mutex_enter(pShmNode->aMutex[iMutex]);
      }
    }
-  }else if( flags & SQLITE_SHM_SHARED ){
-    assert( n==1 );
-    assert( (p->exclMask & (1<<ofst))==0 );
-    if( (p->sharedMask & mask)==0 ){
-      if( aLock[ofst]<0 ){
-        rc = SQLITE_BUSY;
-      }else if( aLock[ofst]==0 ){
-        rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n);
-      }
+#else
+    sqlite3_mutex_enter(pShmNode->pShmMutex);
+#endif

-      /* Get the local shared locks */
-      if( rc==SQLITE_OK ){
-        p->sharedMask |= mask;
-        aLock[ofst]++;
-      }
-    }
-  }else{
-    /* Make sure no sibling connections hold locks that will block this
-    ** lock.  If any do, return SQLITE_BUSY right away.  */
-    int ii;
-    for(ii=ofst; ii<ofst+n; ii++){
-      assert( (p->sharedMask & mask)==0 );
-      if( ALWAYS((p->exclMask & (1<<ii))==0) && aLock[ii] ){
-        rc = SQLITE_BUSY;
-        break;
-      }
-    }
-
-    /* Get the exclusive locks at the system level. Then if successful
-    ** also update the in-memory values. */
    if( rc==SQLITE_OK ){
-      rc = unixShmSystemLock(pDbFd, F_WRLCK, ofst+UNIX_SHM_BASE, n);
-      if( rc==SQLITE_OK ){
+      if( flags & SQLITE_SHM_UNLOCK ){
+        /* Case (a) - unlock.  */
+        int bUnlock = 1;
+        assert( (p->exclMask & p->sharedMask)==0 );
+        assert( !(flags & SQLITE_SHM_EXCLUSIVE) || (p->exclMask & mask)==mask );
+        assert( !(flags & SQLITE_SHM_SHARED) || (p->sharedMask & mask)==mask );
+  
+        /* If this is a SHARED lock being unlocked, it is possible that other
+        ** clients within this process are holding the same SHARED lock. In
+        ** this case, set bUnlock to 0 so that the posix lock is not removed
+        ** from the file-descriptor below.  */
+        if( flags & SQLITE_SHM_SHARED ){
+          assert( n==1 );
+          assert( aLock[ofst]>=1 );
+          if( aLock[ofst]>1 ){
+            bUnlock = 0;
+            aLock[ofst]--;
+            p->sharedMask &= ~mask;
+          }
+        }
+  
+        if( bUnlock ){
+          rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n);
+          if( rc==SQLITE_OK ){
+            memset(&aLock[ofst], 0, sizeof(int)*n);
+            p->sharedMask &= ~mask;
+            p->exclMask &= ~mask;
+          }
+        }
+      }else if( flags & SQLITE_SHM_SHARED ){
+        /* Case (b) - a shared lock.  */
+  
+        if( aLock[ofst]<0 ){
+          /* An exclusive lock is held by some other connection. BUSY. */ 
+          rc = SQLITE_BUSY;
+        }else if( aLock[ofst]==0 ){
+          rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n);
+        }
+  
+        /* Get the local shared locks */
+        if( rc==SQLITE_OK ){
+          p->sharedMask |= mask;
+          aLock[ofst]++;
+        }
+      }else{
+        /* Case (c) - an exclusive lock.  */
+        int ii;
+  
+        assert( flags==(SQLITE_SHM_LOCK|SQLITE_SHM_EXCLUSIVE) );
        assert( (p->sharedMask & mask)==0 );
-        p->exclMask |= mask;
+        assert( (p->exclMask & mask)==0 );
+  
+        /* Make sure no sibling connections hold locks that will block this
+        ** lock.  If any do, return SQLITE_BUSY right away.  */
        for(ii=ofst; ii<ofst+n; ii++){
-          aLock[ii] = -1;
+          if( aLock[ii] ){
+            rc = SQLITE_BUSY;
+            break;
+          }
+        }
+  
+        /* Get the exclusive locks at the system level. Then if successful
+        ** also update the in-memory values. */
+        if( rc==SQLITE_OK ){
+          rc = unixShmSystemLock(pDbFd, F_WRLCK, ofst+UNIX_SHM_BASE, n);
+          if( rc==SQLITE_OK ){
+            p->exclMask |= mask;
+            for(ii=ofst; ii<ofst+n; ii++){
+              aLock[ii] = -1;
+            }
+          }
        }
      }
+      assert( assertLockingArrayOk(pShmNode) );
    }
+
+    /* Drop the mutexes acquired above. */
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+    for(iMutex--; iMutex>=ofst; iMutex--){
+      sqlite3_mutex_leave(pShmNode->aMutex[iMutex]);
+    }
+#else
+    sqlite3_mutex_leave(pShmNode->pShmMutex);
+#endif
  }
-  assert( assertLockingArrayOk(pShmNode) );
-  sqlite3_mutex_leave(pShmNode->pShmMutex);
+
  OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x\n",
           p->id, osGetpid(0), p->sharedMask, p->exclMask));
  return rc;
@@ -1779,7 +1779,8 @@ void sqlite3Pragma(
          if( pVTab->pModule->iVersion<4 ) continue;
          if( pVTab->pModule->xIntegrity==0 ) continue;
          sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick);
-          sqlite3VdbeAppendP4(v, pTab, P4_TABLE);
+          pTab->nTabRef++;
+          sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF);
          a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v);
          integrityCheckResultRow(v);
          sqlite3VdbeJumpHere(v, a1);
@@ -868,6 +868,7 @@ static int sqlite3LockAndPrepare(
  assert( (rc&db->errMask)==rc );
  db->busyHandler.nBusy = 0;
  sqlite3_mutex_leave(db->mutex);
+  assert( rc==SQLITE_OK || (*ppStmt)==0 );
  return rc;
 }

@@ -1369,7 +1369,7 @@ void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){


 /*****************************************************************************
-** Reference counted string storage
+** Reference counted string/blob storage
 *****************************************************************************/

 /*
@@ -1249,11 +1249,12 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
          while( pNC2
              && sqlite3ReferencesSrcList(pParse, pExpr, pNC2->pSrcList)==0
          ){
-            pExpr->op2++;
+            pExpr->op2 += (1 + pNC2->nNestedSelect);
            pNC2 = pNC2->pNext;
          }
          assert( pDef!=0 || IN_RENAME_OBJECT );
          if( pNC2 && pDef ){
+            pExpr->op2 += pNC2->nNestedSelect;
            assert( SQLITE_FUNC_MINMAX==NC_MinMaxAgg );
            assert( SQLITE_FUNC_ANYORDER==NC_OrderAgg );
            testcase( (pDef->funcFlags & SQLITE_FUNC_MINMAX)!=0 );
@@ -1812,6 +1813,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){
 
    /* Recursively resolve names in all subqueries in the FROM clause
    */
+    if( pOuterNC ) pOuterNC->nNestedSelect++;
    for(i=0; i<p->pSrc->nSrc; i++){
      SrcItem *pItem = &p->pSrc->a[i];
      if( pItem->pSelect && (pItem->pSelect->selFlags & SF_Resolved)==0 ){
@@ -1836,6 +1838,9 @@ static int resolveSelectStep(Walker *pWalker, Select *p){
        }
      }
    }
+    if( pOuterNC && ALWAYS(pOuterNC->nNestedSelect>0) ){
+      pOuterNC->nNestedSelect--;
+    }
 
    /* Set up the local name-context to pass to sqlite3ResolveExprNames() to
    ** resolve the result-set expression list.
@@ -184,6 +184,9 @@ Select *sqlite3SelectNew(
 void sqlite3SelectDelete(sqlite3 *db, Select *p){
  if( OK_IF_ALWAYS_TRUE(p) ) clearSelect(db, p, 1);
 }
+void sqlite3SelectDeleteGeneric(sqlite3 *db, void *p){
+  if( ALWAYS(p) ) clearSelect(db, (Select*)p, 1);
+}

 /*
 ** Return a pointer to the right-most SELECT statement in a compound.
@@ -3204,9 +3207,7 @@ multi_select_end:
  pDest->iSdst = dest.iSdst;
  pDest->nSdst = dest.nSdst;
  if( pDelete ){
-    sqlite3ParserAddCleanup(pParse,
-        (void(*)(sqlite3*,void*))sqlite3SelectDelete,
-        pDelete);
+    sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pDelete);
  }
  return rc;
 }
@@ -3757,8 +3758,7 @@ static int multiSelectOrderBy(
  /* Make arrangements to free the 2nd and subsequent arms of the compound
  ** after the parse has finished */
  if( pSplit->pPrior ){
-    sqlite3ParserAddCleanup(pParse,
-       (void(*)(sqlite3*,void*))sqlite3SelectDelete, pSplit->pPrior);
+    sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pSplit->pPrior);
  }
  pSplit->pPrior = pPrior;
  pPrior->pNext = pSplit;
@@ -4579,9 +4579,7 @@ static int flattenSubquery(
    Table *pTabToDel = pSubitem->pTab;
    if( pTabToDel->nTabRef==1 ){
      Parse *pToplevel = sqlite3ParseToplevel(pParse);
-      sqlite3ParserAddCleanup(pToplevel,
-         (void(*)(sqlite3*,void*))sqlite3DeleteTable,
-         pTabToDel);
+      sqlite3ParserAddCleanup(pToplevel, sqlite3DeleteTableGeneric, pTabToDel);
      testcase( pToplevel->earlyCleanup );
    }else{
      pTabToDel->nTabRef--;
@@ -5628,8 +5626,7 @@ static struct Cte *searchWith(
 With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){
  if( pWith ){
    if( bFree ){
-      pWith = (With*)sqlite3ParserAddCleanup(pParse,
-                      (void(*)(sqlite3*,void*))sqlite3WithDelete,
+      pWith = (With*)sqlite3ParserAddCleanup(pParse, sqlite3WithDeleteGeneric,
                      pWith);
      if( pWith==0 ) return 0;
    }
@@ -7037,7 +7034,8 @@ static SrcItem *isSelfJoinView(
 /*
 ** Deallocate a single AggInfo object
 */
-static void agginfoFree(sqlite3 *db, AggInfo *p){
+static void agginfoFree(sqlite3 *db, void *pArg){
+  AggInfo *p = (AggInfo*)pArg;
  sqlite3DbFree(db, p->aCol);
  sqlite3DbFree(db, p->aFunc);
  sqlite3DbFreeNN(db, p);
@@ -7291,9 +7289,8 @@ int sqlite3Select(
        sqlite3TreeViewExprList(0, p->pOrderBy, 0, "ORDERBY");
      }
 #endif   
-      sqlite3ParserAddCleanup(pParse,
-        (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
-        p->pOrderBy);
+      sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric,
+                              p->pOrderBy);
      testcase( pParse->earlyCleanup );
      p->pOrderBy = 0;
    }
@@ -7485,9 +7482,8 @@ int sqlite3Select(
    ){
      TREETRACE(0x800,pParse,p,
                ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1));
-      sqlite3ParserAddCleanup(pParse,
-         (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
-         pSub->pOrderBy);
+      sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric,
+                              pSub->pOrderBy);
      pSub->pOrderBy = 0;
    }

@@ -8016,8 +8012,7 @@ int sqlite3Select(
    */
    pAggInfo = sqlite3DbMallocZero(db, sizeof(*pAggInfo) );
    if( pAggInfo ){
-      sqlite3ParserAddCleanup(pParse,
-          (void(*)(sqlite3*,void*))agginfoFree, pAggInfo);
+      sqlite3ParserAddCleanup(pParse, agginfoFree, pAggInfo);
      testcase( pParse->earlyCleanup );
    }
    if( db->mallocFailed ){
@@ -251,6 +251,7 @@ INCLUDE ../ext/consio/console_io.h
 INCLUDE ../ext/consio/console_io.c

 #ifndef SQLITE_SHELL_FIDDLE
+
 /* From here onward, fgets() is redirected to the console_io library. */
 # define fgets(b,n,f) fGetsUtf8(b,n,f)
 /*
@@ -275,6 +276,7 @@ INCLUDE ../ext/consio/console_io.c
 # define eputz(z) ePutsUtf8(z)
 # define eputf ePrintfUtf8
 # define oputb(buf,na) oPutbUtf8(buf,na)
+
 #else
 /* For Fiddle, all console handling and emit redirection is omitted. */
 # define sputz(fp,z) fputs(z,fp)
@@ -358,7 +360,7 @@ static void endTimer(void){
    sqlite3_int64 iEnd = timeOfDay();
    struct rusage sEnd;
    getrusage(RUSAGE_SELF, &sEnd);
-    oputf("Run Time: real %.3f user %f sys %f\n",
+    sputf(stdout, "Run Time: real %.3f user %f sys %f\n",
          (iEnd - iBegin)*0.001,
          timeDiff(&sBegin.ru_utime, &sEnd.ru_utime),
          timeDiff(&sBegin.ru_stime, &sEnd.ru_stime));
@@ -437,7 +439,7 @@ static void endTimer(void){
    FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd;
    sqlite3_int64 ftWallEnd = timeOfDay();
    getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd);
-    oputf("Run Time: real %.3f user %f sys %f\n",
+    sputf(stdout, "Run Time: real %.3f user %f sys %f\n",
          (ftWallEnd - ftWallBegin)*0.001,
          timeDiff(&ftUserBegin, &ftUserEnd),
          timeDiff(&ftKernelBegin, &ftKernelEnd));
@@ -10754,6 +10756,7 @@ static int do_meta_command(char *zLine, ShellState *p){
    {"fk_no_action",       SQLITE_TESTCTRL_FK_NO_ACTION, 0, "BOOLEAN"       },
    {"imposter",         SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"},
    {"internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,""          },
+    {"json_selfcheck",     SQLITE_TESTCTRL_JSON_SELFCHECK ,0,"BOOLEAN"      },
    {"localtime_fault",    SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN"      },
    {"never_corrupt",      SQLITE_TESTCTRL_NEVER_CORRUPT,1, "BOOLEAN"       },
    {"optimizations",      SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK"   },
@@ -10972,6 +10975,16 @@ static int do_meta_command(char *zLine, ShellState *p){
            isOk = 3;
          }
          break;
+        case SQLITE_TESTCTRL_JSON_SELFCHECK:
+          if( nArg==2 ){
+            rc2 = -1;
+            isOk = 1;
+          }else{
+            rc2 = booleanValue(azArg[2]);
+            isOk = 3;
+          }
+          sqlite3_test_control(testctrl, &rc2);
+          break;
      }
    }
    if( isOk==0 && iCtrl>=0 ){
@@ -11858,14 +11871,14 @@ static void printBold(const char *zText){
         FOREGROUND_RED|FOREGROUND_INTENSITY
  );
 #endif
-  oputz(zText);
+  sputz(stdout, zText);
 #if !SQLITE_OS_WINRT
  SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes);
 #endif
 }
 #else
 static void printBold(const char *zText){
-  oputf("\033[1m%s\033[0m", zText);
+  sputf(stdout, "\033[1m%s\033[0m", zText);
 }
 #endif

@@ -12059,10 +12072,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
    }else if( cli_strcmp(z,"-init")==0 ){
      zInitFile = cmdline_option_value(argc, argv, ++i);
    }else if( cli_strcmp(z,"-interactive")==0 ){
-      /* Need to check for interactive override here to so that it can
-      ** affect console setup (for Windows only) and testing thereof.
-      */
-      stdin_is_interactive = 1;
    }else if( cli_strcmp(z,"-batch")==0 ){
      /* Need to check for batch mode here to so we can avoid printing
      ** informational messages (like from process_sqliterc) before
@@ -12332,11 +12341,14 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
    }else if( cli_strcmp(z,"-bail")==0 ){
      /* No-op.  The bail_on_error flag should already be set. */
    }else if( cli_strcmp(z,"-version")==0 ){
-      oputf("%s %s (%d-bit)\n", sqlite3_libversion(), sqlite3_sourceid(),
-            8*(int)sizeof(char*));
+      sputf(stdout, "%s %s (%d-bit)\n",
+            sqlite3_libversion(), sqlite3_sourceid(), 8*(int)sizeof(char*));
      return 0;
    }else if( cli_strcmp(z,"-interactive")==0 ){
-      /* already handled */
+      /* Need to check for interactive override here to so that it can
+      ** affect console setup (for Windows only) and testing thereof.
+      */
+      stdin_is_interactive = 1;
    }else if( cli_strcmp(z,"-batch")==0 ){
      /* already handled */
    }else if( cli_strcmp(z,"-utf8")==0 ){
@@ -12465,13 +12477,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #else
 # define SHELL_CIO_CHAR_SET ""
 #endif
-      oputf("SQLite version %s %.19s%s\n" /*extra-version-info*/
+      sputf(stdout, "SQLite version %s %.19s%s\n" /*extra-version-info*/
            "Enter \".help\" for usage hints.\n",
            sqlite3_libversion(), sqlite3_sourceid(), SHELL_CIO_CHAR_SET);
      if( warnInmemoryDb ){
-        oputz("Connected to a ");
+        sputz(stdout, "Connected to a ");
        printBold("transient in-memory database");
-        oputz(".\nUse \".open FILENAME\" to reopen on a"
+        sputz(stdout, ".\nUse \".open FILENAME\" to reopen on a"
              " persistent database.\n");
      }
      zHistory = getenv("SQLITE_HISTORY");
@@ -3954,15 +3954,17 @@ void sqlite3_free_filename(sqlite3_filename);
 ** </ul>
 **
 ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language
-** text that describes the error, as either UTF-8 or UTF-16 respectively.
+** text that describes the error, as either UTF-8 or UTF-16 respectively,
+** or NULL if no error message is available.
 ** (See how SQLite handles [invalid UTF] for exceptions to this rule.)
 ** ^(Memory to hold the error message string is managed internally.
 ** The application does not need to worry about freeing the result.
 ** However, the error string might be overwritten or deallocated by
 ** subsequent calls to other SQLite interface functions.)^
 **
-** ^The sqlite3_errstr() interface returns the English-language text
-** that describes the [result code], as UTF-8.
+** ^The sqlite3_errstr(E) interface returns the English-language text
+** that describes the [result code] E, as UTF-8, or NULL if E is not an
+** result code for which a text error message is available.
 ** ^(Memory to hold the error message string is managed internally
 ** and must not be freed by the application)^.
 **
@@ -8298,6 +8300,7 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_ASSERT                  12
 #define SQLITE_TESTCTRL_ALWAYS                  13
 #define SQLITE_TESTCTRL_RESERVE                 14  /* NOT USED */
+#define SQLITE_TESTCTRL_JSON_SELFCHECK          14
 #define SQLITE_TESTCTRL_OPTIMIZATIONS           15
 #define SQLITE_TESTCTRL_ISKEYWORD               16  /* NOT USED */
 #define SQLITE_TESTCTRL_SCRATCHMALLOC           17  /* NOT USED */
--- a/Show More
+++ b/Show More
@@ -1 +1 @@
 .44.2
 .45.0