Add NEVER to two unreachable branches in JSON.

FossilOrigin-Name: c96ebb086feb89341565cc52b970ae7799ce1327fe1ad4fc790f1b0dcaa6e229

Makefile.in

@@ -710,6 +710,21 @@ fuzzcheck-asan$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
 fuzzcheck-ubsan$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
 	$(LTLINK) -o $@ -fsanitize=undefined $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
 
+# Usage:    FUZZDB=filename make run-fuzzcheck
+#
+# Where filename is a fuzzcheck database, this target builds and runs
+# fuzzcheck, fuzzcheck-asan, and fuzzcheck-ubsan on that database.
+#
+# FUZZDB can be a glob pattern of two or more databases. Example:
+#
+#     FUZZDB=test/fuzzdata*.db make run-fuzzcheck
+#
+run-fuzzcheck:	fuzzcheck$(TEXE) fuzzcheck-asan$(TEXE) fuzzcheck-ubsan$(TEXE)
+	@if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi
+	./fuzzcheck$(TEXE) --spinner $(FUZZDB)
+	./fuzzcheck-asan$(TEXE) --spinner $(FUZZDB)
+	./fuzzcheck-ubsan$(TEXE) --spinner $(FUZZDB)
+
 ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
              $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
@@ -1135,19 +1150,21 @@ keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
 # Source files that go into making shell.c
 SHELL_SRC = \
 	$(TOP)/src/shell.c.in \
-        $(TOP)/ext/misc/appendvfs.c \
+	$(TOP)/ext/consio/console_io.c \
+	$(TOP)/ext/consio/console_io.h \
+	$(TOP)/ext/misc/appendvfs.c \
 	$(TOP)/ext/misc/completion.c \
-        $(TOP)/ext/misc/decimal.c \
-        $(TOP)/ext/misc/basexx.c \
-        $(TOP)/ext/misc/base64.c \
-        $(TOP)/ext/misc/base85.c \
+	$(TOP)/ext/misc/decimal.c \
+	$(TOP)/ext/misc/basexx.c \
+	$(TOP)/ext/misc/base64.c \
+	$(TOP)/ext/misc/base85.c \
 	$(TOP)/ext/misc/fileio.c \
-        $(TOP)/ext/misc/ieee754.c \
-        $(TOP)/ext/misc/regexp.c \
-        $(TOP)/ext/misc/series.c \
+	$(TOP)/ext/misc/ieee754.c \
+	$(TOP)/ext/misc/regexp.c \
+	$(TOP)/ext/misc/series.c \
 	$(TOP)/ext/misc/shathree.c \
 	$(TOP)/ext/misc/sqlar.c \
-        $(TOP)/ext/misc/uint.c \
+	$(TOP)/ext/misc/uint.c \
 	$(TOP)/ext/expert/sqlite3expert.c \
 	$(TOP)/ext/expert/sqlite3expert.h \
 	$(TOP)/ext/misc/zipfile.c \
@@ -1156,7 +1173,7 @@ SHELL_SRC = \
 	$(TOP)/ext/recover/dbdata.c \
 	$(TOP)/ext/recover/sqlite3recover.c \
 	$(TOP)/ext/recover/sqlite3recover.h \
-        $(TOP)/src/test_windirent.c
+	$(TOP)/src/test_windirent.c
 
 shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl has_tclsh84
 	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c

Makefile.msc

@@ -2261,6 +2261,8 @@ keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
 # Source files that go into making shell.c
 SHELL_SRC = \
 	$(TOP)\src\shell.c.in \
+	$(TOP)\ext\consio\console_io.c \
+	$(TOP)\ext\consio\console_io.h \
 	$(TOP)\ext\misc\appendvfs.c \
 	$(TOP)\ext\misc\completion.c \
 	$(TOP)\ext\misc\base64.c \

README.md

@@ -159,7 +159,7 @@ extension and only later escaped to the wild as an independent library.)
 
 Test scripts and programs are found in the **test/** subdirectory.
 Additional test code is found in other source repositories.
-See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for
+See [How SQLite Is Tested](https://www.sqlite.org/testing.html) for
 additional information.
 
 The **ext/** subdirectory contains code for extensions.  The
@@ -183,7 +183,7 @@ manually-edited files and automatically-generated files.
 
 The SQLite interface is defined by the **sqlite3.h** header file, which is
 generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION.  The
-[Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
+[Tcl script](https://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
 The manifest.uuid file contains the SHA3 hash of the particular check-in
 and is used to generate the SQLITE\_SOURCE\_ID macro.  The VERSION file
 contains the current SQLite version number.  The sqlite3.h header is really
@@ -250,14 +250,14 @@ individual source file exceeds 32K lines in length.
 ## How It All Fits Together
 
 SQLite is modular in design.
-See the [architectural description](http://www.sqlite.org/arch.html)
+See the [architectural description](https://www.sqlite.org/arch.html)
 for details. Other documents that are useful in
 (helping to understand how SQLite works include the
-[file format](http://www.sqlite.org/fileformat2.html) description,
-the [virtual machine](http://www.sqlite.org/opcode.html) that runs
+[file format](https://www.sqlite.org/fileformat2.html) description,
+the [virtual machine](https://www.sqlite.org/opcode.html) that runs
 prepared statements, the description of
-[how transactions work](http://www.sqlite.org/atomiccommit.html), and
-the [overview of the query planner](http://www.sqlite.org/optoverview.html).
+[how transactions work](https://www.sqlite.org/atomiccommit.html), and
+the [overview of the query planner](https://www.sqlite.org/optoverview.html).
 
 Years of effort have gone into optimizing SQLite, both
 for small size and high performance.  And optimizations tend to result in
@@ -353,7 +353,7 @@ hidden by also modifying the makefiles.
 
 ## Contacts
 
-The main SQLite website is [http:/sqlite.org/](http://sqlite.org/)
+The main SQLite website is [https://sqlite.org/](https://sqlite.org/)
 with geographically distributed backups at
-[http://www2.sqlite.org/](http://www2.sqlite.org) and
-[http://www3.sqlite.org/](http://www3.sqlite.org).
+[https://www2.sqlite.org/](https://www2.sqlite.org) and
+[https://www3.sqlite.org/](https://www3.sqlite.org).

doc/testrunner.md

@@ -2,6 +2,26 @@
 
 # The testrunner.tcl Script
 
+<ul type=none>
+  <li> 1. <a href=#overview>Overview</a>
+  <li> 2. <a href=#binary_tests>Binary Tests</a>
+<ul type=none>
+  <li> 2.1. <a href=#organization_tests>Organization of Tcl Tests</a>
+  <li> 2.2. <a href=#run_tests>Commands to Run Tests</a>
+  <li> 2.3. <a href=#binary_test_failures>Investigating Binary Test Failures</a>
+</ul>
+  <li> 3. <a href=#source_code_tests>Source Tests</a>
+<ul type=none>
+  <li> 3.1. <a href=#commands_to_run_tests>Commands to Run SQLite Tests</a>
+  <li> 3.2. <a href=#zipvfs_tests>Running ZipVFS Tests</a>
+  <li> 3.3. <a href=#source_code_test_failures>Investigating Source Code Test Failures</a>
+</ul>
+  <li> 4. <a href=#testrunner_options>Extra testrunner.tcl Options</a>
+# 4. Extra testrunner.tcl Options
+  <li> 5. <a href=#cpu_cores>Controlling CPU Core Utilization</a>
+</ul>
+
+<a name=overview></a>
 # 1. Overview
 
 testrunner.tcl is a Tcl script used to run multiple SQLite tests using 
@@ -44,6 +64,7 @@ Sometimes testrunner.tcl uses the [testfixture] binary that it is run with
 to run tests (see "Binary Tests" below). Sometimes it builds testfixture and
 other binaries in specific configurations to test (see "Source Tests").
 
+<a name=binary_tests></a>
 # 2. Binary Tests
 
 The commands described in this section all run various combinations of the Tcl
@@ -61,6 +82,7 @@ these tests is therefore:
 The following sub-sections describe the various options that can be
 passed to testrunner.tcl to test binary testfixture builds.
 
+<a name=organization_tests></a>
 ## 2.1. Organization of Tcl Tests
 
 Tcl tests are stored in files that match the pattern *\*.test*. They are
@@ -91,6 +113,7 @@ Running **all** tests is to run all tests in the full test set, plus a dozen
 or so permutations. The specific permutations that are run as part of "all"
 are defined in file *testrunner_data.tcl*.
 
+<a name=run_tests></a>
 ## 2.2. Commands to Run Tests
 
 To run the "veryquick" test set, use either of the following:
@@ -114,6 +137,12 @@ a specified pattern (e.g. all tests that start with "fts5"), either of:
   ./testfixture $TESTDIR/testrunner.tcl 'fts5*'
 ```
 
+Strictly speaking, for a test to be run the pattern must match the script
+filename, not including the directory, using the rules of Tcl's 
+\[string match\] command. Except that before the matching is done, any "%"
+characters specified as part of the pattern are transformed to "\*".
+
+
 To run "all" tests (full + permutations):
 
 ```
@@ -141,6 +170,7 @@ Or, if the failure occured as part of a permutation:
 
 TODO: An example instead of "$PERMUTATION" and $PATH\_TO\_SCRIPT?
 
+<a name=source_code_tests></a>
 # 3. Source Code Tests
 
 The commands described in this section invoke the C compiler to build 
@@ -159,7 +189,8 @@ shell that supports SQLite 3.31.1 or newer via "package require sqlite3".
 
 TODO: ./configure + Makefile.msc build systems.
 
-## Commands to Run SQLite Tests
+<a name=commands_to_run_tests></a>
+## 3.1. Commands to Run SQLite Tests
 
 The **mdevtest** command is equivalent to running the veryquick tests and
 the [make fuzztest] target once for each of two --enable-all builds - one 
@@ -201,7 +232,18 @@ of the specific tests run.
   tclsh $TESTDIR/testrunner.tcl release
 ```
 
-## Running ZipVFS Tests
+As with <a href=#source code tests>source code tests</a>, one or more patterns
+may be appended to any of the above commands (mdevtest, sdevtest or release).
+In that case only Tcl tests (no fuzz or other tests) that match the specified
+pattern are run. For example, to run the just the Tcl rtree tests in all 
+builds and configurations supported by "release":
+
+```
+  tclsh $TESTDIR/testrunner.tcl release rtree%
+```
+
+<a name=zipvfs_tests></a>
+## 3.2. Running ZipVFS Tests
 
 testrunner.tcl can build a zipvfs-enabled testfixture and use it to run
 tests from the Zipvfs project with the following command:
@@ -217,7 +259,8 @@ test both SQLite and Zipvfs with a single command:
   tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS mdevtest
 ```
 
-## Investigating Source Code Test Failures
+<a name=source_code_test_failures></a>
+## 3.3. Investigating Source Code Test Failures
 
 Investigating a test failure that occurs during source code testing is a
 two step process:
@@ -244,9 +287,31 @@ target to build. This may be used either to run a [make] command test directly,
 or else to build a testfixture (or testfixture.exe) binary with which to
 run a Tcl test script, as <a href=#binary_test_failures>described above</a>.
 
+<a name=testrunner_options></a>
+# 4. Extra testrunner.tcl Options
 
+The testrunner.tcl script options in this section may be used with both source
+code and binary tests.
 
-# 4. Controlling CPU Core Utilization
+The **--buildonly** option instructs testrunner.tcl just to build the binaries
+required by a test, not to run any actual tests. For example:
+
+```
+  # Build binaries required by release test.
+  tclsh $TESTDIR/testrunner.tcl --buildonly release"
+```
+
+The **--dryrun** option prevents testrunner.tcl from building any binaries
+or running any tests. Instead, it just writes the shell commands that it
+would normally execute into the testrunner.log file. Example:
+
+```
+  # Log the shell commmands that make up the mdevtest test.
+  tclsh $TESTDIR/testrunner.tcl --dryrun mdevtest"
+```
+
+<a name=cpu_cores></a>
+# 5. Controlling CPU Core Utilization
 
 When running either binary or source code tests, testrunner.tcl reports the
 number of jobs it intends to use to stdout. e.g.
@@ -277,8 +342,3 @@ testrunner.log and testrunner.db files:
 
 
 
-
-
-
-
-

ext/consio/console_io.c

@@ -0,0 +1,678 @@
+/*
+** 2023 November 4
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+********************************************************************************
+** This file implements various interfaces used for console and stream I/O
+** by the SQLite project command-line tools, as explained in console_io.h .
+** Functions prefixed by "SQLITE_INTERNAL_LINKAGE" behave as described there.
+*/
+
+#ifndef SQLITE_CDECL
+# define SQLITE_CDECL
+#endif
+
+#ifndef SHELL_NO_SYSINC
+# include <stdarg.h>
+# include <string.h>
+# include <stdlib.h>
+# include <limits.h>
+# include <assert.h>
+# include "console_io.h"
+# include "sqlite3.h"
+#endif
+
+#ifndef SQLITE_CIO_NO_TRANSLATE
+# if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT
+#  ifndef SHELL_NO_SYSINC
+#   include <io.h>
+#   include <fcntl.h>
+#   undef WIN32_LEAN_AND_MEAN
+#   define WIN32_LEAN_AND_MEAN
+#   include <windows.h>
+#  endif
+#  define CIO_WIN_WC_XLATE 1 /* Use WCHAR Windows APIs for console I/O */
+# else
+#  ifndef SHELL_NO_SYSINC
+#   include <unistd.h>
+#  endif
+#  define CIO_WIN_WC_XLATE 0 /* Use plain C library stream I/O at console */
+# endif
+#else
+# define CIO_WIN_WC_XLATE 0 /* Not exposing translation routines at all */
+#endif
+
+#if CIO_WIN_WC_XLATE
+/* Character used to represent a known-incomplete UTF-8 char group (�) */
+static WCHAR cBadGroup = 0xfffd;
+#endif
+
+#if CIO_WIN_WC_XLATE
+static HANDLE handleOfFile(FILE *pf){
+  int fileDesc = _fileno(pf);
+  union { intptr_t osfh; HANDLE fh; } fid = {
+    (fileDesc>=0)? _get_osfhandle(fileDesc) : (intptr_t)INVALID_HANDLE_VALUE
+  };
+  return fid.fh;
+}
+#endif
+
+#ifndef SQLITE_CIO_NO_TRANSLATE
+typedef struct PerStreamTags {
+# if CIO_WIN_WC_XLATE
+  HANDLE hx;
+  DWORD consMode;
+  char acIncomplete[4];
+# else
+  short reachesConsole;
+# endif
+  FILE *pf;
+} PerStreamTags;
+
+/* Define NULL-like value for things which can validly be 0. */
+# define SHELL_INVALID_FILE_PTR ((FILE *)~0)
+# if CIO_WIN_WC_XLATE
+#  define SHELL_INVALID_CONS_MODE 0xFFFF0000
+# endif
+
+# if CIO_WIN_WC_XLATE
+#  define PST_INITIALIZER { INVALID_HANDLE_VALUE, SHELL_INVALID_CONS_MODE, \
+      {0,0,0,0}, SHELL_INVALID_FILE_PTR }
+# else
+#  define PST_INITIALIZER { 0, SHELL_INVALID_FILE_PTR }
+# endif
+
+/* Quickly say whether a known output is going to the console. */
+# if CIO_WIN_WC_XLATE
+static short pstReachesConsole(PerStreamTags *ppst){
+  return (ppst->hx != INVALID_HANDLE_VALUE);
+}
+# else
+#  define pstReachesConsole(ppst) 0
+# endif
+
+# if CIO_WIN_WC_XLATE
+static void restoreConsoleArb(PerStreamTags *ppst){
+  if( pstReachesConsole(ppst) ) SetConsoleMode(ppst->hx, ppst->consMode);
+}
+# else
+#  define restoreConsoleArb(ppst)
+# endif
+
+/* Say whether FILE* appears to be a console, collect associated info. */
+static short streamOfConsole(FILE *pf, /* out */ PerStreamTags *ppst){
+# if CIO_WIN_WC_XLATE
+  short rv = 0;
+  DWORD dwCM = SHELL_INVALID_CONS_MODE;
+  HANDLE fh = handleOfFile(pf);
+  ppst->pf = pf;
+  if( INVALID_HANDLE_VALUE != fh ){
+    rv = (GetFileType(fh) == FILE_TYPE_CHAR && GetConsoleMode(fh,&dwCM));
+  }
+  ppst->hx = (rv)? fh : INVALID_HANDLE_VALUE;
+  ppst->consMode = dwCM;
+  return rv;
+# else
+  ppst->pf = pf;
+  ppst->reachesConsole = ( (short)isatty(fileno(pf)) );
+  return ppst->reachesConsole;
+# endif
+}
+
+# if CIO_WIN_WC_XLATE
+/* Define console modes for use with the Windows Console API. */
+#  define SHELL_CONI_MODE \
+  (ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | 0x80 \
+  | ENABLE_QUICK_EDIT_MODE | ENABLE_EXTENDED_FLAGS | ENABLE_PROCESSED_INPUT)
+#  define SHELL_CONO_MODE (ENABLE_PROCESSED_OUTPUT | ENABLE_WRAP_AT_EOL_OUTPUT \
+  | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
+# endif
+
+typedef struct ConsoleInfo {
+  PerStreamTags pstSetup[3];
+  PerStreamTags pstDesignated[3];
+  StreamsAreConsole sacSetup;
+} ConsoleInfo;
+
+static short isValidStreamInfo(PerStreamTags *ppst){
+  return (ppst->pf != SHELL_INVALID_FILE_PTR);
+}
+
+static ConsoleInfo consoleInfo = {
+  { /* pstSetup */ PST_INITIALIZER, PST_INITIALIZER, PST_INITIALIZER },
+  { /* pstDesignated[] */ PST_INITIALIZER, PST_INITIALIZER, PST_INITIALIZER },
+  SAC_NoConsole /* sacSetup */
+};
+
+SQLITE_INTERNAL_LINKAGE FILE* invalidFileStream = (FILE *)~0;
+
+# if CIO_WIN_WC_XLATE
+static void maybeSetupAsConsole(PerStreamTags *ppst, short odir){
+  if( pstReachesConsole(ppst) ){
+    DWORD cm = odir? SHELL_CONO_MODE : SHELL_CONI_MODE;
+    SetConsoleMode(ppst->hx, cm);
+  }
+}
+# else
+#  define maybeSetupAsConsole(ppst,odir)
+# endif
+
+SQLITE_INTERNAL_LINKAGE void consoleRenewSetup(void){
+# if CIO_WIN_WC_XLATE
+  int ix = 0;
+  while( ix < 6 ){
+    PerStreamTags *ppst = (ix<3)?
+      &consoleInfo.pstSetup[ix] : &consoleInfo.pstDesignated[ix-3];
+    maybeSetupAsConsole(ppst, (ix % 3)>0);
+    ++ix;
+  }
+# endif
+}
+
+SQLITE_INTERNAL_LINKAGE StreamsAreConsole
+consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr ){
+  StreamsAreConsole rv = SAC_NoConsole;
+  FILE* apf[3] = { pfIn, pfOut, pfErr };
+  int ix;
+  for( ix = 2; ix >= 0; --ix ){
+    PerStreamTags *ppst = &consoleInfo.pstSetup[ix];
+    if( streamOfConsole(apf[ix], ppst) ){
+      rv |= (SAC_InConsole<<ix);
+    }
+    consoleInfo.pstDesignated[ix] = *ppst;
+    if( ix > 0 ) fflush(apf[ix]);
+  }
+  consoleInfo.sacSetup = rv;
+  consoleRenewSetup();
+  return rv;
+}
+
+SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void ){
+# if CIO_WIN_WC_XLATE
+  static ConsoleInfo *pci = &consoleInfo;
+  if( pci->sacSetup ){
+    int ix;
+    for( ix=0; ix<3; ++ix ){
+      if( pci->sacSetup & (SAC_InConsole<<ix) ){
+        PerStreamTags *ppst = &pci->pstSetup[ix];
+        SetConsoleMode(ppst->hx, ppst->consMode);
+      }
+    }
+  }
+# endif
+}
+#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
+
+#ifdef SQLITE_CIO_INPUT_REDIR
+/* Say whether given FILE* is among those known, via either
+** consoleClassifySetup() or set{Output,Error}Stream, as
+** readable, and return an associated PerStreamTags pointer
+** if so. Otherwise, return 0.
+*/
+static PerStreamTags * isKnownReadable(FILE *pf){
+  static PerStreamTags *apst[] = {
+    &consoleInfo.pstDesignated[0], &consoleInfo.pstSetup[0], 0
+  };
+  int ix = 0;
+  do {
+    if( apst[ix]->pf == pf ) break;
+  } while( apst[++ix] != 0 );
+  return apst[ix];
+}
+#endif
+
+#ifndef SQLITE_CIO_NO_TRANSLATE
+/* Say whether given FILE* is among those known, via either
+** consoleClassifySetup() or set{Output,Error}Stream, as
+** writable, and return an associated PerStreamTags pointer
+** if so. Otherwise, return 0.
+*/
+static PerStreamTags * isKnownWritable(FILE *pf){
+  static PerStreamTags *apst[] = {
+    &consoleInfo.pstDesignated[1], &consoleInfo.pstDesignated[2],
+    &consoleInfo.pstSetup[1], &consoleInfo.pstSetup[2], 0
+  };
+  int ix = 0;
+  do {
+    if( apst[ix]->pf == pf ) break;
+  } while( apst[++ix] != 0 );
+  return apst[ix];
+}
+
+static FILE *designateEmitStream(FILE *pf, unsigned chix){
+  FILE *rv = consoleInfo.pstDesignated[chix].pf;
+  if( pf == invalidFileStream ) return rv;
+  else{
+    /* Setting a possibly new output stream. */
+    PerStreamTags *ppst = isKnownWritable(pf);
+    if( ppst != 0 ){
+      PerStreamTags pst = *ppst;
+      consoleInfo.pstDesignated[chix] = pst;
+    }else streamOfConsole(pf, &consoleInfo.pstDesignated[chix]);
+  }
+  return rv;
+}
+
+SQLITE_INTERNAL_LINKAGE FILE *setOutputStream(FILE *pf){
+  return designateEmitStream(pf, 1);
+}
+# ifdef CONSIO_SET_ERROR_STREAM
+SQLITE_INTERNAL_LINKAGE FILE *setErrorStream(FILE *pf){
+  return designateEmitStream(pf, 2);
+}
+# endif
+#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
+
+#ifndef SQLITE_CIO_NO_SETMODE
+# if CIO_WIN_WC_XLATE
+static void setModeFlushQ(FILE *pf, short bFlush, int mode){
+  if( bFlush ) fflush(pf);
+  _setmode(_fileno(pf), mode);
+}
+# else
+#  define setModeFlushQ(f, b, m) if(b) fflush(f)
+# endif
+
+SQLITE_INTERNAL_LINKAGE void setBinaryMode(FILE *pf, short bFlush){
+  setModeFlushQ(pf, bFlush, _O_BINARY);
+}
+SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *pf, short bFlush){
+  setModeFlushQ(pf, bFlush, _O_TEXT);
+}
+# undef setModeFlushQ
+
+#else /* defined(SQLITE_CIO_NO_SETMODE) */
+# define setBinaryMode(f, bFlush) do{ if((bFlush)) fflush(f); }while(0)
+# define setTextMode(f, bFlush) do{ if((bFlush)) fflush(f); }while(0)
+#endif /* defined(SQLITE_CIO_NO_SETMODE) */
+
+#ifndef SQLITE_CIO_NO_TRANSLATE
+# if CIO_WIN_WC_XLATE
+/* Write buffer cBuf as output to stream known to reach console,
+** limited to ncTake char's. Return ncTake on success, else 0. */
+static int conZstrEmit(PerStreamTags *ppst, const char *z, int ncTake){
+  int rv = 0;
+  if( z!=NULL ){
+    int nwc = MultiByteToWideChar(CP_UTF8,0, z,ncTake, 0,0);
+    if( nwc > 0 ){
+      WCHAR *zw = sqlite3_malloc64(nwc*sizeof(WCHAR));
+      if( zw!=NULL ){
+        nwc = MultiByteToWideChar(CP_UTF8,0, z,ncTake, zw,nwc);
+        if( nwc > 0 ){
+          /* Translation from UTF-8 to UTF-16, then WCHARs out. */
+          if( WriteConsoleW(ppst->hx, zw,nwc, 0, NULL) ){
+            rv = ncTake;
+          }
+        }
+        sqlite3_free(zw);
+      }
+    }
+  }
+  return rv;
+}
+
+/* For {f,o,e}PrintfUtf8() when stream is known to reach console. */
+static int conioVmPrintf(PerStreamTags *ppst, const char *zFormat, va_list ap){
+  char *z = sqlite3_vmprintf(zFormat, ap);
+  if( z ){
+    int rv = conZstrEmit(ppst, z, (int)strlen(z));
+    sqlite3_free(z);
+    return rv;
+  }else return 0;
+}
+# endif /* CIO_WIN_WC_XLATE */
+
+# ifdef CONSIO_GET_EMIT_STREAM
+static PerStreamTags * getDesignatedEmitStream(FILE *pf, unsigned chix,
+                                               PerStreamTags *ppst){
+  PerStreamTags *rv = isKnownWritable(pf);
+  short isValid = (rv!=0)? isValidStreamInfo(rv) : 0;
+  if( rv != 0 && isValid ) return rv;
+  streamOfConsole(pf, ppst);
+  return ppst;
+}
+# endif
+
+/* Get stream info, either for designated output or error stream when
+** chix equals 1 or 2, or for an arbitrary stream when chix == 0.
+** In either case, ppst references a caller-owned PerStreamTags
+** struct which may be filled in if none of the known writable
+** streams is being held by consoleInfo. The ppf parameter is a
+** byref output when chix!=0 and a byref input when chix==0.
+ */
+static PerStreamTags *
+getEmitStreamInfo(unsigned chix, PerStreamTags *ppst,
+                  /* in/out */ FILE **ppf){
+  PerStreamTags *ppstTry;
+  FILE *pfEmit;
+  if( chix > 0 ){
+    ppstTry = &consoleInfo.pstDesignated[chix];
+    if( !isValidStreamInfo(ppstTry) ){
+      ppstTry = &consoleInfo.pstSetup[chix];
+      pfEmit = ppst->pf;
+    }else pfEmit = ppstTry->pf;
+    if( !isValidStreamInfo(ppstTry) ){
+      pfEmit = (chix > 1)? stderr : stdout;
+      ppstTry = ppst;
+      streamOfConsole(pfEmit, ppstTry);
+    }
+    *ppf = pfEmit;
+  }else{
+    ppstTry = isKnownWritable(*ppf);
+    if( ppstTry != 0 ) return ppstTry;
+    streamOfConsole(*ppf, ppst);
+    return ppst;
+  }
+  return ppstTry;
+}
+
+SQLITE_INTERNAL_LINKAGE int oPrintfUtf8(const char *zFormat, ...){
+  va_list ap;
+  int rv;
+  FILE *pfOut;
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+# if CIO_WIN_WC_XLATE
+  PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut);
+# else
+  getEmitStreamInfo(1, &pst, &pfOut);
+# endif
+  assert(zFormat!=0);
+  va_start(ap, zFormat);
+# if CIO_WIN_WC_XLATE
+  if( pstReachesConsole(ppst) ){
+    rv = conioVmPrintf(ppst, zFormat, ap);
+  }else{
+# endif
+    rv = vfprintf(pfOut, zFormat, ap);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+  va_end(ap);
+  return rv;
+}
+
+SQLITE_INTERNAL_LINKAGE int ePrintfUtf8(const char *zFormat, ...){
+  va_list ap;
+  int rv;
+  FILE *pfErr;
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+# if CIO_WIN_WC_XLATE
+  PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr);
+# else
+  getEmitStreamInfo(2, &pst, &pfErr);
+# endif
+  assert(zFormat!=0);
+  va_start(ap, zFormat);
+# if CIO_WIN_WC_XLATE
+  if( pstReachesConsole(ppst) ){
+    rv = conioVmPrintf(ppst, zFormat, ap);
+  }else{
+# endif
+    rv = vfprintf(pfErr, zFormat, ap);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+  va_end(ap);
+  return rv;
+}
+
+SQLITE_INTERNAL_LINKAGE int fPrintfUtf8(FILE *pfO, const char *zFormat, ...){
+  va_list ap;
+  int rv;
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+# if CIO_WIN_WC_XLATE
+  PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
+# else
+  getEmitStreamInfo(0, &pst, &pfO);
+# endif
+  assert(zFormat!=0);
+  va_start(ap, zFormat);
+# if CIO_WIN_WC_XLATE
+  if( pstReachesConsole(ppst) ){
+    maybeSetupAsConsole(ppst, 1);
+    rv = conioVmPrintf(ppst, zFormat, ap);
+    if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
+  }else{
+# endif
+    rv = vfprintf(pfO, zFormat, ap);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+  va_end(ap);
+  return rv;
+}
+
+SQLITE_INTERNAL_LINKAGE int fPutsUtf8(const char *z, FILE *pfO){
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+# if CIO_WIN_WC_XLATE
+  PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
+# else
+  getEmitStreamInfo(0, &pst, &pfO);
+# endif
+  assert(z!=0);
+# if CIO_WIN_WC_XLATE
+  if( pstReachesConsole(ppst) ){
+    int rv;
+    maybeSetupAsConsole(ppst, 1);
+    rv = conZstrEmit(ppst, z, (int)strlen(z));
+    if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
+    return rv;
+  }else {
+# endif
+    return (fputs(z, pfO)<0)? 0 : (int)strlen(z);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+}
+
+SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z){
+  FILE *pfErr;
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+# if CIO_WIN_WC_XLATE
+  PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr);
+# else
+  getEmitStreamInfo(2, &pst, &pfErr);
+# endif
+  assert(z!=0);
+# if CIO_WIN_WC_XLATE
+  if( pstReachesConsole(ppst) ) return conZstrEmit(ppst, z, (int)strlen(z));
+  else {
+# endif
+    return (fputs(z, pfErr)<0)? 0 : (int)strlen(z);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+}
+
+SQLITE_INTERNAL_LINKAGE int oPutsUtf8(const char *z){
+  FILE *pfOut;
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+# if CIO_WIN_WC_XLATE
+  PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut);
+# else
+  getEmitStreamInfo(1, &pst, &pfOut);
+# endif
+  assert(z!=0);
+# if CIO_WIN_WC_XLATE
+  if( pstReachesConsole(ppst) ) return conZstrEmit(ppst, z, (int)strlen(z));
+  else {
+# endif
+    return (fputs(z, pfOut)<0)? 0 : (int)strlen(z);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+}
+
+#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
+
+#if !(defined(SQLITE_CIO_NO_UTF8SCAN) && defined(SQLITE_CIO_NO_TRANSLATE))
+/* Skip over as much z[] input char sequence as is valid UTF-8,
+** limited per nAccept char's or whole characters and containing
+** no char cn such that ((1<<cn) & ccm)!=0. On return, the
+** sequence z:return (inclusive:exclusive) is validated UTF-8.
+** Limit: nAccept>=0 => char count, nAccept<0 => character
+ */
+SQLITE_INTERNAL_LINKAGE const char*
+zSkipValidUtf8(const char *z, int nAccept, long ccm){
+  int ng = (nAccept<0)? -nAccept : 0;
+  const char *pcLimit = (nAccept>=0)? z+nAccept : 0;
+  assert(z!=0);
+  while( (pcLimit)? (z<pcLimit) : (ng-- != 0) ){
+    char c = *z;
+    if( (c & 0x80) == 0 ){
+      if( ccm != 0L && c < 0x20 && ((1L<<c) & ccm) != 0 ) return z;
+      ++z; /* ASCII */
+    }else if( (c & 0xC0) != 0xC0 ) return z; /* not a lead byte */
+    else{
+      const char *zt = z+1; /* Got lead byte, look at trail bytes.*/
+      do{
+        if( pcLimit && zt >= pcLimit ) return z;
+        else{
+          char ct = *zt++;
+          if( ct==0 || (zt-z)>4 || (ct & 0xC0)!=0x80 ){
+            /* Trailing bytes are too few, too many, or invalid. */
+            return z;
+          }
+        }
+      } while( ((c <<= 1) & 0x40) == 0x40 ); /* Eat lead byte's count. */
+      z = zt;
+    }
+  }
+  return z;
+}
+#endif /*!(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE))*/
+
+#ifndef SQLITE_CIO_NO_TRANSLATE
+# ifdef CONSIO_SPUTB
+SQLITE_INTERNAL_LINKAGE int
+fPutbUtf8(FILE *pfO, const char *cBuf, int nAccept){
+  assert(pfO!=0);
+#  if CIO_WIN_WC_XLATE
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+  PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
+  if( pstReachesConsole(ppst) ){
+    int rv;
+    maybeSetupAsConsole(ppst, 1);
+    rv = conZstrEmit(ppst, cBuf, nAccept);
+    if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
+    return rv;
+  }else {
+#  endif
+    return (int)fwrite(cBuf, 1, nAccept, pfO);
+#  if CIO_WIN_WC_XLATE
+  }
+#  endif
+}
+# endif
+
+SQLITE_INTERNAL_LINKAGE int
+oPutbUtf8(const char *cBuf, int nAccept){
+  FILE *pfOut;
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+# if CIO_WIN_WC_XLATE
+  PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut);
+# else
+  getEmitStreamInfo(1, &pst, &pfOut);
+# endif
+# if CIO_WIN_WC_XLATE
+  if( pstReachesConsole(ppst) ){
+    return conZstrEmit(ppst, cBuf, nAccept);
+  }else {
+# endif
+    return (int)fwrite(cBuf, 1, nAccept, pfOut);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+}
+
+# ifdef CONSIO_EPUTB
+SQLITE_INTERNAL_LINKAGE int
+ePutbUtf8(const char *cBuf, int nAccept){
+  FILE *pfErr;
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+  PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr);
+#  if CIO_WIN_WC_XLATE
+  if( pstReachesConsole(ppst) ){
+    return conZstrEmit(ppst, cBuf, nAccept);
+  }else {
+#  endif
+    return (int)fwrite(cBuf, 1, nAccept, pfErr);
+#  if CIO_WIN_WC_XLATE
+  }
+#  endif
+}
+# endif /* defined(CONSIO_EPUTB) */
+
+SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){
+  if( pfIn==0 ) pfIn = stdin;
+# if CIO_WIN_WC_XLATE
+  if( pfIn == consoleInfo.pstSetup[0].pf
+      && (consoleInfo.sacSetup & SAC_InConsole)!=0 ){
+#  if CIO_WIN_WC_XLATE==1
+#   define SHELL_GULP 150 /* Count of WCHARS to be gulped at a time */
+    WCHAR wcBuf[SHELL_GULP+1];
+    int lend = 0, noc = 0;
+    if( ncMax > 0 ) cBuf[0] = 0;
+    while( noc < ncMax-8-1 && !lend ){
+      /* There is room for at least 2 more characters and a 0-terminator. */
+      int na = (ncMax > SHELL_GULP*4+1 + noc)? SHELL_GULP : (ncMax-1 - noc)/4;
+#   undef SHELL_GULP
+      DWORD nbr = 0;
+      BOOL bRC = ReadConsoleW(consoleInfo.pstSetup[0].hx, wcBuf, na, &nbr, 0);
+      if( bRC && nbr>0 && (wcBuf[nbr-1]&0xF800)==0xD800 ){
+        /* Last WHAR read is first of a UTF-16 surrogate pair. Grab its mate. */
+        DWORD nbrx;
+        bRC &= ReadConsoleW(consoleInfo.pstSetup[0].hx, wcBuf+nbr, 1, &nbrx, 0);
+        if( bRC ) nbr += nbrx;
+      }
+      if( !bRC || (noc==0 && nbr==0) ) return 0;
+      if( nbr > 0 ){
+        int nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,nbr,0,0,0,0);
+        if( nmb != 0 && noc+nmb <= ncMax ){
+          int iseg = noc;
+          nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,nbr,cBuf+noc,nmb,0,0);
+          noc += nmb;
+          /* Fixup line-ends as coded by Windows for CR (or "Enter".)
+          ** This is done without regard for any setMode{Text,Binary}()
+          ** call that might have been done on the interactive input.
+          */
+          if( noc > 0 ){
+            if( cBuf[noc-1]=='\n' ){
+              lend = 1;
+              if( noc > 1 && cBuf[noc-2]=='\r' ) cBuf[--noc-1] = '\n';
+            }
+          }
+          /* Check for ^Z (anywhere in line) too, to act as EOF. */
+          while( iseg < noc ){
+            if( cBuf[iseg]=='\x1a' ){
+              noc = iseg; /* Chop ^Z and anything following. */
+              lend = 1; /* Counts as end of line too. */
+              break;
+            }
+            ++iseg;
+          }
+        }else break; /* Drop apparent garbage in. (Could assert.) */
+      }else break;
+    }
+    /* If got nothing, (after ^Z chop), must be at end-of-file. */
+    if( noc > 0 ){
+      cBuf[noc] = 0;
+      return cBuf;
+    }else return 0;
+#  endif
+  }else{
+# endif
+    return fgets(cBuf, ncMax, pfIn);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+}
+#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
+
+#undef SHELL_INVALID_FILE_PTR

ext/consio/console_io.h

@@ -0,0 +1,280 @@
+/*
+** 2023 November 1
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+********************************************************************************
+** This file exposes various interfaces used for console and other I/O
+** by the SQLite project command-line tools. These interfaces are used
+** at either source conglomeration time, compilation time, or run time.
+** This source provides for either inclusion into conglomerated,
+** "single-source" forms or separate compilation then linking.
+**
+** Platform dependencies are "hidden" here by various stratagems so
+** that, provided certain conditions are met, the programs using this
+** source or object code compiled from it need no explicit conditional
+** compilation in their source for their console and stream I/O.
+**
+** The symbols and functionality exposed here are not a public API.
+** This code may change in tandem with other project code as needed.
+**
+** When this .h file and its companion .c are directly incorporated into
+** a source conglomeration (such as shell.c), the preprocessor symbol
+** CIO_WIN_WC_XLATE is defined as 0 or 1, reflecting whether console I/O
+** translation for Windows is effected for the build.
+*/
+
+#ifndef SQLITE_INTERNAL_LINKAGE
+# define SQLITE_INTERNAL_LINKAGE extern /* external to translation unit */
+# include <stdio.h>
+#else
+# define SHELL_NO_SYSINC /* Better yet, modify mkshellc.tcl for this. */
+#endif
+
+#ifndef SQLITE3_H
+# include "sqlite3.h"
+#endif
+
+#ifndef SQLITE_CIO_NO_CLASSIFY
+
+/* Define enum for use with following function. */
+typedef enum StreamsAreConsole {
+  SAC_NoConsole = 0,
+  SAC_InConsole = 1, SAC_OutConsole = 2, SAC_ErrConsole = 4,
+  SAC_AnyConsole = 0x7
+} StreamsAreConsole;
+
+/*
+** Classify the three standard I/O streams according to whether
+** they are connected to a console attached to the process.
+**
+** Returns the bit-wise OR of SAC_{In,Out,Err}Console values,
+** or SAC_NoConsole if none of the streams reaches a console.
+**
+** This function should be called before any I/O is done with
+** the given streams. As a side-effect, the given inputs are
+** recorded so that later I/O operations on them may be done
+** differently than the C library FILE* I/O would be done,
+** iff the stream is used for the I/O functions that follow,
+** and to support the ones that use an implicit stream.
+**
+** On some platforms, stream or console mode alteration (aka
+** "Setup") may be made which is undone by consoleRestore().
+*/
+SQLITE_INTERNAL_LINKAGE StreamsAreConsole
+consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr );
+/* A usual call for convenience: */
+#define SQLITE_STD_CONSOLE_INIT() consoleClassifySetup(stdin,stdout,stderr)
+
+/*
+** After an initial call to consoleClassifySetup(...), renew
+** the same setup it effected. (A call not after is an error.)
+** This will restore state altered by consoleRestore();
+**
+** Applications which run an inferior (child) process which
+** inherits the same I/O streams may call this function after
+** such a process exits to guard against console mode changes.
+*/
+SQLITE_INTERNAL_LINKAGE void consoleRenewSetup(void);
+
+/*
+** Undo any side-effects left by consoleClassifySetup(...).
+**
+** This should be called after consoleClassifySetup() and
+** before the process terminates normally. It is suitable
+** for use with the atexit() C library procedure. After
+** this call, no console I/O should be done until one of
+** console{Classify or Renew}Setup(...) is called again.
+**
+** Applications which run an inferior (child) process that
+** inherits the same I/O streams might call this procedure
+** before so that said process will have a console setup
+** however users have configured it or come to expect.
+*/
+SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void );
+
+#else /* defined(SQLITE_CIO_NO_CLASSIFY) */
+# define consoleClassifySetup(i,o,e)
+# define consoleRenewSetup()
+# define consoleRestore()
+#endif /* defined(SQLITE_CIO_NO_CLASSIFY) */
+
+#ifndef SQLITE_CIO_NO_REDIRECT
+/*
+** Set stream to be used for the functions below which write
+** to "the designated X stream", where X is Output or Error.
+** Returns the previous value.
+**
+** Alternatively, pass the special value, invalidFileStream,
+** to get the designated stream value without setting it.
+**
+** Before the designated streams are set, they default to
+** those passed to consoleClassifySetup(...), and before
+** that is called they default to stdout and stderr.
+**
+** It is error to close a stream so designated, then, without
+** designating another, use the corresponding {o,e}Emit(...).
+*/
+SQLITE_INTERNAL_LINKAGE FILE *invalidFileStream;
+SQLITE_INTERNAL_LINKAGE FILE *setOutputStream(FILE *pf);
+# ifdef CONSIO_SET_ERROR_STREAM
+SQLITE_INTERNAL_LINKAGE FILE *setErrorStream(FILE *pf);
+# endif
+#else
+# define setOutputStream(pf)
+# define setErrorStream(pf)
+#endif /* !defined(SQLITE_CIO_NO_REDIRECT) */
+
+#ifndef SQLITE_CIO_NO_TRANSLATE
+/*
+** Emit output like fprintf(). If the output is going to the
+** console and translation from UTF-8 is necessary, perform
+** the needed translation. Otherwise, write formatted output
+** to the provided stream almost as-is, possibly with newline
+** translation as specified by set{Binary,Text}Mode().
+*/
+SQLITE_INTERNAL_LINKAGE int fPrintfUtf8(FILE *pfO, const char *zFormat, ...);
+/* Like fPrintfUtf8 except stream is always the designated output. */
+SQLITE_INTERNAL_LINKAGE int oPrintfUtf8(const char *zFormat, ...);
+/* Like fPrintfUtf8 except stream is always the designated error. */
+SQLITE_INTERNAL_LINKAGE int ePrintfUtf8(const char *zFormat, ...);
+
+/*
+** Emit output like fputs(). If the output is going to the
+** console and translation from UTF-8 is necessary, perform
+** the needed translation. Otherwise, write given text to the
+** provided stream almost as-is, possibly with newline
+** translation as specified by set{Binary,Text}Mode().
+*/
+SQLITE_INTERNAL_LINKAGE int fPutsUtf8(const char *z, FILE *pfO);
+/* Like fPutsUtf8 except stream is always the designated output. */
+SQLITE_INTERNAL_LINKAGE int oPutsUtf8(const char *z);
+/* Like fPutsUtf8 except stream is always the designated error. */
+SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z);
+
+/*
+** Emit output like fPutsUtf8(), except that the length of the
+** accepted char or character sequence is limited by nAccept.
+**
+** Returns the number of accepted char values.
+*/
+#ifdef CONSIO_SPUTB
+SQLITE_INTERNAL_LINKAGE int
+fPutbUtf8(FILE *pfOut, const char *cBuf, int nAccept);
+/* Like fPutbUtf8 except stream is always the designated output. */
+#endif
+SQLITE_INTERNAL_LINKAGE int
+oPutbUtf8(const char *cBuf, int nAccept);
+/* Like fPutbUtf8 except stream is always the designated error. */
+#ifdef CONSIO_EPUTB
+SQLITE_INTERNAL_LINKAGE int
+ePutbUtf8(const char *cBuf, int nAccept);
+#endif
+
+/*
+** Collect input like fgets(...) with special provisions for input
+** from the console on platforms that require same. Defers to the
+** C library fgets() when input is not from the console. Newline
+** translation may be done as set by set{Binary,Text}Mode(). As a
+** convenience, pfIn==NULL is treated as stdin.
+*/
+SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn);
+/* Like fGetsUtf8 except stream is always the designated input. */
+/* SQLITE_INTERNAL_LINKAGE char* iGetsUtf8(char *cBuf, int ncMax); */
+
+#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
+
+#ifndef SQLITE_CIO_NO_SETMODE
+/*
+** Set given stream for binary mode, where newline translation is
+** not done, or for text mode where, for some platforms, newlines
+** are translated to the platform's conventional char sequence.
+** If bFlush true, flush the stream.
+**
+** An additional side-effect is that if the stream is one passed
+** to consoleClassifySetup() as an output, it is flushed first.
+**
+** Note that binary/text mode has no effect on console I/O
+** translation. On all platforms, newline to the console starts
+** a new line and CR,LF chars from the console become a newline.
+*/
+SQLITE_INTERNAL_LINKAGE void setBinaryMode(FILE *, short bFlush);
+SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *, short bFlush);
+#endif
+
+#ifdef SQLITE_CIO_PROMPTED_IN
+typedef struct Prompts {
+  int numPrompts;
+  const char **azPrompts;
+} Prompts;
+
+/*
+** Macros for use of a line editor.
+**
+** The following macros define operations involving use of a
+** line-editing library or simple console interaction.
+** A "T" argument is a text (char *) buffer or filename.
+** A "N" argument is an integer.
+**
+** SHELL_ADD_HISTORY(T) // Record text as line(s) of history.
+** SHELL_READ_HISTORY(T) // Read history from file named by T.
+** SHELL_WRITE_HISTORY(T) // Write history to file named by T.
+** SHELL_STIFLE_HISTORY(N) // Limit history to N entries.
+**
+** A console program which does interactive console input is
+** expected to call:
+** SHELL_READ_HISTORY(T) before collecting such input;
+** SHELL_ADD_HISTORY(T) as record-worthy input is taken;
+** SHELL_STIFLE_HISTORY(N) after console input ceases; then
+** SHELL_WRITE_HISTORY(T) before the program exits.
+*/
+
+/*
+** Retrieve a single line of input text from an input stream.
+**
+** If pfIn is the input stream passed to consoleClassifySetup(),
+** and azPrompt is not NULL, then a prompt is issued before the
+** line is collected, as selected by the isContinuation flag.
+** Array azPrompt[{0,1}] holds the {main,continuation} prompt.
+**
+** If zBufPrior is not NULL then it is a buffer from a prior
+** call to this routine that can be reused, or will be freed.
+**
+** The result is stored in space obtained from malloc() and
+** must either be freed by the caller or else passed back to
+** this function as zBufPrior for reuse.
+**
+** This function may call upon services of a line-editing
+** library to interactively collect line edited input.
+*/
+SQLITE_INTERNAL_LINKAGE char *
+shellGetLine(FILE *pfIn, char *zBufPrior, int nLen,
+             short isContinuation, Prompts azPrompt);
+#endif /* defined(SQLITE_CIO_PROMPTED_IN) */
+/*
+** TBD: Define an interface for application(s) to generate
+** completion candidates for use by the line-editor.
+**
+** This may be premature; the CLI is the only application
+** that does this. Yet, getting line-editing melded into
+** console I/O is desirable because a line-editing library
+** may have to establish console operating mode, possibly
+** in a way that interferes with the above functionality.
+*/
+
+#if !(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE))
+/* Skip over as much z[] input char sequence as is valid UTF-8,
+** limited per nAccept char's or whole characters and containing
+** no char cn such that ((1<<cn) & ccm)!=0. On return, the
+** sequence z:return (inclusive:exclusive) is validated UTF-8.
+** Limit: nAccept>=0 => char count, nAccept<0 => character
+ */
+SQLITE_INTERNAL_LINKAGE const char*
+zSkipValidUtf8(const char *z, int nAccept, long ccm);
+
+#endif

ext/fts5/extract_api_docs.tcl

@@ -223,10 +223,12 @@ proc main {data} {
     Fts5ExtensionApi {
       set struct [get_fts5_struct $data "^struct Fts5ExtensionApi" "^.;"]
       set map [list]
+      set lKey [list]
       foreach {k v} [get_struct_members $data] {
         if {[string match x* $k]==0} continue
-        lappend map $k "<a href=#$k>$k</a>"
+        lappend lKey $k
       }
+      foreach k [lsort -decr $lKey] { lappend map $k "<a href=#$k>$k</a>" }
       output [string map $map $struct]
     }
 

ext/fts5/fts5.h

@@ -261,9 +261,34 @@ struct Fts5PhraseIter {
 **
 ** xPhraseNextColumn()
 **   See xPhraseFirstColumn above.
+**
+** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)
+**   This is used to access token iToken of phrase iPhrase of the current
+**   query. Before returning, output parameter *ppToken is set to point
+**   to a buffer containing the requested token, and *pnToken to the
+**   size of this buffer in bytes.
+**
+**   The output text is not a copy of the query text that specified the
+**   token. It is the output of the tokenizer module. For tokendata=1
+**   tables, this includes any embedded 0x00 and trailing data.
+**
+** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)
+**   This is used to access token iToken of phrase hit iIdx within the
+**   current row. Output variable (*ppToken) is set to point to a buffer
+**   containing the matching document token, and (*pnToken) to the size
+**   of that buffer in bytes. This API is not available if the specified 
+**   token matches a prefix query term. In that case both output variables 
+**   are always set to 0.
+**
+**   The output text is not a copy of the document text that was tokenized.
+**   It is the output of the tokenizer module. For tokendata=1 tables, this 
+**   includes any embedded 0x00 and trailing data.
+**
+**   This API can be quite slow if used with an FTS5 table created with the
+**   "detail=none" or "detail=column" option.
 */
 struct Fts5ExtensionApi {
-  int iVersion;                   /* Currently always set to 2 */
+  int iVersion;                   /* Currently always set to 3 */
 
   void *(*xUserData)(Fts5Context*);
 
@@ -298,6 +323,13 @@ struct Fts5ExtensionApi {
 
   int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
   void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
+
+  /* Below this point are iVersion>=3 only */
+  int (*xQueryToken)(Fts5Context*, 
+      int iPhrase, int iToken, 
+      const char **ppToken, int *pnToken
+  );
+  int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
 };
 
 /* 

ext/fts5/fts5Int.h

@@ -196,6 +196,7 @@ struct Fts5Config {
   char *zContent;                 /* content table */ 
   char *zContentRowid;            /* "content_rowid=" option value */ 
   int bColumnsize;                /* "columnsize=" option value (dflt==1) */
+  int bTokendata;                 /* "tokendata=" option value (dflt==0) */
   int eDetail;                    /* FTS5_DETAIL_XXX value */
   char *zContentExprlist;
   Fts5Tokenizer *pTok;
@@ -384,17 +385,19 @@ struct Fts5IndexIter {
 /*
 ** Values used as part of the flags argument passed to IndexQuery().
 */
-#define FTS5INDEX_QUERY_PREFIX     0x0001   /* Prefix query */
-#define FTS5INDEX_QUERY_DESC       0x0002   /* Docs in descending rowid order */
-#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004   /* Do not use prefix index */
-#define FTS5INDEX_QUERY_SCAN       0x0008   /* Scan query (fts5vocab) */
+#define FTS5INDEX_QUERY_PREFIX      0x0001  /* Prefix query */
+#define FTS5INDEX_QUERY_DESC        0x0002  /* Docs in descending rowid order */
+#define FTS5INDEX_QUERY_TEST_NOIDX  0x0004  /* Do not use prefix index */
+#define FTS5INDEX_QUERY_SCAN        0x0008  /* Scan query (fts5vocab) */
 
 /* The following are used internally by the fts5_index.c module. They are
 ** defined here only to make it easier to avoid clashes with the flags
 ** above. */
-#define FTS5INDEX_QUERY_SKIPEMPTY  0x0010
-#define FTS5INDEX_QUERY_NOOUTPUT   0x0020
-#define FTS5INDEX_QUERY_SKIPHASH   0x0040
+#define FTS5INDEX_QUERY_SKIPEMPTY   0x0010
+#define FTS5INDEX_QUERY_NOOUTPUT    0x0020
+#define FTS5INDEX_QUERY_SKIPHASH    0x0040
+#define FTS5INDEX_QUERY_NOTOKENDATA 0x0080
+#define FTS5INDEX_QUERY_SCANONETERM 0x0100
 
 /*
 ** Create/destroy an Fts5Index object.
@@ -463,6 +466,10 @@ void *sqlite3Fts5StructureRef(Fts5Index*);
 void sqlite3Fts5StructureRelease(void*);
 int sqlite3Fts5StructureTest(Fts5Index*, void*);
 
+/*
+** Used by xInstToken():
+*/
+int sqlite3Fts5IterToken(Fts5IndexIter*, i64, int, int, const char**, int*);
 
 /*
 ** Insert or remove data to or from the index. Each time a document is 
@@ -540,6 +547,13 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin);
 int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid);
 
+void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*);
+
+/* Used to populate hash tables for xInstToken in detail=none/column mode. */
+int sqlite3Fts5IndexIterWriteTokendata(
+    Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff
+);
+
 /*
 ** End of interface to code in fts5_index.c.
 **************************************************************************/
@@ -645,6 +659,7 @@ void sqlite3Fts5HashScanNext(Fts5Hash*);
 int sqlite3Fts5HashScanEof(Fts5Hash*);
 void sqlite3Fts5HashScanEntry(Fts5Hash *,
   const char **pzTerm,            /* OUT: term (nul-terminated) */
+  int *pnTerm,                    /* OUT: Size of term in bytes */
   const u8 **ppDoclist,           /* OUT: pointer to doclist */
   int *pnDoclist                  /* OUT: size of doclist in bytes */
 );
@@ -771,6 +786,10 @@ int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
 
 int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
 
+int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*);
+int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*);
+void sqlite3Fts5ExprClearTokens(Fts5Expr*);
+
 /*******************************************
 ** The fts5_expr.c API above this point is used by the other hand-written
 ** C code in this module. The interfaces below this point are called by

ext/fts5/fts5_config.c

@@ -398,6 +398,16 @@ static int fts5ConfigParseSpecial(
     return rc;
   }
 
+  if( sqlite3_strnicmp("tokendata", zCmd, nCmd)==0 ){
+    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
+      *pzErr = sqlite3_mprintf("malformed tokendata=... directive");
+      rc = SQLITE_ERROR;
+    }else{
+      pConfig->bTokendata = (zArg[0]=='1');
+    }
+    return rc;
+  }
+
   *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
   return SQLITE_ERROR;
 }

ext/fts5/fts5_expr.c

@@ -100,7 +100,9 @@ struct Fts5ExprNode {
 struct Fts5ExprTerm {
   u8 bPrefix;                     /* True for a prefix term */
   u8 bFirst;                      /* True if token must be first in column */
-  char *zTerm;                    /* nul-terminated term */
+  char *pTerm;                    /* Term data */
+  int nQueryTerm;                 /* Effective size of term in bytes */
+  int nFullTerm;                  /* Size of term in bytes incl. tokendata */
   Fts5IndexIter *pIter;           /* Iterator for this term */
   Fts5ExprTerm *pSynonym;         /* Pointer to first in list of synonyms */
 };
@@ -967,7 +969,7 @@ static int fts5ExprNearInitAll(
             p->pIter = 0;
           }
           rc = sqlite3Fts5IndexQuery(
-              pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
+              pExpr->pIndex, p->pTerm, p->nQueryTerm,
               (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
               (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
               pNear->pColset,
@@ -1604,7 +1606,7 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
       Fts5ExprTerm *pSyn;
       Fts5ExprTerm *pNext;
       Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
-      sqlite3_free(pTerm->zTerm);
+      sqlite3_free(pTerm->pTerm);
       sqlite3Fts5IterClose(pTerm->pIter);
       for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){
         pNext = pSyn->pSynonym;
@@ -1702,6 +1704,7 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
 typedef struct TokenCtx TokenCtx;
 struct TokenCtx {
   Fts5ExprPhrase *pPhrase;
+  Fts5Config *pConfig;
   int rc;
 };
 
@@ -1735,8 +1738,10 @@ static int fts5ParseTokenize(
       rc = SQLITE_NOMEM;
     }else{
       memset(pSyn, 0, (size_t)nByte);
-      pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
-      memcpy(pSyn->zTerm, pToken, nToken);
+      pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
+      pSyn->nFullTerm = pSyn->nQueryTerm = nToken;
+      if( pCtx->pConfig->bTokendata ) pSyn->nQueryTerm = strlen(pSyn->pTerm);
+      memcpy(pSyn->pTerm, pToken, nToken);
       pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
       pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn;
     }
@@ -1761,7 +1766,11 @@ static int fts5ParseTokenize(
     if( rc==SQLITE_OK ){
       pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
       memset(pTerm, 0, sizeof(Fts5ExprTerm));
-      pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
+      pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
+      pTerm->nFullTerm = pTerm->nQueryTerm = nToken;
+      if( pCtx->pConfig->bTokendata && rc==SQLITE_OK ){ 
+        pTerm->nQueryTerm = strlen(pTerm->pTerm);
+      }
     }
   }
 
@@ -1828,6 +1837,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
 
   memset(&sCtx, 0, sizeof(TokenCtx));
   sCtx.pPhrase = pAppend;
+  sCtx.pConfig = pConfig;
 
   rc = fts5ParseStringFromToken(pToken, &z);
   if( rc==SQLITE_OK ){
@@ -1877,8 +1887,7 @@ int sqlite3Fts5ExprClonePhrase(
   int rc = SQLITE_OK;             /* Return code */
   Fts5ExprPhrase *pOrig;          /* The phrase extracted from pExpr */
   Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
-  TokenCtx sCtx = {0,0};          /* Context object for fts5ParseTokenize */
-
+  TokenCtx sCtx = {0,0,0};        /* Context object for fts5ParseTokenize */
   pOrig = pExpr->apExprPhrase[iPhrase];
   pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
   if( rc==SQLITE_OK ){
@@ -1909,13 +1918,12 @@ int sqlite3Fts5ExprClonePhrase(
 
   if( pOrig->nTerm ){
     int i;                          /* Used to iterate through phrase terms */
+    sCtx.pConfig = pExpr->pConfig;
     for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
       int tflags = 0;
       Fts5ExprTerm *p;
       for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
-        const char *zTerm = p->zTerm;
-        rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
-            0, 0);
+        rc = fts5ParseTokenize((void*)&sCtx, tflags, p->pTerm,p->nFullTerm,0,0);
         tflags = FTS5_TOKEN_COLOCATED;
       }
       if( rc==SQLITE_OK ){
@@ -2296,11 +2304,13 @@ static Fts5ExprNode *fts5ParsePhraseToAnd(
         if( parseGrowPhraseArray(pParse) ){
           fts5ExprPhraseFree(pPhrase);
         }else{
+          Fts5ExprTerm *p = &pNear->apPhrase[0]->aTerm[ii];
+          Fts5ExprTerm *pTo = &pPhrase->aTerm[0];
           pParse->apPhrase[pParse->nPhrase++] = pPhrase;
           pPhrase->nTerm = 1;
-          pPhrase->aTerm[0].zTerm = sqlite3Fts5Strndup(
-              &pParse->rc, pNear->apPhrase[0]->aTerm[ii].zTerm, -1
-          );
+          pTo->pTerm = sqlite3Fts5Strndup(&pParse->rc, p->pTerm, p->nFullTerm);
+          pTo->nQueryTerm = p->nQueryTerm;
+          pTo->nFullTerm = p->nFullTerm;
           pRet->apChild[ii] = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 
               0, 0, sqlite3Fts5ParseNearset(pParse, 0, pPhrase)
           );
@@ -2485,16 +2495,17 @@ static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
 
   /* Determine the maximum amount of space required. */
   for(p=pTerm; p; p=p->pSynonym){
-    nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2;
+    nByte += pTerm->nQueryTerm * 2 + 3 + 2;
   }
   zQuoted = sqlite3_malloc64(nByte);
 
   if( zQuoted ){
     int i = 0;
     for(p=pTerm; p; p=p->pSynonym){
-      char *zIn = p->zTerm;
+      char *zIn = p->pTerm;
+      char *zEnd = &zIn[p->nQueryTerm];
       zQuoted[i++] = '"';
-      while( *zIn ){
+      while( zIn<zEnd ){
         if( *zIn=='"' ) zQuoted[i++] = '"';
         zQuoted[i++] = *zIn++;
       }
@@ -2572,8 +2583,10 @@ static char *fts5ExprPrintTcl(
 
       zRet = fts5PrintfAppend(zRet, " {");
       for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){
-        char *zTerm = pPhrase->aTerm[iTerm].zTerm;
-        zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm);
+        Fts5ExprTerm *p = &pPhrase->aTerm[iTerm];
+        zRet = fts5PrintfAppend(zRet, "%s%.*s", iTerm==0?"":" ", 
+            p->nQueryTerm, p->pTerm
+        );
         if( pPhrase->aTerm[iTerm].bPrefix ){
           zRet = fts5PrintfAppend(zRet, "*");
         }
@@ -2974,6 +2987,17 @@ static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){
   return 0;
 }
 
+/*
+** pToken is a buffer nToken bytes in size that may or may not contain
+** an embedded 0x00 byte. If it does, return the number of bytes in
+** the buffer before the 0x00. If it does not, return nToken.
+*/
+static int fts5QueryTerm(const char *pToken, int nToken){
+  int ii;
+  for(ii=0; ii<nToken && pToken[ii]; ii++){}
+  return ii;
+}
+
 static int fts5ExprPopulatePoslistsCb(
   void *pCtx,                /* Copy of 2nd argument to xTokenize() */
   int tflags,                /* Mask of FTS5_TOKEN_* flags */
@@ -2985,22 +3009,33 @@ static int fts5ExprPopulatePoslistsCb(
   Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
   Fts5Expr *pExpr = p->pExpr;
   int i;
+  int nQuery = nToken;
+  i64 iRowid = pExpr->pRoot->iRowid;
 
   UNUSED_PARAM2(iUnused1, iUnused2);
 
-  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
+  if( nQuery>FTS5_MAX_TOKEN_SIZE ) nQuery = FTS5_MAX_TOKEN_SIZE;
+  if( pExpr->pConfig->bTokendata ){
+    nQuery = fts5QueryTerm(pToken, nQuery);
+  }
   if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
   for(i=0; i<pExpr->nPhrase; i++){
-    Fts5ExprTerm *pTerm;
+    Fts5ExprTerm *pT;
     if( p->aPopulator[i].bOk==0 ) continue;
-    for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
-      int nTerm = (int)strlen(pTerm->zTerm);
-      if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
-       && memcmp(pTerm->zTerm, pToken, nTerm)==0
+    for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){
+      if( (pT->nQueryTerm==nQuery || (pT->nQueryTerm<nQuery && pT->bPrefix))
+       && memcmp(pT->pTerm, pToken, pT->nQueryTerm)==0
       ){
         int rc = sqlite3Fts5PoslistWriterAppend(
             &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
         );
+        if( rc==SQLITE_OK && pExpr->pConfig->bTokendata && !pT->bPrefix ){
+          int iCol = p->iOff>>32;
+          int iTokOff = p->iOff & 0x7FFFFFFF;
+          rc = sqlite3Fts5IndexIterWriteTokendata(
+              pT->pIter, pToken, nToken, iRowid, iCol, iTokOff
+          );
+        }
         if( rc ) return rc;
         break;
       }
@@ -3135,3 +3170,81 @@ int sqlite3Fts5ExprPhraseCollist(
 
   return rc;
 }
+
+/*
+** Does the work of the fts5_api.xQueryToken() API method.
+*/
+int sqlite3Fts5ExprQueryToken(
+  Fts5Expr *pExpr, 
+  int iPhrase, 
+  int iToken, 
+  const char **ppOut, 
+  int *pnOut
+){
+  Fts5ExprPhrase *pPhrase = 0;
+
+  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
+    return SQLITE_RANGE;
+  }
+  pPhrase = pExpr->apExprPhrase[iPhrase];
+  if( iToken<0 || iToken>=pPhrase->nTerm ){
+    return SQLITE_RANGE;
+  }
+
+  *ppOut = pPhrase->aTerm[iToken].pTerm;
+  *pnOut = pPhrase->aTerm[iToken].nFullTerm;
+  return SQLITE_OK;
+}
+
+/*
+** Does the work of the fts5_api.xInstToken() API method.
+*/
+int sqlite3Fts5ExprInstToken(
+  Fts5Expr *pExpr, 
+  i64 iRowid,
+  int iPhrase, 
+  int iCol, 
+  int iOff, 
+  int iToken, 
+  const char **ppOut, 
+  int *pnOut
+){
+  Fts5ExprPhrase *pPhrase = 0;
+  Fts5ExprTerm *pTerm = 0;
+  int rc = SQLITE_OK;
+
+  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
+    return SQLITE_RANGE;
+  }
+  pPhrase = pExpr->apExprPhrase[iPhrase];
+  if( iToken<0 || iToken>=pPhrase->nTerm ){
+    return SQLITE_RANGE;
+  }
+  pTerm = &pPhrase->aTerm[iToken];
+  if( pTerm->bPrefix==0 ){
+    if( pExpr->pConfig->bTokendata ){
+      rc = sqlite3Fts5IterToken(
+          pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut
+      );
+    }else{
+      *ppOut = pTerm->pTerm;
+      *pnOut = pTerm->nFullTerm;
+    }
+  }
+  return rc;
+}
+
+/*
+** Clear the token mappings for all Fts5IndexIter objects mannaged by 
+** the expression passed as the only argument.
+*/
+void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){
+  int ii;
+  for(ii=0; ii<pExpr->nPhrase; ii++){
+    Fts5ExprTerm *pT;
+    for(pT=&pExpr->apExprPhrase[ii]->aTerm[0]; pT; pT=pT->pSynonym){
+      sqlite3Fts5IndexIterClearTokendata(pT->pIter);
+    }
+  }
+}
+

ext/fts5/fts5_hash.c

@@ -36,10 +36,15 @@ struct Fts5Hash {
 
 /*
 ** Each entry in the hash table is represented by an object of the 
-** following type. Each object, its key (a nul-terminated string) and 
-** its current data are stored in a single memory allocation. The 
-** key immediately follows the object in memory. The position list
-** data immediately follows the key data in memory.
+** following type. Each object, its key, and its current data are stored 
+** in a single memory allocation. The key immediately follows the object 
+** in memory. The position list data immediately follows the key data 
+** in memory.
+**
+** The key is Fts5HashEntry.nKey bytes in size. It consists of a single
+** byte identifying the index (either the main term index or a prefix-index),
+** followed by the term data. For example: "0token". There is no 
+** nul-terminator - in this case nKey=6.
 **
 ** The data that follows the key is in a similar, but not identical format
 ** to the doclist data stored in the database. It is:
@@ -174,8 +179,7 @@ static int fts5HashResize(Fts5Hash *pHash){
       unsigned int iHash;
       Fts5HashEntry *p = apOld[i];
       apOld[i] = p->pHashNext;
-      iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p),
-                          (int)strlen(fts5EntryKey(p)));
+      iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), p->nKey);
       p->pHashNext = apNew[iHash];
       apNew[iHash] = p;
     }
@@ -259,7 +263,7 @@ int sqlite3Fts5HashWrite(
   for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
     char *zKey = fts5EntryKey(p);
     if( zKey[0]==bByte 
-     && p->nKey==nToken
+     && p->nKey==nToken+1
      && memcmp(&zKey[1], pToken, nToken)==0 
     ){
       break;
@@ -289,9 +293,9 @@ int sqlite3Fts5HashWrite(
     zKey[0] = bByte;
     memcpy(&zKey[1], pToken, nToken);
     assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) );
-    p->nKey = nToken;
+    p->nKey = nToken+1;
     zKey[nToken+1] = '\0';
-    p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry);
+    p->nData = nToken+1 + sizeof(Fts5HashEntry);
     p->pHashNext = pHash->aSlot[iHash];
     pHash->aSlot[iHash] = p;
     pHash->nEntry++;
@@ -408,12 +412,17 @@ static Fts5HashEntry *fts5HashEntryMerge(
       *ppOut = p1;
       p1 = 0;
     }else{
-      int i = 0;
       char *zKey1 = fts5EntryKey(p1);
       char *zKey2 = fts5EntryKey(p2);
-      while( zKey1[i]==zKey2[i] ) i++;
+      int nMin = MIN(p1->nKey, p2->nKey);
 
-      if( ((u8)zKey1[i])>((u8)zKey2[i]) ){
+      int cmp = memcmp(zKey1, zKey2, nMin);
+      if( cmp==0 ){
+        cmp = p1->nKey - p2->nKey;
+      }
+      assert( cmp!=0 );
+
+      if( cmp>0 ){
         /* p2 is smaller */
         *ppOut = p2;
         ppOut = &p2->pScanNext;
@@ -455,7 +464,7 @@ static int fts5HashEntrySort(
     Fts5HashEntry *pIter;
     for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
       if( pTerm==0 
-       || (pIter->nKey+1>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm))
+       || (pIter->nKey>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm))
       ){
         Fts5HashEntry *pEntry = pIter;
         pEntry->pScanNext = 0;
@@ -494,12 +503,11 @@ int sqlite3Fts5HashQuery(
 
   for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
     zKey = fts5EntryKey(p);
-    assert( p->nKey+1==(int)strlen(zKey) );
-    if( nTerm==p->nKey+1 && memcmp(zKey, pTerm, nTerm)==0 ) break;
+    if( nTerm==p->nKey && memcmp(zKey, pTerm, nTerm)==0 ) break;
   }
 
   if( p ){
-    int nHashPre = sizeof(Fts5HashEntry) + nTerm + 1;
+    int nHashPre = sizeof(Fts5HashEntry) + nTerm;
     int nList = p->nData - nHashPre;
     u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10));
     if( pRet ){
@@ -560,19 +568,22 @@ int sqlite3Fts5HashScanEof(Fts5Hash *p){
 void sqlite3Fts5HashScanEntry(
   Fts5Hash *pHash,
   const char **pzTerm,            /* OUT: term (nul-terminated) */
+  int *pnTerm,                    /* OUT: Size of term in bytes */
   const u8 **ppDoclist,           /* OUT: pointer to doclist */
   int *pnDoclist                  /* OUT: size of doclist in bytes */
 ){
   Fts5HashEntry *p;
   if( (p = pHash->pScan) ){
     char *zKey = fts5EntryKey(p);
-    int nTerm = (int)strlen(zKey);
+    int nTerm = p->nKey;
     fts5HashAddPoslistSize(pHash, p, 0);
     *pzTerm = zKey;
-    *ppDoclist = (const u8*)&zKey[nTerm+1];
-    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
+    *pnTerm = nTerm;
+    *ppDoclist = (const u8*)&zKey[nTerm];
+    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm);
   }else{
     *pzTerm = 0;
+    *pnTerm = 0;
     *ppDoclist = 0;
     *pnDoclist = 0;
   }

ext/fts5/fts5_main.c

@@ -118,7 +118,7 @@ struct Fts5FullTable {
   Fts5Global *pGlobal;            /* Global (connection wide) data */
   Fts5Cursor *pSortCsr;           /* Sort data from this cursor */
   int iSavepoint;                 /* Successful xSavepoint()+1 */
-  int bInSavepoint;
+  
 #ifdef SQLITE_DEBUG
   struct Fts5TransactionState ts;
 #endif
@@ -656,12 +656,15 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
   }
   idxStr[iIdxStr] = '\0';
 
-  /* Set idxFlags flags for the ORDER BY clause */
+  /* Set idxFlags flags for the ORDER BY clause
+  **
+  ** Note that tokendata=1 tables cannot currently handle "ORDER BY rowid DESC".
+  */
   if( pInfo->nOrderBy==1 ){
     int iSort = pInfo->aOrderBy[0].iColumn;
     if( iSort==(pConfig->nCol+1) && bSeenMatch ){
       idxFlags |= FTS5_BI_ORDER_RANK;
-    }else if( iSort==-1 ){
+    }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){
       idxFlags |= FTS5_BI_ORDER_ROWID;
     }
     if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){
@@ -913,6 +916,16 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
   );
   assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) );
 
+  /* If this cursor uses FTS5_PLAN_MATCH and this is a tokendata=1 table,
+  ** clear any token mappings accumulated at the fts5_index.c level. In
+  ** other cases, specifically FTS5_PLAN_SOURCE and FTS5_PLAN_SORTED_MATCH,
+  ** we need to retain the mappings for the entire query.  */
+  if( pCsr->ePlan==FTS5_PLAN_MATCH 
+   && ((Fts5Table*)pCursor->pVtab)->pConfig->bTokendata 
+  ){
+    sqlite3Fts5ExprClearTokens(pCsr->pExpr);
+  }
+
   if( pCsr->ePlan<3 ){
     int bSkip = 0;
     if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
@@ -2063,12 +2076,6 @@ static int fts5ApiInst(
   ){
     if( iIdx<0 || iIdx>=pCsr->nInstCount ){
       rc = SQLITE_RANGE;
-#if 0
-    }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){
-      *piPhrase = pCsr->aInst[iIdx*3];
-      *piCol = pCsr->aInst[iIdx*3 + 2];
-      *piOff = -1;
-#endif
     }else{
       *piPhrase = pCsr->aInst[iIdx*3];
       *piCol = pCsr->aInst[iIdx*3 + 1];
@@ -2323,13 +2330,56 @@ static int fts5ApiPhraseFirstColumn(
   return rc;
 }
 
+/*
+** xQueryToken() API implemenetation.
+*/
+static int fts5ApiQueryToken(
+  Fts5Context* pCtx, 
+  int iPhrase, 
+  int iToken, 
+  const char **ppOut, 
+  int *pnOut
+){
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
+  return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut);
+}
+
+/*
+** xInstToken() API implemenetation.
+*/
+static int fts5ApiInstToken(
+  Fts5Context *pCtx,
+  int iIdx, 
+  int iToken,
+  const char **ppOut, int *pnOut
+){
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
+  int rc = SQLITE_OK;
+  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 
+   || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) 
+  ){
+    if( iIdx<0 || iIdx>=pCsr->nInstCount ){
+      rc = SQLITE_RANGE;
+    }else{
+      int iPhrase = pCsr->aInst[iIdx*3];
+      int iCol = pCsr->aInst[iIdx*3 + 1];
+      int iOff = pCsr->aInst[iIdx*3 + 2];
+      i64 iRowid = fts5CursorRowid(pCsr);
+      rc = sqlite3Fts5ExprInstToken(
+          pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut
+      );
+    }
+  }
+  return rc;
+}
+
 
 static int fts5ApiQueryPhrase(Fts5Context*, int, void*, 
     int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
 );
 
 static const Fts5ExtensionApi sFts5Api = {
-  2,                            /* iVersion */
+  3,                            /* iVersion */
   fts5ApiUserData,
   fts5ApiColumnCount,
   fts5ApiRowCount,
@@ -2349,6 +2399,8 @@ static const Fts5ExtensionApi sFts5Api = {
   fts5ApiPhraseNext,
   fts5ApiPhraseFirstColumn,
   fts5ApiPhraseNextColumn,
+  fts5ApiQueryToken,
+  fts5ApiInstToken
 };
 
 /*
@@ -2615,9 +2667,7 @@ static int fts5RenameMethod(
 ){
   int rc;
   Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
-  pTab->bInSavepoint = 1;
   rc = sqlite3Fts5StorageRename(pTab->pStorage, zName);
-  pTab->bInSavepoint = 0;
   return rc;
 }
 
@@ -2634,26 +2684,12 @@ int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
 static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
   Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
   int rc = SQLITE_OK;
-  char *zSql = 0;
+
   fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
-
-  if( pTab->bInSavepoint==0 ){
-    zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')",
-        pTab->p.pConfig->zDb, pTab->p.pConfig->zName, pTab->p.pConfig->zName
-    );
-    if( zSql ){
-      pTab->bInSavepoint = 1;
-      rc = sqlite3_exec(pTab->p.pConfig->db, zSql, 0, 0, 0);
-      pTab->bInSavepoint = 0;
-      sqlite3_free(zSql);
-    }else{
-      rc = SQLITE_NOMEM;
-    }
-    if( rc==SQLITE_OK ){
-      pTab->iSavepoint = iSavepoint+1;
-    }
+  rc = sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
+  if( rc==SQLITE_OK ){
+    pTab->iSavepoint = iSavepoint+1;
   }
-
   return rc;
 }
 
@@ -2914,7 +2950,7 @@ static int fts5ShadowName(const char *zName){
 ** if anything is found amiss.  Return a NULL pointer if everything is
 ** OK.
 */
-static int fts5Integrity(
+static int fts5IntegrityMethod(
   sqlite3_vtab *pVtab,    /* the FTS5 virtual table to check */
   const char *zSchema,    /* Name of schema in which this table lives */
   const char *zTabname,   /* Name of the table itself */
@@ -2972,7 +3008,7 @@ static int fts5Init(sqlite3 *db){
     /* xRelease      */ fts5ReleaseMethod,
     /* xRollbackTo   */ fts5RollbackToMethod,
     /* xShadowName   */ fts5ShadowName,
-    /* xIntegrity    */ fts5Integrity
+    /* xIntegrity    */ fts5IntegrityMethod
   };
 
   int rc;

ext/fts5/fts5_tcl.c

@@ -244,6 +244,9 @@ static int SQLITE_TCLAPI xF5tApi(
     { "xGetAuxdataInt",    1, "CLEAR" },              /* 15 */
     { "xPhraseForeach",    4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */
     { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */
+
+    { "xQueryToken",       2, "IPHRASE ITERM" },      /* 18 */
+    { "xInstToken",        2, "IDX ITERM" },          /* 19 */
     { 0, 0, 0}
   };
 
@@ -500,6 +503,38 @@ static int SQLITE_TCLAPI xF5tApi(
       break;
     }
 
+    CASE(18, "xQueryToken") {
+      const char *pTerm = 0;
+      int nTerm = 0;
+      int iPhrase = 0;
+      int iTerm = 0;
+
+      if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
+      if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
+      rc = p->pApi->xQueryToken(p->pFts, iPhrase, iTerm, &pTerm, &nTerm);
+      if( rc==SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
+      }
+
+      break;
+    }
+
+    CASE(19, "xInstToken") {
+      const char *pTerm = 0;
+      int nTerm = 0;
+      int iIdx = 0;
+      int iTerm = 0;
+
+      if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ) return TCL_ERROR;
+      if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
+      rc = p->pApi->xInstToken(p->pFts, iIdx, iTerm, &pTerm, &nTerm);
+      if( rc==SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
+      }
+
+      break;
+    }
+
     default: 
       assert( 0 );
       break;
@@ -1117,6 +1152,176 @@ static int SQLITE_TCLAPI f5tRegisterTok(
   return TCL_OK;
 }
 
+typedef struct OriginTextCtx OriginTextCtx;
+struct OriginTextCtx {
+  sqlite3 *db;
+  fts5_api *pApi;
+};
+
+typedef struct OriginTextTokenizer OriginTextTokenizer;
+struct OriginTextTokenizer {
+  Fts5Tokenizer *pTok;            /* Underlying tokenizer object */
+  fts5_tokenizer tokapi;          /* API implementation for pTok */
+};
+
+/*
+** Delete the OriginTextCtx object indicated by the only argument.
+*/
+static void f5tOrigintextTokenizerDelete(void *pCtx){
+  OriginTextCtx *p = (OriginTextCtx*)pCtx;
+  ckfree(p);
+}
+
+static int f5tOrigintextCreate(
+  void *pCtx, 
+  const char **azArg, 
+  int nArg, 
+  Fts5Tokenizer **ppOut
+){
+  OriginTextCtx *p = (OriginTextCtx*)pCtx;
+  OriginTextTokenizer *pTok = 0;
+  void *pTokCtx = 0;
+  int rc = SQLITE_OK;
+
+  pTok = (OriginTextTokenizer*)sqlite3_malloc(sizeof(OriginTextTokenizer));
+  if( pTok==0 ){
+    rc = SQLITE_NOMEM;
+  }else if( nArg<1 ){
+    rc = SQLITE_ERROR;
+  }else{
+    /* Locate the underlying tokenizer */
+    rc = p->pApi->xFindTokenizer(p->pApi, azArg[0], &pTokCtx, &pTok->tokapi);
+  }
+
+  /* Create the new tokenizer instance */
+  if( rc==SQLITE_OK ){
+    rc = pTok->tokapi.xCreate(pTokCtx, &azArg[1], nArg-1, &pTok->pTok);
+  }
+
+  if( rc!=SQLITE_OK ){
+    sqlite3_free(pTok);
+    pTok = 0;
+  }
+  *ppOut = (Fts5Tokenizer*)pTok;
+  return rc;
+}
+
+static void f5tOrigintextDelete(Fts5Tokenizer *pTokenizer){
+  OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
+  if( p->pTok ){
+    p->tokapi.xDelete(p->pTok);
+  }
+  sqlite3_free(p);
+}
+
+typedef struct OriginTextCb OriginTextCb;
+struct OriginTextCb {
+  void *pCtx;
+  const char *pText; 
+  int nText;
+  int (*xToken)(void *, int, const char *, int, int, int);
+
+  char *aBuf;                     /* Buffer to use */
+  int nBuf;                       /* Allocated size of aBuf[] */
+};
+
+static int xOriginToken(
+  void *pCtx,                     /* Copy of 2nd argument to xTokenize() */
+  int tflags,                     /* Mask of FTS5_TOKEN_* flags */
+  const char *pToken,             /* Pointer to buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStart,                     /* Byte offset of token within input text */
+  int iEnd                        /* Byte offset of end of token within input */
+){
+  OriginTextCb *p = (OriginTextCb*)pCtx;
+  int ret = 0;
+
+  if( nToken==(iEnd-iStart) && 0==memcmp(pToken, &p->pText[iStart], nToken) ){
+    /* Token exactly matches document text. Pass it through as is. */
+    ret = p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd);
+  }else{
+    int nReq = nToken + 1 + (iEnd-iStart);
+    if( nReq>p->nBuf ){
+      sqlite3_free(p->aBuf);
+      p->aBuf = sqlite3_malloc(nReq*2);
+      if( p->aBuf==0 ) return SQLITE_NOMEM;
+      p->nBuf = nReq*2;
+    }
+
+    memcpy(p->aBuf, pToken, nToken);
+    p->aBuf[nToken] = '\0';
+    memcpy(&p->aBuf[nToken+1], &p->pText[iStart], iEnd-iStart);
+    ret = p->xToken(p->pCtx, tflags, p->aBuf, nReq, iStart, iEnd);
+  }
+
+  return ret;
+}
+
+
+static int f5tOrigintextTokenize(
+  Fts5Tokenizer *pTokenizer, 
+  void *pCtx,
+  int flags,                      /* Mask of FTS5_TOKENIZE_* flags */
+  const char *pText, int nText, 
+  int (*xToken)(void *, int, const char *, int, int, int)
+){
+  OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
+  OriginTextCb cb;
+  int ret;
+
+  memset(&cb, 0, sizeof(cb));
+  cb.pCtx = pCtx;
+  cb.pText = pText;
+  cb.nText = nText;
+  cb.xToken = xToken;
+
+  ret = p->tokapi.xTokenize(p->pTok,(void*)&cb,flags,pText,nText,xOriginToken);
+  sqlite3_free(cb.aBuf);
+  return ret;
+}
+
+/*
+**      sqlite3_fts5_register_origintext DB
+**
+** Description...
+*/
+static int SQLITE_TCLAPI f5tRegisterOriginText(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3 *db = 0;
+  fts5_api *pApi = 0;
+  int rc;
+  fts5_tokenizer tok = {0, 0, 0};
+  OriginTextCtx *pCtx = 0;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }
+  if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR;
+
+  pCtx = (OriginTextCtx*)ckalloc(sizeof(OriginTextCtx));
+  pCtx->db = db;
+  pCtx->pApi = pApi;
+
+  tok.xCreate = f5tOrigintextCreate;
+  tok.xDelete = f5tOrigintextDelete;
+  tok.xTokenize = f5tOrigintextTokenize;
+  rc = pApi->xCreateTokenizer(
+      pApi, "origintext", (void*)pCtx, &tok, f5tOrigintextTokenizerDelete
+  );
+
+  Tcl_ResetResult(interp);
+  if( rc!=SQLITE_OK ){
+    Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
+    return TCL_ERROR;
+  }
+  return TCL_OK;
+}
+
 /*
 ** Entry point.
 */
@@ -1133,7 +1338,8 @@ int Fts5tcl_Init(Tcl_Interp *interp){
     { "sqlite3_fts5_may_be_corrupt",     f5tMayBeCorrupt, 0 },
     { "sqlite3_fts5_token_hash",         f5tTokenHash, 0 },
     { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 },
-    { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 }
+    { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 },
+    { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 }
   };
   int i;
   F5tTokenizerContext *pContext;

ext/fts5/fts5_vocab.c

@@ -629,7 +629,7 @@ static int fts5VocabFilterMethod(
   if( pEq ){
     zTerm = (const char *)sqlite3_value_text(pEq);
     nTerm = sqlite3_value_bytes(pEq);
-    f = 0;
+    f = FTS5INDEX_QUERY_NOTOKENDATA;
   }else{
     if( pGe ){
       zTerm = (const char *)sqlite3_value_text(pGe);

ext/fts5/test/fts5_common.tcl

@@ -438,6 +438,20 @@ proc detail_is_none {} { detail_check ; expr {$::detail == "none"} }
 proc detail_is_col {}  { detail_check ; expr {$::detail == "col" } }
 proc detail_is_full {} { detail_check ; expr {$::detail == "full"} }
 
+proc foreach_tokenizer_mode {prefix script} {
+  set saved $::testprefix
+  foreach {d mapping} {
+    ""              {}
+    "-origintext"   {, tokenize="origintext unicode61", tokendata=1}
+  } {
+    set s [string map [list %TOKENIZER% $mapping] $script]
+    set ::testprefix "$prefix$d"
+    reset_db
+    sqlite3_fts5_register_origintext db
+    uplevel $s
+  }
+  set ::testprefix $saved
+}
 
 #-------------------------------------------------------------------------
 # Convert a poslist of the type returned by fts5_test_poslist() to a 

ext/fts5/test/fts5aa.test

@@ -22,6 +22,7 @@ ifcapable !fts5 {
 }
 
 foreach_detail_mode $::testprefix {
+foreach_tokenizer_mode $::testprefix {
 
 do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
@@ -44,7 +45,7 @@ do_execsql_test 1.1 {
 #
 
 do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
 }
 do_execsql_test 2.1 {
   INSERT INTO t1 VALUES('a b c', 'd e f');
@@ -73,8 +74,9 @@ do_execsql_test 2.4 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 3.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
 }
 foreach {i x y} {
    1  {g f d b f} {h h e i a}
@@ -97,8 +99,9 @@ foreach {i x y} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 4.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
   INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }
 foreach {i x y} {
@@ -121,8 +124,9 @@ foreach {i x y} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 5.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
   INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }
 foreach {i x y} {
@@ -145,8 +149,9 @@ foreach {i x y} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 6.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
   INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }
 
@@ -181,6 +186,7 @@ do_execsql_test 6.6 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 expr srand(0)
 do_execsql_test 7.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(x,y,z);
@@ -222,6 +228,7 @@ for {set i 1} {$i <= 10} {incr i} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 8.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3");
   INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
@@ -236,6 +243,7 @@ do_execsql_test 8.1 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 
 expr srand(0)
 
@@ -280,8 +288,9 @@ for {set i 1} {$i <= 10} {incr i} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 10.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
 }
 set d10 {
    1  {g f d b f} {h h e i a}
@@ -314,19 +323,19 @@ do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
 #-------------------------------------------------------------------------
 #
 do_catchsql_test 11.1 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL% %TOKENIZER%);
 } {1 {reserved fts5 column name: rank}}
 do_catchsql_test 11.2 {
-  CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL% %TOKENIZER%);
 } {1 {reserved fts5 table name: rank}}
 do_catchsql_test 11.3 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL% %TOKENIZER%);
 } {1 {reserved fts5 column name: rowid}}
 
 #-------------------------------------------------------------------------
 #
 do_execsql_test 12.1 {
-  CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
 } {}
 
 do_catchsql_test 12.2 {
@@ -341,8 +350,9 @@ do_test 12.3 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 13.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
   INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
 } {}
 
@@ -365,8 +375,9 @@ do_execsql_test 13.6 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 14.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
   INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
   WITH d(x,y) AS (
     SELECT NULL, 'xyz xyz xyz xyz xyz xyz'
@@ -449,8 +460,9 @@ do_catchsql_test 16.2 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 17.1 {
-  CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
   INSERT INTO b2 VALUES('a');
   INSERT INTO b2 VALUES('b');
   INSERT INTO b2 VALUES('c');
@@ -466,8 +478,9 @@ do_test 17.2 {
 
 if {[string match n* %DETAIL%]==0} {
   reset_db
+  sqlite3_fts5_register_origintext db
   do_execsql_test 17.3 {
-    CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%);
+    CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
     INSERT INTO c2 VALUES('x x x', 'x x x');
     SELECT rowid FROM c2 WHERE c2 MATCH 'y:x';
   } {1}
@@ -476,8 +489,9 @@ if {[string match n* %DETAIL%]==0} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 17.1 {
-  CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL% %TOKENIZER%);
   INSERT INTO uio VALUES(NULL);
   INSERT INTO uio SELECT NULL FROM uio;
   INSERT INTO uio SELECT NULL FROM uio;
@@ -524,8 +538,8 @@ do_execsql_test 17.9 {
 #--------------------------------------------------------------------
 #
 do_execsql_test 18.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
-  CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL% %TOKENIZER%);
   INSERT INTO t1 VALUES('abc*', NULL);
   INSERT INTO t2 VALUES(1, 'abcdefg');
 }
@@ -540,8 +554,9 @@ do_execsql_test 18.3 {
 # fts5 table in the temp schema.
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 19.0 {
-  CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
   INSERT INTO t1 VALUES('x y z');
   INSERT INTO t1 VALUES('w x 1');
   SELECT rowid FROM t1 WHERE t1 MATCH 'x';
@@ -551,8 +566,9 @@ do_execsql_test 19.0 {
 # Test that 6 and 7 byte varints can be read.
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 20.0 {
-  CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL% %TOKENIZER%);
 }
 set ::ids [list \
   0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43]
@@ -570,7 +586,7 @@ do_test 20.1 {
 #
 do_execsql_test 21.0 {
   CREATE TEMP TABLE t8(a, b);
-  CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL% %TOKENIZER%);
 }
 
 do_execsql_test 21.1 {
@@ -581,7 +597,7 @@ do_execsql_test 21.1 {
 }
 
 do_execsql_test 22.0 {
-  CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
   INSERT INTO t9(rowid, x) VALUES(2, 'bbb');
   BEGIN;
     INSERT INTO t9(rowid, x) VALUES(1, 'aaa');
@@ -596,7 +612,7 @@ do_execsql_test 22.1 {
 
 #-------------------------------------------------------------------------
 do_execsql_test 23.0 {
-  CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
   CREATE TABLE t11(x);
 }
 do_execsql_test 23.1 {
@@ -608,7 +624,7 @@ do_execsql_test 23.2 {
 
 #-------------------------------------------------------------------------
 do_execsql_test 24.0 {
-  CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
   INSERT INTO t12 VALUES('aaaa');
 }
 do_execsql_test 24.1 {
@@ -618,6 +634,9 @@ do_execsql_test 24.1 {
     INSERT INTO t12 VALUES('aaaa');
   END;
 }
+execsql_pp {
+  SELECT rowid, hex(block) FROM t12_data
+}
 do_execsql_test 24.2 {
   INSERT INTO t12(t12) VALUES('integrity-check');
 }
@@ -627,7 +646,7 @@ do_execsql_test 24.3 {
 
 #-------------------------------------------------------------------------
 do_execsql_test 25.0 {
-  CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
 }
 do_execsql_test 25.1 {
   BEGIN;
@@ -638,6 +657,7 @@ SELECT * FROM t13('BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
 }
 
 
+}
 }
 
 expand_all_sql db

ext/fts5/test/fts5corrupt5.test

@@ -880,6 +880,347 @@ do_catchsql_test 5.4 {
   UPDATE t1 SET content=randomblob(500);
 } {1 {database disk image is malformed}}
 
+#-------------------------------------------------------------------------
+reset_db
+do_test 6.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+.open --hexdb
+| size 32768 pagesize 4096 filename crash-42fa37b694d45a.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07   .....@  ........
+|     96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d   ...............m
+|    112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00   .....N..........
+|   3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74   .........1tablet
+|   3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45   2t2.CREATE TABLE
+|   3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61    t2(x)V.......ta
+|   3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63   blet1_configt1_c
+|   3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42   onfig.CREATE TAB
+|   3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b   LE 't1_config'(k
+|   3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29    PRIMARY KEY, v)
+|   3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05    WITHOUT ROWID[.
+|   3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64   ..!!...tablet1_d
+|   3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65   ocsizet1_docsize
+|   3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74   .CREATE TABLE 't
+|   3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e   1_docsize'(id IN
+|   3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45   TEGER PRIMARY KE
+|   3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21   Y, sz BLOB)U...!
+|   3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65   !.wtablet1_conte
+|   3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45   ntt1_content.CRE
+|   3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f   ATE TABLE 't1_co
+|   3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45   ntent'(id INTEGE
+|   3824: 52 20 50 52 49 4d 41 52 49 20 4b 45 59 2c 20 63   R PRIMARI KEY, c
+|   3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65   0)i.......-table
+|   3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45   t1_idxt1_idx.CRE
+|   3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64   ATE TABLE 't1_id
+|   3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20   x'(segid, term, 
+|   3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45   pgno, PRIMARY KE
+|   3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20   Y(segid, term)) 
+|   3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07   WITHOUT ROWIDU..
+|   3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61   ......tablet1_da
+|   3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45   tat1_data.CREATE
+|   3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27    TABLE 't1_data'
+|   4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d   (id INTEGER PRIM
+|   4016: 41 52 b9 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42   AR. KEY, block B
+|   4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c   LOB):......ctabl
+|   4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54   et1t1CREATE VIRT
+|   4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49   UAL TABLE t1 USI
+|   4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29   NG fts5(content)
+| page 2 offset 4096
+|      0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd f0 00   ................
+|     16: 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
+|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80   .............$..
+|   4032: 80 80 80 01 03 00 4e 00 10 00 1e 06 30 61 62 61   ......N.....0aba
+|   4048: 63 6c 01 02 02 04 02 66 74 02 5f 02 04 04 6e 64   cl.....ft._...nd
+|   4064: 6f 6e 02 02 02 04 0a 07 05 01 03 00 10 03 03 0f   on..............
+|   4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 11   ...$............
+| page 3 offset 8192
+|      0: 0a 00 00 00 01 0f 00 01 00 00 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
+| page 4 offset 12288
+|      0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00   ................
+|   4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00   .....abandon....
+|   4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b   .abaft.....aback
+| page 5 offset 16384
+|      0: 0d 00 00 00 03 0f ee 00 0f fa 0f 00 00 00 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03   ................
+|   4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01   ................
+| page 6 offset 20480
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| page 7 offset 24576
+|      0: 0d 00 00 10 03 0f d6 00 0f 00 00 00 00 00 00 00   ................
+|   4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c   ..........rebuil
+|   4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63   d...+integrity-c
+|   4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 00 00 00   heck....optim...
+| page 8 offset 28672
+|      0: 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
+| end crash-42fa37b694d45a.db
+}]} {}
+
+do_execsql_test 6.1 {
+  INSERT INTO t1(t1,rank) VALUES('secure-delete',1);
+}
+do_catchsql_test 6.2 {
+  UPDATE t1 SET content=randomblob(500) WHERE t1;
+} {1 {constraint failed}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_test 7.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+.open --hexdb
+| size 40960 pagesize 4096 filename crash-d8b4a99207c10b.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04   ................
+|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     96: 00 00 00 00 0d 00 00 00 0d 0b 62 00 0f 97 0f 40   ..........b....@
+|    112: 0e d5 0e 75 0e 18 0d c0 0d 66 0d 0f 0c a4 0c 44   ...u.....f.....D
+|    128: 0b ec 0b a7 0b 62 00 00 00 00 00 00 00 00 00 00   .....b..........
+|   2912: 00 00 43 0d 06 17 11 11 08 75 74 61 62 6c 65 74   ..C......utablet
+|   2928: 34 74 34 43 52 45 41 54 45 20 56 49 52 54 55 41   4t4CREATE VIRTUA
+|   2944: 4c 20 54 41 42 4c 45 20 74 34 20 55 53 49 4e 47   L TABLE t4 USING
+|   2960: 20 66 74 73 35 76 6f 63 61 62 28 27 74 32 27 2c    fts5vocab('t2',
+|   2976: 20 27 72 6f 77 27 29 43 0c 06 17 11 11 08 75 74    'row')C......ut
+|   2992: 61 62 6c 65 74 33 74 33 43 52 45 41 54 45 20 56   ablet3t3CREATE V
+|   3008: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 33 20   IRTUAL TABLE t3 
+|   3024: 55 53 49 4e 47 20 66 74 73 35 76 6f 63 61 62 28   USING fts5vocab(
+|   3040: 27 74 31 27 2c 20 27 72 6f 77 27 29 56 0b 06 17   't1', 'row')V...
+|   3056: 1f 1f 01 7d 74 61 62 6c 65 74 32 5f 63 6f 6e 66   ....tablet2_conf
+|   3072: 69 67 74 32 5f 63 6f 6e 66 69 67 0a 43 52 45 41   igt2_config.CREA
+|   3088: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 63 6f 6e   TE TABLE 't2_con
+|   3104: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b   fig'(k PRIMARY K
+|   3120: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52   EY, v) WITHOUT R
+|   3136: 4f 57 49 44 5e 0a 07 17 21 21 01 81 07 74 61 62   OWID^...!!...tab
+|   3152: 6c 65 74 32 5f 63 6f 6e 74 65 6e 74 74 32 5f 63   let2_contentt2_c
+|   3168: 6f 6e 74 65 6e 74 09 43 52 45 41 54 45 20 54 41   ontent.CREATE TA
+|   3184: 42 4c 45 20 27 74 32 5f 63 6f 6e 74 65 6e 74 27   BLE 't2_content'
+|   3200: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d   (id INTEGER PRIM
+|   3216: 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c   ARY KEY, c0, c1,
+|   3232: 20 63 32 29 69 09 07 17 19 19 01 81 2d 74 61 62    c2)i.......-tab
+|   3248: 6c 65 74 32 5f 69 64 78 74 32 5f 69 64 78 08 43   let2_idxt2_idx.C
+|   3264: 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 32 5f   REATE TABLE 't2_
+|   3280: 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d   idx'(segid, term
+|   3296: 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20   , pgno, PRIMARY 
+|   3312: 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29   KEY(segid, term)
+|   3328: 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55   ) WITHOUT ROWIDU
+|   3344: 08 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 32 5f   ........tablet2_
+|   3360: 64 61 74 61 74 32 5f 64 61 74 61 07 43 52 45 41   datat2_data.CREA
+|   3376: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 64 61 74   TE TABLE 't2_dat
+|   3392: 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52   a'(id INTEGER PR
+|   3408: 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b   IMARY KEY, block
+|   3424: 20 42 4c 4f 42 29 58 07 07 17 11 11 08 81 1d 74    BLOB)X........t
+|   3440: 61 62 6c 65 74 32 74 32 43 52 45 41 54 45 20 56   ablet2t2CREATE V
+|   3456: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 32 20   IRTUAL TABLE t2 
+|   3472: 55 53 49 4e 47 20 66 74 73 35 28 27 61 27 2c 5b   USING fts5('a',[
+|   3488: 62 5d 2c 22 63 22 2c 64 65 74 61 69 6c 3d 6e 6f   b],.c.,detail=no
+|   3504: 6e 65 2c 63 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29   ne,columnsize=0)
+|   3520: 56 06 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f   V.......tablet1_
+|   3536: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 06   configt1_config.
+|   3552: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
+|   3568: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41   _config'(k PRIMA
+|   3584: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f   RY KEY, v) WITHO
+|   3600: 55 54 20 52 4f 57 49 44 5b 05 07 17 21 21 01 81   UT ROWID[...!!..
+|   3616: 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65   .tablet1_docsize
+|   3632: 74 31 5f 64 6f 63 73 69 7a 65 05 43 52 45 41 54   t1_docsize.CREAT
+|   3648: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73   E TABLE 't1_docs
+|   3664: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20   ize'(id INTEGER 
+|   3680: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20   PRIMARY KEY, sz 
+|   3696: 42 4c 4f 42 29 5e 04 07 17 21 21 01 81 07 74 61   BLOB)^...!!...ta
+|   3712: 62 6c 65 74 31 5f 63 6f 6e 74 65 6e 74 74 31 5f   blet1_contentt1_
+|   3728: 63 6f 6e 74 65 6e 74 04 43 52 45 41 54 45 20 54   content.CREATE T
+|   3744: 41 42 4c 45 20 27 74 31 5f 63 6f 6e 74 65 6e 74   ABLE 't1_content
+|   3760: 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49   '(id INTEGER PRI
+|   3776: 4d 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31   MARY KEY, c0, c1
+|   3792: 2c 20 63 32 29 69 03 07 17 19 19 01 81 2d 74 61   , c2)i.......-ta
+|   3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03   blet1_idxt1_idx.
+|   3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
+|   3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72   _idx'(segid, ter
+|   3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59   m, pgno, PRIMARY
+|   3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d    KEY(segid, term
+|   3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44   )) WITHOUT ROWID
+|   3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31   U........tablet1
+|   3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45   _datat1_data.CRE
+|   3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61   ATE TABLE 't1_da
+|   3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50   ta'(id INTEGER P
+|   3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63   RIMARY KEY, bloc
+|   3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b   k BLOB)g.......;
+|   4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20   tablet1t1CREATE 
+|   4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31   VIRTUAL TABLE t1
+|   4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 62 20    USING fts5(a,b 
+|   4048: 75 6e 69 6e 64 65 78 65 64 2c 63 2c 74 6f 6b 65   unindexed,c,toke
+|   4064: 6e 69 7a 65 3d 22 70 6f 72 74 65 72 20 61 73 63   nize=.porter asc
+|   4080: 69 69 22 2c 74 6f 6b 65 6e 64 61 74 61 3d 31 29   ii.,tokendata=1)
+| page 2 offset 4096
+|      0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c   ..h............|
+|     16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00   .*..............
+|   3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00   .......0........
+|   3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80   ..........7.....
+|   3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01   ...t.....0a.....
+|   3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01   b.....c.....g...
+|   3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01   ....h.......i...
+|   3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00   ............*...
+|   3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80   ............%...
+|   3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02   .....P.....0g...
+|   3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02   .....h........i.
+|   4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 7f f1 03   ........7.......
+|   4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01   .t.....0a.....b.
+|   4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02   ....c.....g.....
+|   4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 04   ..h.......i.....
+|   4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00   ................
+|   4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
+| page 3 offset 8192
+|      0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c   ................
+|   4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02   ................
+| page 4 offset 12288
+|      0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00   ................
+|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
+|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
+|   4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69    h i.......g h i
+|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
+|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
+| page 5 offset 16384
+|      0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03   ................
+|   4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03   ................
+| page 6 offset 20480
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| page 7 offset 24576
+|      0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00   ................
+|   3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84   ..............A.
+|   4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61   ............4.0a
+|   4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01   .....b.....c....
+|   4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01   .d...e...f...g..
+|   4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06   ...h.....i......
+|   4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f   ................
+|   4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
+| page 8 offset 28672
+|      0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
+| page 9 offset 32768
+|      0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00   ................
+|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
+|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
+|   4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69    h i.......g h i
+|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
+|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
+| page 10 offset 36864
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| end crash-d8b4a99207c10b.db
+}]} {}
+
+do_catchsql_test 7.1 {
+  SELECT snippet(t1, -1, '.', '..', '[', ']'), 
+         highlight(t1, 2, '[', ']') 
+           FROM t1('g + h') 
+           WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank;
+} {1 {database disk image is malformed}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_test 8.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+.open --hexdb
+| size 20480 pagesize 4096 filename crash-d57c01958e48ab.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 05   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 04   ................
+|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     96: 00 00 00 00 0d 00 00 00 05 0e 10 00 0f 97 0f 40   ...............@
+|    112: 0e d5 0e 68 0e 10 01 00 00 00 00 00 00 00 00 00   ...h............
+|   3600: 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f   V.......tablet1_
+|   3616: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 05   configt1_config.
+|   3632: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
+|   3648: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41   _config'(k PRIMA
+|   3664: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f   RY KEY, v) WITHO
+|   3680: 55 54 20 52 4f 57 49 44 6b 04 07 17 21 21 01 81   UT ROWIDk...!!..
+|   3696: 21 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65   !tablet1_docsize
+|   3712: 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 45 41 54   t1_docsize.CREAT
+|   3728: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73   E TABLE 't1_docs
+|   3744: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20   ize'(id INTEGER 
+|   3760: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20   PRIMARY KEY, sz 
+|   3776: 42 4c 4f 42 2c 20 6f 72 69 67 69 6e 20 49 4e 54   BLOB, origin INT
+|   3792: 45 47 45 52 29 69 03 07 17 19 19 01 81 2d 74 61   EGER)i.......-ta
+|   3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03   blet1_idxt1_idx.
+|   3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
+|   3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72   _idx'(segid, ter
+|   3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59   m, pgno, PRIMARY
+|   3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d    KEY(segid, term
+|   3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44   )) WITHOUT ROWID
+|   3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31   U........tablet1
+|   3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45   _datat1_data.CRE
+|   3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61   ATE TABLE 't1_da
+|   3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50   ta'(id INTEGER P
+|   3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63   RIMARY KEY, bloc
+|   3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b   k BLOB)g.......;
+|   4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20   tablet1t1CREATE 
+|   4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31   VIRTUAL TABLE t1
+|   4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 20 62    USING fts5(a, b
+|   4048: 2c 20 63 6f 6e 74 65 6e 74 3d 27 27 2c 20 63 6f   , content='', co
+|   4064: 6e 74 65 6e 74 6c 65 73 73 5f 64 65 6c 65 74 65   ntentless_delete
+|   4080: 3d 31 2c 20 74 6f 6b 65 6e 64 61 74 61 3d 31 29   =1, tokendata=1)
+| page 2 offset 4096
+|      0: 0d 0f eb 00 03 0e 17 00 0f e2 0e 17 0e 31 00 00   .............1..
+|     16: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
+|   3600: 00 00 00 00 00 00 00 18 0a 03 00 36 00 00 00 00   ...........6....
+|   3616: ff 00 00 01 01 01 01 00 01 01 01 01 01 01 00 00   ................
+|   3632: 07 83 29 84 80 80 80 80 01 04 00 86 56 00 00 01   ..).........V...
+|   3648: 96 04 30 61 61 61 01 02 02 01 04 02 04 01 08 02   ..0aaa..........
+|   3664: 04 04 04 01 10 02 04 04 04 04 04 04 04 01 20 02   .............. .
+|   3680: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 01   ................
+|   3696: 40 02 04 04 04 04 04 04 04 04 04 04 04 04 04 04   @...............
+|   3712: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3728: 04 01 81 00 02 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3744: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3760: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3776: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3792: 04 04 04 04 02 02 62 63 01 06 01 01 02 01 03 62   ......bc.......b
+|   3808: 62 62 02 02 03 01 04 03 06 01 08 03 06 06 06 01   bb..............
+|   3824: 10 03 06 06 06 06 06 06 06 01 20 03 06 06 06 06   .......... .....
+|   3840: 06 06 06 06 06 06 06 06 06 06 06 01 40 03 06 06   ............@...
+|   3856: 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06   ................
+|   3872: 06 06 06 06 06 06 06 06 06 06 16 06 06 02 02 63   ...............c
+|   3888: 64 02 06 01 01 02 01 03 63 63 63 03 02 05 01 04   d.......ccc.....
+|   3904: 05 0a 01 08 05 0a 0a 0a 01 10 05 0a 0a 0a 0a 0a   ................
+|   3920: 0a 0a 01 20 05 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a   ... ............
+|   3936: 0a 0a 0a 0a 02 02 64 65 03 06 01 01 02 01 03 64   ......de.......d
+|   3952: 64 64 04 02 09 01 04 09 12 01 08 09 12 12 12 01   dd..............
+|   3968: 10 09 12 12 12 12 12 12 12 02 02 65 66 04 06 01   ...........ef...
+|   3984: 01 02 01 03 65 65 65 05 02 11 01 04 11 22 01 08   ....eee.........
+|   4000: 11 22 22 22 02 02 66 67 05 06 01 01 02 01 03 66   ......fg.......f
+|   4016: 56 66 06 02 21 01 04 21 42 02 02 67 68 06 06 01   Vf..!..!B..gh...
+|   4032: 01 02 cb 03 67 67 67 07 02 41 02 02 68 69 07 06   ....ggg..A..hi..
+|   4048: 01 01 02 04 81 13 09 50 09 2e 09 1c 09 12 09 0c   .......P........
+|   4064: 09 08 07 01 03 00 14 07 81 77 07 00 00 00 15 22   .........w......
+|   4080: 00 00 00 00 ff 00 00 01 00 00 00 00 00 00 05 0c   ................
+| page 3 offset 8192
+|      0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
+| page 4 offset 12288
+|      0: 0d 00 00 00 07 0f c8 00 0f f8 0f f0 0f e8 0f e0   ................
+|     16: 0f d8 0f d0 0f c8 00 00 00 00 00 00 00 00 00 00   ................
+|   4032: 00 00 00 00 00 00 00 00 06 07 04 00 10 09 7f 01   ................
+|   4048: 06 06 04 00 10 09 3f 01 06 05 04 00 10 09 1f 01   ......?.........
+|   4064: 06 04 04 00 10 09 0f 01 06 03 04 00 10 09 07 01   ................
+|   4080: 06 02 04 00 10 09 03 01 06 01 04 00 10 09 01 01   ................
+| page 5 offset 16384
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| end crash-d57c01958e48ab.db
+}]} {}
+
+do_catchsql_test 8.1 {
+  SELECT rowid FROM t1('a* NOT ý‘') ;
+} {0 {1 2 3 4 5 6 7}}
 
 sqlite3_fts5_may_be_corrupt 0
 finish_test

ext/fts5/test/fts5faultG.test

@@ -46,5 +46,31 @@ do_faultsim_test 1 -faults oom* -prep {
   faultsim_test_result {0 {}}
 }
 
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, content=, contentless_delete=1);
+  BEGIN;
+    INSERT INTO t1 VALUES('here''s some text');
+    INSERT INTO t1 VALUES('useful stuff, text');
+    INSERT INTO t1 VALUES('what would we do without text!'); 
+  COMMIT;
+}
+faultsim_save_and_close
+do_faultsim_test 2 -faults oom* -prep { 
+  faultsim_restore_and_reopen
+  execsql {
+    BEGIN;
+      DELETE FROM t1 WHERE rowid=2;
+  }
+} -body {
+  execsql {
+    INSERT INTO t1(t1) VALUES('optimize');
+  }
+} -test {
+  faultsim_integrity_check
+  faultsim_test_result {0 {}}
+}
+
+
 
 finish_test

ext/fts5/test/fts5faultH.test

@@ -0,0 +1,141 @@
+# 2010 June 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultG
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+set ::testprefix fts5faultH
+
+sqlite3_fts5_register_origintext db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+
+  BEGIN;
+    INSERT INTO t1 VALUES('oNe tWo thRee');
+    INSERT INTO t1 VALUES('One Two Three');
+    INSERT INTO t1 VALUES('onE twO threE');
+  COMMIT;
+  BEGIN;
+    INSERT INTO t1 VALUES('one two three');
+    INSERT INTO t1 VALUES('one two three');
+    INSERT INTO t1 VALUES('one two three');
+  COMMIT;
+}
+
+do_faultsim_test 1 -faults oom* -prep { 
+} -body {
+  execsql {
+    SELECT rowid FROM t1('three');
+  }
+} -test {
+  faultsim_integrity_check
+  faultsim_test_result {0 {1 2 3 4 5 6}}
+}
+
+
+reset_db
+sqlite3_fts5_register_origintext db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+
+  BEGIN;
+    INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(12, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(13, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(14, 'bbb BBB bbb');
+    INSERT INTO t1(rowid, x) VALUES(15, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(16, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(17, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(18, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(19, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(20, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(21, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(22, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(23, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(24, 'aaa bbb BBB');
+  COMMIT;
+}
+
+do_faultsim_test 2 -faults oom* -prep { 
+} -body {
+  execsql {
+    SELECT rowid FROM t1('BBB AND AAA');
+  }
+} -test {
+  faultsim_integrity_check
+  faultsim_test_result {0 {10 24}}
+}
+
+reset_db
+sqlite3_fts5_register_origintext db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+
+  INSERT INTO t1(rowid, x) VALUES(9, 'bbb Bbb BBB');
+  BEGIN;
+    INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(11, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(12, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(13, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(14, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(15, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(16, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(17, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(18, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(19, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(20, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(21, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(22, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(23, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(24, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(25, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(26, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(27, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(28, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(29, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(30, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(31, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(32, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(33, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(34, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(35, 'aaa bbb BBB');
+  COMMIT;
+}
+
+do_faultsim_test 3 -faults oom* -prep { 
+} -body {
+  execsql {
+    SELECT rowid FROM t1('BBB AND AAA');
+  }
+} -test {
+  faultsim_integrity_check
+  faultsim_test_result {0 {10 35}}
+}
+
+
+finish_test

ext/fts5/test/fts5misc.test

@@ -91,7 +91,6 @@ do_execsql_test 2.2.1 {
   INSERT INTO vt0(c0) VALUES ('xyz');
 }
 
-breakpoint
 do_execsql_test 2.2.2 {
   ALTER TABLE t0 RENAME TO t1;
 }
@@ -500,6 +499,21 @@ do_execsql_test 17.5 {
   SELECT c0 FROM t0 WHERE c0 GLOB '*faul*';
 } {assertionfaultproblem}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 18.0 {
+  BEGIN;
+    CREATE VIRTUAL TABLE t1 USING fts5(text);
+    ALTER TABLE t1 RENAME TO t2;
+}
+
+do_execsql_test 18.1 {
+    DROP TABLE t2;
+}
+
+do_execsql_test 18.2 {
+  COMMIT;
+}
 
 finish_test
 

ext/fts5/test/fts5origintext.test

@@ -0,0 +1,297 @@
+# 2014 Jan 08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", detail=%DETAIL%
+  );
+  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO ft VALUES('Hello world');
+}
+
+do_execsql_test 1.2 {
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+proc b {x} { string map [list "\0" "."] $x }
+db func b b
+
+do_execsql_test 1.3 {
+  select b(term) from vocab;
+} {
+  hello.Hello
+  world
+}
+
+do_execsql_test 1.4 {
+  SELECT rowid FROM ft('Hello');
+} {1}
+
+#-------------------------------------------------------------------------
+reset_db
+
+# Return a random integer between 0 and n-1.
+#
+proc random {n} {
+  expr {abs(int(rand()*$n))}
+}
+
+proc select_one {list} {
+  set n [llength $list]
+  lindex $list [random $n]
+}
+
+proc term {} {
+  set first_letter {
+    a b c d e f g h i j k l m n o p q r s t u v w x y z
+    A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
+  }
+
+  set term [select_one $first_letter]
+  append term [random 100]
+}
+
+proc document {} {
+  set nTerm [expr [random 5] + 5]
+  set doc ""
+  for {set ii 0} {$ii < $nTerm} {incr ii} {
+    lappend doc [term]
+  }
+  set doc
+}
+db func document document
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", detail=%DETAIL%
+  );
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 128);
+  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
+}
+
+do_test 2.1 {
+  for {set ii 0} {$ii < 500} {incr ii} {
+    execsql { INSERT INTO ft VALUES( document() ) }
+  }
+} {}
+
+do_execsql_test 2.2 {
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+do_execsql_test 2.3 {
+  INSERT INTO ft(ft, rank) VALUES('merge', 16);
+}
+
+do_execsql_test 2.4 {
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+do_execsql_test 2.5 {
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", detail=%DETAIL%
+  );
+  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
+
+  INSERT INTO ft(rowid, x) VALUES(1, 'hello');
+  INSERT INTO ft(rowid, x) VALUES(2, 'Hello');
+  INSERT INTO ft(rowid, x) VALUES(3, 'HELLO');
+}
+
+#proc b {x} { string map [list "\0" "."] $x }
+#db func b b
+#execsql_pp { SELECT b(term) FROM vocab }
+
+do_execsql_test 3.1.1 { SELECT rowid FROM ft('hello') } 1
+do_execsql_test 3.1.2 { SELECT rowid FROM ft('Hello') } 2
+do_execsql_test 3.1.3 { SELECT rowid FROM ft('HELLO') } 3
+
+do_execsql_test 3.2 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(x, 
+      tokenize="origintext unicode61", 
+      tokendata=1,
+      detail=%DETAIL%
+  );
+  CREATE VIRTUAL TABLE vocab2 USING fts5vocab(ft2, instance);
+
+  INSERT INTO ft2(rowid, x) VALUES(1, 'hello');
+  INSERT INTO ft2(rowid, x) VALUES(2, 'Hello');
+  INSERT INTO ft2(rowid, x) VALUES(3, 'HELLO');
+
+  INSERT INTO ft2(rowid, x) VALUES(10, 'helloooo');
+}
+
+#proc b {x} { string map [list "\0" "."] $x }
+#db func b b
+#execsql_pp { SELECT b(term) FROM vocab }
+
+do_execsql_test 3.3.1 { SELECT rowid FROM ft2('hello') } {1 2 3}
+do_execsql_test 3.3.2 { SELECT rowid FROM ft2('Hello') } {1 2 3}
+do_execsql_test 3.3.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3}
+
+do_execsql_test 3.3.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+sqlite3_fts5_register_origintext db
+proc querytoken {cmd iPhrase iToken} { 
+  set txt [$cmd xQueryToken $iPhrase $iToken]
+  string map [list "\0" "."] $txt
+}
+sqlite3_fts5_create_function db querytoken querytoken
+
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
+  );
+  INSERT INTO ft VALUES('one two three four');
+}
+
+do_execsql_test 4.1 {
+  SELECT rowid, querytoken(ft, 0, 0) FROM ft('TwO')
+} {1 two.TwO}
+do_execsql_test 4.2 {
+  SELECT rowid, querytoken(ft, 0, 0) FROM ft('one TWO ThreE')
+} {1 one}
+do_execsql_test 4.3 {
+  SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE')
+} {1 two.TWO}
+
+if {"%DETAIL%"=="full"} {
+  # Phrase queries are only supported for detail=full.
+  #
+  do_execsql_test 4.4 {
+    SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"')
+  } {1 three.ThreE}
+  do_catchsql_test 4.5 {
+    SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"')
+  } {1 SQLITE_RANGE}
+  do_catchsql_test 4.6 {
+    SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"')
+  } {1 SQLITE_RANGE}
+  do_catchsql_test 4.7 {
+    SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"')
+  } {1 SQLITE_RANGE}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+sqlite3_fts5_register_origintext db
+proc insttoken {cmd iIdx iToken} { 
+  set txt [$cmd xInstToken $iIdx $iToken]
+  string map [list "\0" "."] $txt
+}
+sqlite3_fts5_create_function db insttoken insttoken
+fts5_aux_test_functions db
+
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
+  );
+  INSERT INTO ft VALUES('one ONE One oNe oNE one');
+}
+
+do_execsql_test 5.1 {
+  SELECT insttoken(ft, 0, 0), 
+         insttoken(ft, 1, 0),
+         insttoken(ft, 2, 0),
+         insttoken(ft, 3, 0),
+         insttoken(ft, 4, 0),
+         insttoken(ft, 5, 0)
+  FROM ft('one');
+} {
+  one one.ONE one.One one.oNe one.oNE one
+}
+
+do_execsql_test 5.2 {
+  SELECT insttoken(ft, 1, 0) FROM ft('one');
+} {
+  one.ONE
+}
+
+do_execsql_test 5.3 {
+  SELECT fts5_test_poslist(ft) FROM ft('one');
+} {
+  {0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5}
+}
+
+#-------------------------------------------------------------------------
+# Test the xInstToken() API with:
+#
+#   * a non tokendata=1 table.
+#   * prefix queries.
+#
+reset_db
+sqlite3_fts5_register_origintext db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, y, tokenize='origintext unicode61', detail=%DETAIL%
+  );
+
+  INSERT INTO ft VALUES('One Two', 'Three two');
+  INSERT INTO ft VALUES('three Three', 'one One');
+}
+proc tokens {cmd} { 
+  set ret [list]
+  for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+    set txt [$cmd xInstToken $iTok 0]
+    set txt [string map [list "\0" "."] $txt]
+    lappend ret $txt
+  }
+  set ret
+}
+sqlite3_fts5_create_function db tokens tokens
+
+do_execsql_test 6.1 {
+  SELECT rowid, tokens(ft) FROM ft('One');
+} {1 one.One 2 one.One}
+
+do_execsql_test 6.2 {
+  SELECT rowid, tokens(ft) FROM ft('on*');
+} {1 {{}} 2 {{} {}}}
+
+do_execsql_test 6.3 {
+  SELECT rowid, tokens(ft) FROM ft('Three*');
+} {1 {{}} 2 {{}}}
+
+}
+
+finish_test
+

ext/fts5/test/fts5origintext2.test

@@ -0,0 +1,146 @@
+# 2014 Jan 08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+}
+
+do_execsql_test 1.1 {
+  BEGIN;
+  INSERT INTO ft VALUES('Hello');
+  INSERT INTO ft VALUES('hello');
+  INSERT INTO ft VALUES('HELLO');
+  INSERT INTO ft VALUES('today');
+  INSERT INTO ft VALUES('today');
+  INSERT INTO ft VALUES('today');
+  INSERT INTO ft VALUES('World');
+  INSERT INTO ft VALUES('world');
+  INSERT INTO ft VALUES('WORLD');
+  COMMIT;
+}
+
+do_execsql_test 1.2 { SELECT rowid FROM ft('hello'); } {1 2 3}
+do_execsql_test 1.3 { SELECT rowid FROM ft('today'); } {4 5 6}
+do_execsql_test 1.4 { SELECT rowid FROM ft('world'); } {7 8 9}
+
+do_execsql_test 1.5 {
+  SELECT count(*) FROM ft_data
+} 3
+
+do_execsql_test 1.6 {
+  DELETE FROM ft;
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
+  BEGIN;
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+    )
+    INSERT INTO ft SELECT 'Hello Hello Hello Hello Hello Hello Hello' FROM s;
+    INSERT INTO ft VALUES ('hELLO hELLO hELLO');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('World World World World World World World');
+    INSERT INTO ft VALUES('world world world world world world world');
+    INSERT INTO ft VALUES('WORLD WORLD WORLD WORLD WORLD WORLD WORLD');
+    INSERT INTO ft VALUES('World World World World World World World');
+    INSERT INTO ft VALUES('world world world world world world world');
+    INSERT INTO ft VALUES('WORLD WORLD WORLD WORLD WORLD WORLD WORLD');
+  COMMIT;
+}
+
+do_execsql_test 1.7 {
+  SELECT count(*) FROM ft_data;
+} 23
+
+do_execsql_test 1.8 { SELECT rowid FROM ft('hello') WHERE rowid>100; } {101}
+
+do_execsql_test 1.9 {
+  DELETE FROM ft;
+  INSERT INTO ft(ft) VALUES('optimize');
+  SELECT count(*) FROM ft_data;
+} {2}
+do_execsql_test 1.10 {
+  BEGIN;
+    INSERT INTO ft VALUES('Hello');
+    INSERT INTO ft VALUES('hello');
+    INSERT INTO ft VALUES('HELLO');
+    INSERT INTO ft VALUES('today');
+    INSERT INTO ft VALUES('today');
+    INSERT INTO ft VALUES('today');
+    INSERT INTO ft VALUES('World');
+    INSERT INTO ft VALUES('world');
+    INSERT INTO ft VALUES('WORLD');
+}
+
+do_execsql_test 1.11 { SELECT rowid FROM ft('hello'); } {1 2 3}
+do_execsql_test 1.12 { SELECT rowid FROM ft('today'); } {4 5 6}
+do_execsql_test 1.13 { SELECT rowid FROM ft('world'); } {7 8 9}
+do_execsql_test 1.14 { SELECT rowid FROM ft('hello') ORDER BY rank; } {1 2 3}
+
+#------------------------------------------------------------------------
+reset_db
+sqlite3_fts5_register_origintext db
+proc tokens {cmd} { 
+  set ret [list]
+  for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+    set txt [$cmd xInstToken $iTok 0]
+    set txt [string map [list "\0" "."] $txt]
+    lappend ret $txt
+  }
+  set ret
+}
+sqlite3_fts5_create_function db tokens tokens
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(
+    v, tokenize="origintext unicode61", tokendata=1, detail=none
+  );
+
+  INSERT INTO x1 VALUES('xxx Xxx XXX yyy YYY yyy');
+  INSERT INTO x1 VALUES('xxx yyy xxx yyy yyy yyy');
+}
+
+do_execsql_test 2.1 {
+  SELECT tokens(x1) FROM x1('xxx');
+} {
+  {xxx xxx.Xxx xxx.XXX} {xxx xxx}
+}
+
+do_execsql_test 2.2 {
+  UPDATE x1_content SET c0 = 'xxx xxX xxx yyy yyy yyy' WHERE id=1;
+}
+
+do_execsql_test 2.3 {
+  SELECT tokens(x1) FROM x1('xxx');
+} {
+  {xxx {} xxx} {xxx xxx}
+}
+
+finish_test
+

ext/fts5/test/fts5origintext3.test

@@ -0,0 +1,101 @@
+# 2023 November 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext3
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+  reset_db
+
+  sqlite3_fts5_register_origintext db
+  fts5_aux_test_functions db
+  proc insttoken {cmd iIdx iToken} { 
+    set txt [$cmd xInstToken $iIdx $iToken]
+    string map [list "\0" "."] $txt
+  }
+  sqlite3_fts5_create_function db insttoken insttoken
+  
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE ft USING fts5(
+        x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
+    );
+  }
+  
+  do_execsql_test 1.1 {
+    INSERT INTO ft VALUES('Hello world HELLO WORLD hello');
+  }
+  
+  do_execsql_test 1.2 {
+    SELECT fts5_test_poslist(ft) FROM ft('hello');
+  } {{0.0.0 0.0.2 0.0.4}}
+
+  do_execsql_test 1.3 {
+    SELECT 
+      insttoken(ft, 0, 0),
+      insttoken(ft, 1, 0),
+      insttoken(ft, 2, 0)
+    FROM ft('hello');
+  } {hello.Hello hello.HELLO hello}
+
+  do_execsql_test 1.4 {
+    SELECT 
+      insttoken(ft, 0, 0),
+      insttoken(ft, 1, 0),
+      insttoken(ft, 2, 0)
+    FROM ft('hello') ORDER BY rank;
+  } {hello.Hello hello.HELLO hello}
+
+  do_execsql_test 1.5 {
+    CREATE VIRTUAL TABLE ft2 USING fts5(
+        x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
+    );
+    INSERT INTO ft2(rowid, x) VALUES(1, 'ONE one two three ONE');
+    INSERT INTO ft2(rowid, x) VALUES(2, 'TWO one two three TWO');
+    INSERT INTO ft2(rowid, x) VALUES(3, 'THREE one two three THREE');
+  }
+
+  do_execsql_test 1.6 {
+    SELECT insttoken(ft2, 0, 0), rowid FROM ft2('three') ORDER BY rank;
+  } {three.THREE 3 three 1 three 2}
+
+  do_execsql_test 1.7 {
+    INSERT INTO ft2(rowid, x) VALUES(10, 'aaa bbb BBB');
+    INSERT INTO ft2(rowid, x) VALUES(12, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(13, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(14, 'bbb BBB bbb');
+    INSERT INTO ft2(rowid, x) VALUES(15, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(16, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(17, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(18, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(19, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(20, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(21, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(22, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(23, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(24, 'aaa bbb BBB');
+  }
+
+  do_execsql_test 1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24}
+  do_execsql_test 1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24}
+
+}
+
+finish_test
+

ext/fts5/test/fts5origintext4.test

@@ -0,0 +1,66 @@
+# 2023 November 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext4
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+}
+
+do_execsql_test 1.1 {
+  BEGIN;
+    INSERT INTO ft SELECT 'the first thing';
+
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<90000
+    )
+    INSERT INTO ft SELECT 'The second thing' FROM s;
+
+    INSERT INTO ft SELECT 'the first thing';
+  COMMIT;
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+
+foreach {tn sql expr} {
+  1 { SELECT rowid FROM ft('the') }       {$mem > 250000}
+  2 { SELECT rowid FROM ft('first') }     {$mem <  50000}
+  3 { SELECT rowid FROM ft('the first') } {$mem <  50000}
+} {
+  db close
+  sqlite3 db test.db
+  sqlite3_fts5_register_origintext db
+
+  execsql $sql
+  do_test 1.2.$tn {
+    set mem [lindex [sqlite3_db_status db CACHE_USED 0] 1]
+    expr $expr
+  } 1
+}
+
+proc b {x} { string map [list "\0" "."] $x }
+db func b b
+# execsql_pp { SELECT segid, b(term), pgno from ft_idx }
+
+finish_test
+

ext/fts5/test/fts5origintext5.test

@@ -0,0 +1,273 @@
+# 2023 Dec 04
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests for tables that use both tokendata=1 and contentless_delete=1.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+# Return a random integer between 0 and n-1.
+#
+proc random {n} { expr {abs(int(rand()*$n))} }
+
+# Select an element of the list passed as the only argument at random and
+# return it. 
+#
+proc select_one {list} {
+  set n [llength $list]
+  lindex $list [random $n]
+}
+
+# Given a term that consists entirely of alphabet characters, return all
+# permutations of the term using upper and lower case characters. e.g.
+#
+#    "abc" -> {CBA cBA CbA cbA CBa cBa Cba cba}
+#
+proc casify {term {lRet {{}}}} {
+  if {$term==""} { return $lRet }
+  set t [string range $term 1 end]
+  set f1 [string toupper [string range $term 0 0]]
+  set f2 [string tolower [string range $term 0 0]]
+  set ret [list]
+  foreach x $lRet {
+    lappend ret "$x$f1"
+    lappend ret "$x$f2"
+  }
+  return [casify $t $ret]
+}
+
+proc vocab {} {
+  list abc def ghi jkl mno pqr stu vwx yza
+}
+
+# Return a random 3 letter term.
+#
+proc term {} {
+  if {[info exists ::expanded_vocab]==0} {
+    foreach v [vocab] { lappend ::expanded_vocab {*}[casify $v] }
+  }
+
+  select_one $::expanded_vocab
+}
+
+# Return a document - between 3 and 10 terms.
+#
+proc document {} {
+  set nTerm [expr [random 3] + 7]
+  set doc ""
+  for {set ii 0} {$ii < $nTerm} {incr ii} {
+    lappend doc [term]
+  }
+  set doc
+}
+db func document document
+
+#-------------------------------------------------------------------------
+
+expr srand(6)
+
+set NDOC  200
+set NLOOP 50
+
+sqlite3_fts5_register_origintext db
+
+proc tokens {cmd} { 
+  set ret [list]
+  for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+    set txt [$cmd xInstToken $iTok 0]
+    set txt [string map [list "\0" "."] $txt]
+    lappend ret $txt
+  }
+  set ret
+}
+sqlite3_fts5_create_function db tokens tokens
+
+proc rankfunc {cmd} { 
+  $cmd xRowid
+}
+sqlite3_fts5_create_function db rankfunc rankfunc
+
+proc ctrl_tokens {term args} {
+  set ret [list]
+  set term [string tolower $term]
+  foreach doc $args {
+    foreach a $doc {
+      if {[string tolower $a]==$term} {
+        if {$a==$term} {
+          lappend ret $a
+        } else {
+          lappend ret [string tolower $a].$a
+        }
+      }
+    }
+  }
+  set ret
+}
+db func ctrl_tokens ctrl_tokens
+
+proc do_all_vocab_test {tn} {
+  foreach ::v [concat [vocab] nnn] {
+    set answer [execsql {
+      SELECT id, ctrl_tokens($::v, x) FROM ctrl WHERE x LIKE '%' || $::v || '%'
+    }]
+    do_execsql_test $tn.$::v.1 {
+      SELECT rowid, tokens(ft) FROM ft($::v)
+    } $answer
+    do_execsql_test $tn.$::v.2 {
+      SELECT rowid, tokens(ft) FROM ft($::v) ORDER BY rank
+    } $answer
+  }
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", content=, contentless_delete=1,
+      tokendata=1
+  );
+
+  CREATE TABLE ctrl(id INTEGER PRIMARY KEY, x TEXT);
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
+  INSERT INTO ft(ft, rank) VALUES('rank', 'rankfunc()');
+}
+do_test 1.1 {
+  for {set ii 0} {$ii < $NDOC} {incr ii} {
+    set doc [document]
+    execsql {
+      INSERT INTO ft(rowid, x) VALUES($ii, $doc);
+      INSERT INTO ctrl(id, x) VALUES($ii, $doc);
+    }
+  }
+} {}
+
+#execsql_pp { SELECT * FROM ctrl }
+#execsql_pp { SELECT * FROM ft }
+#fts5_aux_test_functions db
+#execsql_pp { SELECT rowid, tokens(ft), fts5_test_poslist(ft) FROM ft('ghi'); }
+
+do_all_vocab_test 1.2
+
+for {set ii 0} {$ii < $NLOOP} {incr ii} {
+  set lRowid [execsql { SELECT id FROM ctrl WHERE random() % 2 }]
+  foreach r $lRowid {
+    execsql { DELETE FROM ft WHERE rowid = $r }
+    execsql { DELETE FROM ctrl WHERE rowid = $r }
+
+    set doc [document]
+    execsql { INSERT INTO ft(rowid, x) VALUES($r, $doc) }
+    execsql { INSERT INTO ctrl(id, x) VALUES($r, $doc) }
+  }
+  do_all_vocab_test 1.3.$ii
+}
+
+#-------------------------------------------------------------------------
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(
+      x, y, tokenize="origintext unicode61", content=, contentless_delete=1,
+      tokendata=1
+  );
+
+  CREATE TABLE ctrl2(id INTEGER PRIMARY KEY, x TEXT, y TEXT);
+  INSERT INTO ft2(ft2, rank) VALUES('pgsz', 64);
+  INSERT INTO ft2(ft2, rank) VALUES('rank', 'rankfunc()');
+}
+do_test 2.1 {
+  for {set ii 0} {$ii < $NDOC} {incr ii} {
+    set doc1 [document]
+    set doc2 [document]
+    execsql {
+      INSERT INTO ft2(rowid, x, y) VALUES($ii, $doc, $doc2);
+      INSERT INTO ctrl2(id, x, y) VALUES($ii, $doc, $doc2);
+    }
+  }
+} {}
+
+proc do_all_vocab_test2 {tn} {
+  foreach ::v [vocab] {
+    set answer [execsql {
+      SELECT id, ctrl_tokens($::v, x, y) FROM ctrl2
+      WHERE x LIKE '%' || $::v || '%' OR y LIKE '%' || $::v || '%';
+    }]
+    do_execsql_test $tn.$::v.1 {
+      SELECT rowid, tokens(ft2) FROM ft2($::v)
+    } $answer
+    do_execsql_test $tn.$::v.2 {
+      SELECT rowid, tokens(ft2) FROM ft2($::v) ORDER BY rank
+    } $answer
+  }
+}
+
+do_all_vocab_test2 2.2
+
+for {set ii 0} {$ii < $NLOOP} {incr ii} {
+  set lRowid [execsql { SELECT id FROM ctrl2 WHERE random() % 2 }]
+  foreach r $lRowid {
+    execsql { DELETE FROM ft2 WHERE rowid = $r }
+    execsql { DELETE FROM ctrl2 WHERE rowid = $r }
+
+    set doc1 [document]
+    set doc2 [document]
+    execsql { INSERT INTO ft2(rowid, x, y) VALUES($r, $doc, $doc1) }
+    execsql { INSERT INTO ctrl2(id, x, y) VALUES($r, $doc, $doc2) }
+  }
+  do_all_vocab_test 2.3.$ii
+}
+
+#-------------------------------------------------------------------------
+
+unset -nocomplain ::expanded_vocab
+proc vocab {} {
+  list abcde fghij klmno
+}
+
+proc do_all_vocab_test3 {tn} {
+  foreach ::v [concat [vocab] nnn] {
+    set answer [execsql {
+      SELECT rowid, ctrl_tokens($::v, w) FROM ctrl3 WHERE w LIKE '%' || $::v || '%'
+    }]
+    do_execsql_test $tn.$::v.1 {
+      SELECT rowid, tokens(ft3) FROM ft3($::v)
+    } $answer
+    do_execsql_test $tn.$::v.2 {
+      SELECT rowid, tokens(ft3) FROM ft3($::v) ORDER BY rank
+    } $answer
+  }
+}
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft3 USING fts5(
+      w, tokenize="origintext unicode61", content=, contentless_delete=1,
+      tokendata=1
+  );
+  INSERT INTO ft3(ft3, rank) VALUES('rank', 'rankfunc()');
+  CREATE TABLE ctrl3(w);
+}
+
+do_execsql_test 3.1 {
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<2
+  )
+  INSERT INTO ctrl3 SELECT document() FROM s;
+  INSERT INTO ft3(rowid, w) SELECT rowid, w FROM ctrl3;
+}
+
+do_all_vocab_test3 3.2
+
+
+finish_test
+

ext/fts5/test/fts5secure8.test

@@ -0,0 +1,51 @@
+# 2023 Nov 23
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5secure8
+
+proc sql_repeat {txt n} {
+  string repeat $txt $n
+}
+db func repeat sql_repeat
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x);
+
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
+
+  INSERT INTO ft(rowid, x) VALUES(100, 'hello world');
+  INSERT INTO ft(rowid, x) VALUES(200, 'one day');
+
+  BEGIN;
+    INSERT INTO ft(rowid, x) VALUES(45, 'one two three');
+    UPDATE ft SET x = repeat('hello world ', 500) WHERE rowid=100;
+  COMMIT
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO ft(ft, rank) VALUES('secure-delete', 1);
+  DELETE FROM ft WHERE rowid=100;
+}
+
+do_execsql_test 1.2 {
+  PRAGMA integrity_check;
+} {ok}
+
+
+
+
+
+finish_test
+
+

ext/fts5/test/fts5simple2.test

@@ -343,7 +343,9 @@ do_execsql_test 17.0 {
     INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
   COMMIT;
 }
-do_execsql_test 17.1 { SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 }
+do_execsql_test 17.1 { 
+  SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 
+}
 do_execsql_test 17.2 {
   BEGIN;
     INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');

ext/fts5/test/fts5vocab2.test

@@ -280,6 +280,30 @@ do_catchsql_test 5.2 {
   INSERT INTO t1 SELECT randomblob(3000) FROM v1
 } {1 {query aborted}}
 
+#-------------------------------------------------------------------------
+reset_db
+sqlite3_fts5_may_be_corrupt 1
+
+do_execsql_test 6.0 {
+  BEGIN TRANSACTION;
+    CREATE VIRTUAL TABLE t1 USING fts5(a,b unindexed,c,tokenize="porter ascii",tokendata=1);
+    REPLACE INTO t1_data VALUES(1,X'03090009');
+    REPLACE INTO t1_data VALUES(10,X'000000000103030003010101020101030101');
+    REPLACE INTO t1_data VALUES(137438953473,X'0000002e023061010202010162010203010163010204010167010601020201016801060102030101690106010204040606060808');
+    REPLACE INTO t1_data VALUES(274877906945,X'0000001f013067020802010202010168020803010203010169020804010204040909');
+    REPLACE INTO t1_data VALUES(412316860417,X'0000002e023061030202010162030203010163030204010167030601020201016803060102030101690306010204040606060808');
+  COMMIT;
+}
+
+do_execsql_test 6.1 {
+  CREATE VIRTUAL TABLE t3 USING fts5vocab('t1', 'row');
+}
+
+do_catchsql_test 6.2 {
+  SELECT * FROM t3;
+} {1 {database disk image is malformed}}
+
+sqlite3_fts5_may_be_corrupt 0
 
 finish_test
 

ext/recover/dbdata.c

@@ -582,6 +582,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
             bNextPage = 1;
           }else{
             iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload);
+            if( nPayload>0x7fffff00 ) nPayload &= 0x3fff;
           }
     
           /* If this is a leaf intkey cell, load the rowid */

ext/wasm/GNUmakefile

@@ -854,7 +854,7 @@ dir.sql := sql
 speedtest1 := ../../speedtest1
 speedtest1.c := ../../test/speedtest1.c
 speedtest1.sql := $(dir.sql)/speedtest1.sql
-speedtest1.cliflags := --size 25 --big-transactions
+speedtest1.cliflags := --size 10 --big-transactions
 $(speedtest1):
 	$(MAKE) -C ../.. speedtest1
 $(speedtest1.sql): $(speedtest1) $(MAKEFILE)

ext/wasm/api/sqlite3-worker1-promiser.c-pp.js

@@ -247,9 +247,8 @@ globalThis.sqlite3Worker1Promiser = function callee(config = callee.defaultConfi
 globalThis.sqlite3Worker1Promiser.defaultConfig = {
   worker: function(){
 //#if target=es6-bundler-friendly
-    return new Worker("sqlite3-worker1-bundler-friendly.mjs",{
-      type: 'module' /* Noting that neither Firefox nor Safari suppor this,
-                        as of this writing. */
+    return new Worker(new URL("sqlite3-worker1-bundler-friendly.mjs", import.meta.url),{
+      type: 'module'
     });
 //#else
     let theJs = "sqlite3-worker1.js";
@@ -267,8 +266,12 @@ globalThis.sqlite3Worker1Promiser.defaultConfig = {
     }
     return new Worker(theJs + globalThis.location.search);
 //#endif
-  }.bind({
+  }
+//#ifnot target=es6-bundler-friendly
+  .bind({
     currentScript: globalThis?.document?.currentScript
-  }),
+  })
+//#endif
+  ,
   onerror: (...args)=>console.error('worker1 promiser error',...args)
 };

ext/wasm/batch-runner-sahpool.html

@@ -0,0 +1,86 @@
+<!doctype html>
+<html lang="en-us">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon">
+    <link rel="stylesheet" href="common/testing.css"/>
+    <title>sqlite3-api batch SQL runner for the SAHPool VFS</title>
+  </head>
+  <body>
+    <header id='titlebar'><span>sqlite3-api batch SQL runner for the SAHPool VFS</span></header>
+    <div>
+      <span class='input-wrapper'>
+        <input type='checkbox' class='disable-during-eval' id='cb-reverse-log-order' checked></input>
+        <label for='cb-reverse-log-order' id='lbl-reverse-log-order'>Reverse log order</label>
+      </span>
+    </div>
+    <div id='test-output' class='reverse'></div>
+    <script>
+      (function(){
+        const E = (sel)=>document.querySelector(sel);
+
+        const eOut = E('#test-output');
+        const log2 = function(cssClass,...args){
+          let ln;
+          if(1 || cssClass){
+            ln = document.createElement('div');
+            if(cssClass) ln.classList.add(cssClass);
+            ln.append(document.createTextNode(args.join(' ')));
+          }else{
+            // This doesn't work with the "reverse order" option!
+            ln = document.createTextNode(args.join(' ')+'\n');
+          }
+          eOut.append(ln);
+        };
+        const log = (...args)=>{
+          //console.log(...args);
+          log2('', ...args);
+        };
+        const logErr = function(...args){
+          console.error(...args);
+          log2('error', ...args);
+        };
+        const logWarn = function(...args){
+          console.warn(...args);
+          log2('warning', ...args);
+        };
+
+        const cbReverseLog = E('#cb-reverse-log-order');
+        const lblReverseLog = E('#lbl-reverse-log-order');
+        if(cbReverseLog.checked){
+            lblReverseLog.classList.add('warning');
+            eOut.classList.add('reverse');
+        }
+        cbReverseLog.addEventListener('change', function(){
+            if(this.checked){
+                eOut.classList.add('reverse');
+                lblReverseLog.classList.add('warning');
+            }else{
+                eOut.classList.remove('reverse');
+                lblReverseLog.classList.remove('warning');
+            }
+        }, false);
+
+        const w = new Worker('batch-runner-sahpool.js?sqlite3.dir=jswasm');
+        w.onmessage = function(msg){
+          msg = msg.data;
+          switch(msg.type){
+            case 'stdout': log(...msg.data); break;
+            case 'warn': logWarn(...msg.data); break;
+            case 'error': logErr(...msg.data); break;
+            default:
+              logErr("Unhandled worker message type:",msg);
+              break;
+          }
+        };
+      })();
+    </script>
+    <style>
+      #test-output {
+          white-space: break-spaces;
+          overflow: auto;
+      }
+    </style>
+  </body>
+</html>

ext/wasm/batch-runner-sahpool.js

@@ -0,0 +1,341 @@
+/*
+  2023-11-30
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  A basic batch SQL runner for the SAHPool VFS. This file must be run in
+  a worker thread. This is not a full-featured app, just a way to get some
+  measurements for batch execution of SQL for the OPFS SAH Pool VFS.
+*/
+'use strict';
+
+const wMsg = function(msgType,...args){
+  postMessage({
+    type: msgType,
+    data: args
+  });
+};
+const toss = function(...args){throw new Error(args.join(' '))};
+const warn = (...args)=>{ wMsg('warn',...args); };
+const error = (...args)=>{ wMsg('error',...args); };
+const log = (...args)=>{ wMsg('stdout',...args); }
+let sqlite3;
+const urlParams = new URL(globalThis.location.href).searchParams;
+const cacheSize = (()=>{
+  if(urlParams.has('cachesize')) return +urlParams.get('cachesize');
+  return 200;
+})();
+
+
+/** Throws if the given sqlite3 result code is not 0. */
+const checkSqliteRc = (dbh,rc)=>{
+  if(rc) toss("Prepare failed:",sqlite3.capi.sqlite3_errmsg(dbh));
+};
+
+const sqlToDrop = [
+  "SELECT type,name FROM sqlite_schema ",
+  "WHERE name NOT LIKE 'sqlite\\_%' escape '\\' ",
+  "AND name NOT LIKE '\\_%' escape '\\'"
+].join('');
+
+const clearDbSqlite = function(db){
+  // This would be SO much easier with the oo1 API, but we specifically want to
+  // inject metrics we can't get via that API, and we cannot reliably (OPFS)
+  // open the same DB twice to clear it using that API, so...
+  const rc = sqlite3.wasm.exports.sqlite3_wasm_db_reset(db.handle);
+  log("reset db rc =",rc,db.id, db.filename);
+};
+
+const App = {
+  db: undefined,
+  cache:Object.create(null),
+  log: log,
+  warn: warn,
+  error: error,
+  metrics: {
+    fileCount: 0,
+    runTimeMs: 0,
+    prepareTimeMs: 0,
+    stepTimeMs: 0,
+    stmtCount: 0,
+    strcpyMs: 0,
+    sqlBytes: 0
+  },
+  fileList: undefined,
+  execSql: async function(name,sql){
+    const db = this.db;
+    const banner = "========================================";
+    this.log(banner,
+             "Running",name,'('+sql.length,'bytes)');
+    const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm;
+    let pStmt = 0, pSqlBegin;
+    const metrics = db.metrics = Object.create(null);
+    metrics.prepTotal = metrics.stepTotal = 0;
+    metrics.stmtCount = 0;
+    metrics.malloc = 0;
+    metrics.strcpy = 0;
+    if(this.gotErr){
+      this.error("Cannot run SQL: error cleanup is pending.");
+      return;
+    }
+    // Run this async so that the UI can be updated for the above header...
+    const endRun = ()=>{
+      metrics.evalSqlEnd = performance.now();
+      metrics.evalTimeTotal = (metrics.evalSqlEnd - metrics.evalSqlStart);
+      this.log("metrics:",JSON.stringify(metrics, undefined, ' '));
+      this.log("prepare() count:",metrics.stmtCount);
+      this.log("Time in prepare_v2():",metrics.prepTotal,"ms",
+               "("+(metrics.prepTotal / metrics.stmtCount),"ms per prepare())");
+      this.log("Time in step():",metrics.stepTotal,"ms",
+               "("+(metrics.stepTotal / metrics.stmtCount),"ms per step())");
+      this.log("Total runtime:",metrics.evalTimeTotal,"ms");
+      this.log("Overhead (time - prep - step):",
+               (metrics.evalTimeTotal - metrics.prepTotal - metrics.stepTotal)+"ms");
+      this.log(banner,"End of",name);
+      this.metrics.prepareTimeMs += metrics.prepTotal;
+      this.metrics.stepTimeMs += metrics.stepTotal;
+      this.metrics.stmtCount += metrics.stmtCount;
+      this.metrics.strcpyMs += metrics.strcpy;
+      this.metrics.sqlBytes += sql.length;
+    };
+
+    const runner = function(resolve, reject){
+      ++this.metrics.fileCount;
+      metrics.evalSqlStart = performance.now();
+      const stack = wasm.scopedAllocPush();
+      try {
+        let t, rc;
+        let sqlByteLen = sql.byteLength;
+        const [ppStmt, pzTail] = wasm.scopedAllocPtr(2);
+        t = performance.now();
+        pSqlBegin = wasm.scopedAlloc( sqlByteLen + 1/*SQL + NUL*/) || toss("alloc(",sqlByteLen,") failed");
+        metrics.malloc = performance.now() - t;
+        metrics.byteLength = sqlByteLen;
+        let pSql = pSqlBegin;
+        const pSqlEnd = pSqlBegin + sqlByteLen;
+        t = performance.now();
+        wasm.heap8().set(sql, pSql);
+        wasm.poke(pSql + sqlByteLen, 0);
+        //log("SQL:",wasm.cstrToJs(pSql));
+        metrics.strcpy = performance.now() - t;
+        let breaker = 0;
+        while(pSql && wasm.peek8(pSql)){
+          wasm.pokePtr(ppStmt, 0);
+          wasm.pokePtr(pzTail, 0);
+          t = performance.now();
+          rc = capi.sqlite3_prepare_v2(
+            db.handle, pSql, sqlByteLen, ppStmt, pzTail
+          );
+          metrics.prepTotal += performance.now() - t;
+          checkSqliteRc(db.handle, rc);
+          pStmt = wasm.peekPtr(ppStmt);
+          pSql = wasm.peekPtr(pzTail);
+          sqlByteLen = pSqlEnd - pSql;
+          if(!pStmt) continue/*empty statement*/;
+          ++metrics.stmtCount;
+          t = performance.now();
+          rc = capi.sqlite3_step(pStmt);
+          capi.sqlite3_finalize(pStmt);
+          pStmt = 0;
+          metrics.stepTotal += performance.now() - t;
+          switch(rc){
+            case capi.SQLITE_ROW:
+            case capi.SQLITE_DONE: break;
+            default: checkSqliteRc(db.handle, rc); toss("Not reached.");
+          }
+        }
+        resolve(this);
+      }catch(e){
+        if(pStmt) capi.sqlite3_finalize(pStmt);
+        this.gotErr = e;
+        reject(e);
+      }finally{
+        capi.sqlite3_exec(db.handle,"rollback;",0,0,0);
+        wasm.scopedAllocPop(stack);
+      }
+    }.bind(this);
+    const p = new Promise(runner);
+    return p.catch(
+      (e)=>this.error("Error via execSql("+name+",...):",e.message)
+    ).finally(()=>{
+      endRun();
+    });
+  },
+
+  /**
+     Loads batch-runner.list and populates the selection list from
+     it. Returns a promise which resolves to nothing in particular
+     when it completes. Only intended to be run once at the start
+     of the app.
+  */
+  loadSqlList: async function(){
+    const infile = 'batch-runner.list';
+    this.log("Loading list of SQL files:", infile);
+    let txt;
+    try{
+      const r = await fetch(infile);
+      if(404 === r.status){
+        toss("Missing file '"+infile+"'.");
+      }
+      if(!r.ok) toss("Loading",infile,"failed:",r.statusText);
+      txt = await r.text();
+    }catch(e){
+      this.error(e.message);
+      throw e;
+    }
+    App.fileList = txt.split(/\n+/).filter(x=>!!x);
+    this.log("Loaded",infile);
+  },
+
+  /** Fetch ./fn and return its contents as a Uint8Array. */
+  fetchFile: async function(fn, cacheIt=false){
+    if(cacheIt && this.cache[fn]) return this.cache[fn];
+    this.log("Fetching",fn,"...");
+    let sql;
+    try {
+      const r = await fetch(fn);
+      if(!r.ok) toss("Fetch failed:",r.statusText);
+      sql = new Uint8Array(await r.arrayBuffer());
+    }catch(e){
+      this.error(e.message);
+      throw e;
+    }
+    this.log("Fetched",sql.length,"bytes from",fn);
+    if(cacheIt) this.cache[fn] = sql;
+    return sql;
+  }/*fetchFile()*/,
+
+  /**
+     Converts this.metrics() to a form which is suitable for easy conversion to
+     CSV. It returns an array of arrays. The first sub-array is the column names.
+     The 2nd and subsequent are the values, one per test file (only the most recent
+     metrics are kept for any given file).
+  */
+  metricsToArrays: function(){
+    const rc = [];
+    Object.keys(this.dbs).sort().forEach((k)=>{
+      const d = this.dbs[k];
+      const m = d.metrics;
+      delete m.evalSqlStart;
+      delete m.evalSqlEnd;
+      const mk = Object.keys(m).sort();
+      if(!rc.length){
+        rc.push(['db', ...mk]);
+      }
+      const row = [k.split('/').pop()/*remove dir prefix from filename*/];
+      rc.push(row);
+      row.push(...mk.map((kk)=>m[kk]));
+    });
+    return rc;
+  },
+
+  metricsToBlob: function(colSeparator='\t'){
+    const ar = [], ma = this.metricsToArrays();
+    if(!ma.length){
+      this.error("Metrics are empty. Run something.");
+      return;
+    }
+    ma.forEach(function(row){
+      ar.push(row.join(colSeparator),'\n');
+    });
+    return new Blob(ar);
+  },
+
+  /**
+     Fetch file fn and eval it as an SQL blob. This is an async
+     operation and returns a Promise which resolves to this
+     object on success.
+  */
+  evalFile: async function(fn){
+    const sql = await this.fetchFile(fn);
+    return this.execSql(fn,sql);
+  }/*evalFile()*/,
+
+  /**
+     Fetches the handle of the db associated with
+     this.e.selImpl.value, opening it if needed.
+  */
+  initDb: function(){
+    const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm;
+    const stack = wasm.scopedAllocPush();
+    let pDb = 0;
+    const d = Object.create(null);
+    d.filename = "/batch.db";
+    try{
+      const oFlags = capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE;
+      const ppDb = wasm.scopedAllocPtr();
+      const rc = capi.sqlite3_open_v2(d.filename, ppDb, oFlags, this.PoolUtil.vfsName);
+      pDb = wasm.peekPtr(ppDb)
+      if(rc) toss("sqlite3_open_v2() failed with code",rc);
+      capi.sqlite3_exec(pDb, "PRAGMA cache_size="+cacheSize, 0, 0, 0);
+      this.log("cache_size =",cacheSize);
+    }catch(e){
+      if(pDb) capi.sqlite3_close_v2(pDb);
+      throw e;
+    }finally{
+      wasm.scopedAllocPop(stack);
+    }
+    d.handle = pDb;
+    this.log("Opened db:",d.filename,'@',d.handle);
+    return d;
+  },
+
+  closeDb: function(){
+    if(this.db.handle){
+      this.sqlite3.capi.sqlite3_close_v2(this.db.handle);
+      this.db.handle = undefined;
+    }
+  },
+
+  run: async function(sqlite3){
+    delete this.run;
+    this.sqlite3 = sqlite3;
+    const capi = sqlite3.capi, wasm = sqlite3.wasm;
+    this.log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid());
+    this.log("WASM heap size =",wasm.heap8().length);
+    let timeStart;
+    sqlite3.installOpfsSAHPoolVfs({
+      clearOnInit: true, initialCapacity: 4,
+      name: 'batch-sahpool',
+      verbosity: 2
+    }).then(PoolUtil=>{
+      App.PoolUtil = PoolUtil;
+      App.db = App.initDb();
+    })
+      .then(async ()=>this.loadSqlList())
+      .then(async ()=>{
+        timeStart = performance.now();
+        for(let i = 0; i < App.fileList.length; ++i){
+          const fn = App.fileList[i];
+          await App.evalFile(fn);
+          if(App.gotErr) throw App.gotErr;
+        }
+      })
+      .then(()=>{
+        App.metrics.runTimeMs = performance.now() - timeStart;
+        App.log("total metrics:",JSON.stringify(App.metrics, undefined, ' '));
+        App.log("Reload the page to run this again.");
+        App.closeDb();
+        App.PoolUtil.removeVfs();
+      })
+      .catch(e=>this.error("ERROR:",e));
+  }/*run()*/
+}/*App*/;
+
+let sqlite3Js = 'sqlite3.js';
+if(urlParams.has('sqlite3.dir')){
+  sqlite3Js = urlParams.get('sqlite3.dir') + '/' + sqlite3Js;
+}
+importScripts(sqlite3Js);
+globalThis.sqlite3InitModule().then(async function(sqlite3_){
+  log("Done initializing. Running batch runner...");
+  sqlite3 = sqlite3_;
+  App.run(sqlite3_);
+});

ext/wasm/batch-runner.js

@@ -72,7 +72,6 @@
     App.logHtml("reset db rc =",rc,db.id, db.filename);
   };
 
-  
   const E = (s)=>document.querySelector(s);
   const App = {
     e: {
@@ -91,6 +90,15 @@
     db: Object.create(null),
     dbs: Object.create(null),
     cache:{},
+    metrics: {
+      fileCount: 0,
+      runTimeMs: 0,
+      prepareTimeMs: 0,
+      stepTimeMs: 0,
+      stmtCount: 0,
+      strcpyMs: 0,
+      sqlBytes: 0
+    },
     log: console.log.bind(console),
     warn: console.warn.bind(console),
     cls: function(){this.e.output.innerHTML = ''},
@@ -117,7 +125,6 @@
                    "Running",name,'('+sql.length,'bytes) using',db.id);
       const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm;
       let pStmt = 0, pSqlBegin;
-      const stack = wasm.scopedAllocPush();
       const metrics = db.metrics = Object.create(null);
       metrics.prepTotal = metrics.stepTotal = 0;
       metrics.stmtCount = 0;
@@ -142,6 +149,11 @@
         this.logHtml("Overhead (time - prep - step):",
                      (metrics.evalTimeTotal - metrics.prepTotal - metrics.stepTotal)+"ms");
         this.logHtml(banner,"End of",name);
+        this.metrics.prepareTimeMs += metrics.prepTotal;
+        this.metrics.stepTimeMs += metrics.stepTotal;
+        this.metrics.stmtCount += metrics.stmtCount;
+        this.metrics.strcpyMs += metrics.strcpy;
+        this.metrics.sqlBytes += sql.length;
       };
 
       let runner;
@@ -214,7 +226,9 @@
         }.bind(this);
       }else{/*sqlite3 db...*/
         runner = function(resolve, reject){
+          ++this.metrics.fileCount;
           metrics.evalSqlStart = performance.now();
+          const stack = wasm.scopedAllocPush();
           try {
             let t;
             let sqlByteLen = sql.byteLength;
@@ -269,7 +283,7 @@
       let p;
       if(1){
         p = new Promise(function(res,rej){
-          setTimeout(()=>runner(res, rej), 50)/*give UI a chance to output the "running" banner*/;
+          setTimeout(()=>runner(res, rej), 0)/*give UI a chance to output the "running" banner*/;
         });
       }else{
         p = new Promise(runner);
@@ -401,7 +415,7 @@
       });
       return new Blob(ar);
     },
-    
+
     downloadMetrics: function(){
       const b = this.metricsToBlob();
       if(!b) return;
@@ -576,6 +590,8 @@
         const timeTotal = performance.now() - timeStart;
         who.logHtml("Run-remaining time:",timeTotal,"ms ("+(timeTotal/1000/60)+" minute(s))");
         who.clearStorage();
+        App.metrics.runTimeMs = timeTotal;
+        who.logHtml("Total metrics:",JSON.stringify(App.metrics,undefined,'  '));
       }, false);
     }/*run()*/
   }/*App*/;

ext/wasm/demo-123.js

@@ -20,7 +20,7 @@
      the main (UI) thread.
   */
   let logHtml;
-  if(self.window === self /* UI thread */){
+  if(globalThis.window === globalThis /* UI thread */){
     console.log("Running demo from main UI thread.");
     logHtml = function(cssClass,...args){
       const ln = document.createElement('div');
@@ -250,7 +250,7 @@
   }/*demo1()*/;
 
   log("Loading and initializing sqlite3 module...");
-  if(self.window!==self) /*worker thread*/{
+  if(globalThis.window!==globalThis) /*worker thread*/{
     /*
       If sqlite3.js is in a directory other than this script, in order
       to get sqlite3.js to resolve sqlite3.wasm properly, we have to
@@ -262,19 +262,20 @@
       that's not needed.
 
       URL arguments passed as part of the filename via importScripts()
-      are simply lost, and such scripts see the self.location of
+      are simply lost, and such scripts see the globalThis.location of
       _this_ script.
     */
     let sqlite3Js = 'sqlite3.js';
-    const urlParams = new URL(self.location.href).searchParams;
+    const urlParams = new URL(globalThis.location.href).searchParams;
     if(urlParams.has('sqlite3.dir')){
       sqlite3Js = urlParams.get('sqlite3.dir') + '/' + sqlite3Js;
     }
     importScripts(sqlite3Js);
   }
-  self.sqlite3InitModule({
-    // We can redirect any stdout/stderr from the module
-    // like so...
+  globalThis.sqlite3InitModule({
+    /* We can redirect any stdout/stderr from the module like so, but
+       note that doing so makes use of Emscripten-isms, not
+       well-defined sqlite APIs. */
     print: log,
     printErr: error
   }).then(function(sqlite3){

ext/wasm/index-dist.html

@@ -46,6 +46,9 @@
           file:// URLs.</li>
         <li>Any OPFS-related pages or tests require:
           <ul>
+            <li>An OPFS-capable browser released after February
+              2023. Some tests will work with Chromium-based browsers
+              going back to around v102.</li>
             <li>That the web server emit the so-called
               <a href='https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cross-Origin-Opener-Policy'>COOP</a>
               and
@@ -53,12 +56,6 @@
               headers. <a href='https://sqlite.org/althttpd'>althttpd</a> requires the
               <code>-enable-sab</code> flag for that.
             </li>
-            <li>A very recent version of a Chromium-based browser
-              (v102 at least, possibly newer). OPFS support in the
-              other major browsers is pending. Development and testing
-              is currently done against a dev-channel release of
-              Chrome (v111 as of 2023-02-10).
-            </li>
           </ul>
         </li>
       </ul>

ext/wasm/index.html

@@ -31,6 +31,9 @@
           file:// URLs.</li>
         <li>Any OPFS-related pages or tests require:
           <ul>
+            <li>An OPFS-capable browser released after February
+              2023. Some tests will work with Chromium-based browsers
+              going back to around v102.</li>
             <li>That the web server emit the so-called
               <a href='https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cross-Origin-Opener-Policy'>COOP</a>
               and
@@ -38,12 +41,6 @@
               headers. <a href='https://sqlite.org/althttpd'>althttpd</a> requires the
               <code>-enable-sab</code> flag for that.
             </li>
-            <li>A very recent version of a
-              Chromium-based browser (v102 at least, possibly newer). OPFS
-              support in the other major browsers is pending. Development
-              and testing is currently done against a dev-channel release
-              of Chrome (v111 as of 2023-02-10).
-            </li>
           </ul>
         </li>
       </ul>

ext/wasm/speedtest1-worker.html

@@ -350,7 +350,7 @@
                 eControls.classList.remove('hidden');
                 break;
             case 'stdout': log(msg.data); break;
-            case 'stdout': logErr(msg.data); break;
+            case 'stderr': logErr(msg.data); break;
             case 'run-start':
                 eControls.disabled = true;
                 log("Running speedtest1 with argv =",msg.data.join(' '));

ext/wasm/speedtest1-worker.js

@@ -7,9 +7,9 @@
   }
   importScripts(speedtestJs);
   /**
-     If this environment contains OPFS, this function initializes it and
-     returns the name of the dir on which OPFS is mounted, else it returns
-     an empty string.
+     If this build includes WASMFS, this function initializes it and
+     returns the name of the dir on which OPFS is mounted, else it
+     returns an empty string.
   */
   const wasmfsDir = function f(wasmUtil){
     if(undefined !== f._) return f._;
@@ -47,6 +47,7 @@
   };
   const log = (...args)=>logMsg('stdout',args);
   const logErr = (...args)=>logMsg('stderr',args);
+  const realSahName = 'opfs-sahpool-speedtest1';
 
   const runSpeedtest = async function(cliFlagsArray){
     const scope = App.wasm.scopedAllocPush();
@@ -57,7 +58,6 @@
       ];
       App.logBuffer.length = 0;
       const ndxSahPool = argv.indexOf('opfs-sahpool');
-      const realSahName = 'opfs-sahpool-speedtest1';
       if(ndxSahPool>0){
         argv[ndxSahPool] = realSahName;
         log("Updated argv for opfs-sahpool: --vfs",realSahName);
@@ -73,7 +73,7 @@
           clearOnInit: true,
           verbosity: 2
         }).then(PoolUtil=>{
-          log("opfs-sahpool successfully installed as",realSahName);
+          log("opfs-sahpool successfully installed as",PoolUtil.vfsName);
           App.sqlite3.$SAHPoolUtil = PoolUtil;
           //console.log("sqlite3.oo1.OpfsSAHPoolDb =", App.sqlite3.oo1.OpfsSAHPoolDb);
         });
@@ -102,19 +102,6 @@
     }
   };
 
-  const sahpSanityChecks = function(sqlite3){
-    log("Attempting OpfsSAHPoolDb sanity checks...");
-    const db = new sqlite3.oo1.OpfsSAHPoolDb('opfs-sahpoool.db');
-    const fn = db.filename;
-    db.exec([
-      'create table t(a);',
-      'insert into t(a) values(1),(2),(3);'
-    ]);
-    db.close();
-    sqlite3.wasm.sqlite3_wasm_vfs_unlink(sqlite3_vfs_find("opfs-sahpool"), fn);
-    log("SAH sanity checks done.");
-  };
-
   const EmscriptenModule = {
     print: log,
     printErr: logErr,

manifest

@@ -1,12 +1,12 @@
-C The\sjsonb_remove()\sroutine\snow\sappears\sto\sbe\sworking.
-D 2023-11-15T18:47:31.537
+C Add\sNEVER\sto\stwo\sunreachable\sbranches\sin\sJSON.
+D 2023-12-12T17:13:10.122
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
-F Makefile.in 8b59912fc1538f96a08555605c5886cdcc733696ae7f22e374b2a4752196ca20
+F Makefile.in 890caf094636c308bc981032baf8f9208bf307755f9197ae4218a9fbcec2e449
 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6
-F Makefile.msc f0cf219350d9af4fba411b4f6306dce2adc897484e8f446de1fb4f40de674d00
-F README.md 963d30019abf0cc06b263cd2824bce022893f3f93a531758f6f04ff2194a16a8
+F Makefile.msc 59bb36dba001f0b38212be0794fb838f25371008b180929bcf08aa799694c168
+F README.md 6358805260a03ebead84e168bbf3740ddf3f683b477e478567186aa7afb490d3
 F VERSION 73573d4545343f001bf5dc5461173a7c78c203dd046cabcf99153878cf25d3a6
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
@@ -41,7 +41,7 @@ F doc/compile-for-windows.md 50b27d77be96195c66031a3181cb8684ed822327ea834e07f9c
 F doc/json-enhancements.md e356fc834781f1f1aa22ee300027a270b2c960122468499bf347bb123ce1ea4f
 F doc/lemon.html 44a53a1d2b42d7751f7b2f478efb23c978e258d794bfd172442307a755b9fa44
 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710
-F doc/testrunner.md 2434864be2219d4f0b6ffc99d0a2172d531c4ca4345340776f67ad4edd90dc90
+F doc/testrunner.md 8d36ec692cf4994bb66d84a4645b9afa1ce9d47dc12cbf8d437c5a5fb6ddeedb
 F doc/trusted-schema.md 33625008620e879c7bcfbbfa079587612c434fa094d338b08242288d358c3e8a
 F doc/vdbesort-memory.md 4da2639c14cd24a31e0af694b1a8dd37eaf277aff3867e9a8cc14046bc49df56
 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a
@@ -50,6 +50,8 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 6f247666b495c477628dd19364d279c78ea48cd90c72d9f9b98ad1aff3294f94
 F ext/async/sqlite3async.h 46b47c79357b97ad85d20d2795942c0020dc20c532114a49808287f04aa5309a
+F ext/consio/console_io.c 097323f60037d70523421f0248958d2280851f8ff2f9a443d4ee1474c4769118 x
+F ext/consio/console_io.h 3228dff1717481202a24f6dcf45ce0b75e7a778bf6877089518a44e1473b76a3
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4
 F ext/expert/expert1.test 0dd5cb096d66bed593e33053a3b364f6ef52ed72064bf5cf298364636dbf3cd6
@@ -86,28 +88,28 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
-F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
-F ext/fts5/fts5.h 05501612cc655504c5dce8ba765ab621d50fc478490089beaa0d75e00b23e520
-F ext/fts5/fts5Int.h 78a63cc0795186cde5384816a9403a68c65774b35d952e05b81a1b4b158e07c8
+F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e
+F ext/fts5/fts5.h 9d7867cc63631296462c90bafe57c4881b56e480fa510ffaee6a77049258c714
+F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880e9834
 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
-F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081
-F ext/fts5/fts5_expr.c bd3b81ce669c4104e34ffe66570af1999a317b142c15fccb112de9fb0caa57a6
-F ext/fts5/fts5_hash.c 076058f93327051952a752dc765df1acfe783eb11b419b30652aa1fc1f987902
-F ext/fts5/fts5_index.c 01b671fedd2189f6969385d96facc4c06d9c441f0f91d584386a62b724282f9f
-F ext/fts5/fts5_main.c a07ed863b8bd9e6fefb62db2fd40a3518eb30a5f7dcfda5be915dd2db45efa2f
+F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf
+F ext/fts5/fts5_expr.c b1ec526371b9ffde82341423a5b9753c42cbea629a41b69f26fa377d13b95a8e
+F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
+F ext/fts5/fts5_index.c ed206045ff0f2226d870fa41fba45f738c0cc953ab74ba68477091b9a574ccd3
+F ext/fts5/fts5_main.c b908696c52410e8383019ac0657c8a5cd0c8f60e78edc169e9b3c4b93f24c933
 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
-F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
+F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b
 F ext/fts5/fts5_tokenize.c 83cfcede3898001cab84432a36ce1503e3080cf9b1c682b022ec82e267ea4c13
 F ext/fts5/fts5_unicode2.c eca63dbc797f8ff0572e97caf4631389c0ab900d6364861b915bdd4735973f00
 F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80
-F ext/fts5/fts5_vocab.c aed56169ae5c1aa9b8189c779ffeef04ed516d3c712c06914e6d91a6759f4e4a
+F ext/fts5/fts5_vocab.c 209e0c151e108d5f3621fa24b91e9b02f3750ee6c3f9ccec312df39481b68a09
 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
-F ext/fts5/test/fts5_common.tcl a9de9c2209cc4e7ae3c753e783504e67206c6c1467d08f209cd0c5923d3e8d8b
-F ext/fts5/test/fts5aa.test ba5158eba7d61359becdfca895ef471072c7bf7b20e5e60dcb4d024c8419c926
+F ext/fts5/test/fts5_common.tcl 8b1848ac2baad10e444e4183034a52050b52d20b3796d9d30e78f01ab0d05583
+F ext/fts5/test/fts5aa.test 4db81519863244a3cab35795fe65ab6b592e7970c7409eba098b23ebbfc08d95
 F ext/fts5/test/fts5ab.test bd932720c748383277456b81f91bc00453de2174f9762cd05f95d0495dc50390
 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d
 F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de
@@ -143,7 +145,7 @@ F ext/fts5/test/fts5corrupt.test b6d4034b682bb3387bc44c510c71b3c67d4349e4df13949
 F ext/fts5/test/fts5corrupt2.test 99e7e23a58b4d89eb7167c6de1669cbc595cd3c79ab333e0eb56405473319e77
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
 F ext/fts5/test/fts5corrupt4.test f4c08e2182a48d8b70975fd869ee5391855c06d8a0ff87b6a2529e7c5a88a1d3
-F ext/fts5/test/fts5corrupt5.test eb6ba5ca28ef7c4c6b01e850d388cdb3dacc8c4c2f383f79d0a98128257742b4
+F ext/fts5/test/fts5corrupt5.test f9dbed6785c38d891c0b947fd6e973e9c4eb44991e1322f8d7e5281a019276f2
 F ext/fts5/test/fts5corrupt6.test bf8eeae07825b088b9665d9d8e4accbd8dc9bf3cb85b6c64cf6c9e18ccc420a4
 F ext/fts5/test/fts5corrupt7.test 80ad7f683a8bda2404731bb77e8c3dbbb620c1f6cc583cca8239f6accd6338c0
 F ext/fts5/test/fts5delete.test 619295b20dbc1d840b403ee07c878f52378849c3c02e44f2ee143b3e978a0aa7
@@ -167,7 +169,8 @@ F ext/fts5/test/fts5faultB.test d606bdb8e81aaeb6f41de3fc9fc7ae315733f0903fbff05c
 F ext/fts5/test/fts5faultD.test e7ed7895abfe6bc98a5e853826f6b74956e7ba7f594f1860bbf9e504b9647996
 F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e4710c77eb8ce7075
 F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1
-F ext/fts5/test/fts5faultG.test 340e59d2c2c1c7c379224f3968ee8d09b0f64bf56c5194217d1ded887b9d47c4
+F ext/fts5/test/fts5faultG.test d2e5a4d9a34e08dcaadcaeafef74d10cbc2abdd11aa2659a18af0294bf2812d3
+F ext/fts5/test/fts5faultH.test 57f53c87ffd59be0265840f2b54a16811f9cb9012db86aad9b41d0d14d85dfe3
 F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b996159f6909dc8079
 F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717
 F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e
@@ -180,7 +183,7 @@ F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fc
 F ext/fts5/test/fts5matchinfo.test 10c9a6f7fe61fb132299c4183c012770b10c4d5c2f2edb6df0b6607f683d737a
 F ext/fts5/test/fts5merge.test e92a8db28b45931e7a9c7b1bbd36101692759d00274df74d83fd29d25d53b3a6
 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2
-F ext/fts5/test/fts5misc.test 5ca82f2a5ee016b0842043155d1382f98a34d0d86b2791165a44d7807f6e0f54
+F ext/fts5/test/fts5misc.test dd97c86c9cbc3e587067e640f6ce88842cfbf5d23bb0e0fbb7f6707623b2d505
 F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581
 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45
 F ext/fts5/test/fts5near.test 211477940142d733ac04fad97cb24095513ab2507073a99c2765c3ddd2ef58bd
@@ -188,6 +191,11 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618
 F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1
 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785
 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca
+F ext/fts5/test/fts5origintext.test d2796fa08ee7aecfabdc0c45bb8a2fb16a00ea8757e63fbc153b718dbe430a39
+F ext/fts5/test/fts5origintext2.test f3b9436de540828d01f0672df855b09ebc0863e126d5b56234701d71dfa73634
+F ext/fts5/test/fts5origintext3.test 0d25933506600452a5ab3873cbb418ed5f2de2446c3672b9997b1ea104b0e7f0
+F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871
+F ext/fts5/test/fts5origintext5.test a037bdf7235a22033c4663837bdb12d9738245464a3ac2f60c71fc40d07ede7d
 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b
 F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a
 F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15
@@ -207,9 +215,10 @@ F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b98
 F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a
 F ext/fts5/test/fts5secure6.test 74bf04733cc523bccca519bb03d3b4e2ed6f6e3db7c59bf6be82c88a0ac857fd
 F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a
+F ext/fts5/test/fts5secure8.test eb3579e9d58b0acad97e8082dee1f99b2d393198f03500b453c2b25761c0c298
 F ext/fts5/test/fts5securefault.test dbca2b6a1c16700017f5051138991b705410889933f2a37c57ae8a23b296b10b
 F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b
-F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0
+F ext/fts5/test/fts5simple2.test 8dd2389ee75e21a1429fe87e5f8c7d9a97ad1470304a8a2d3ba4b8c3c345fecd
 F ext/fts5/test/fts5simple3.test d5c74a9d3ca71bd5dd5cacb7c55b86ea12cdddfc8b1910e3de2995206898380f
 F ext/fts5/test/fts5synonym.test 1651815b8008de170e8e600dcacc17521d765482ea8f074ae82cfa870d8bb7fb
 F ext/fts5/test/fts5synonym2.test 8f891fc49cc1e8daed727051e77e1f42849c784a6a54bef82564761b2cb3e016
@@ -229,7 +238,7 @@ F ext/fts5/test/fts5unindexed.test 9021af86a0fb9fc616f7a69a996db0116e7936d0db638
 F ext/fts5/test/fts5update.test b8affd796e45c94a4d19ad5c26606ea06065a0f162a9562d9f005b5a80ccf0bc
 F ext/fts5/test/fts5version.test d6e5a5897550afeccc2f8531d87404dc1c289ee89385dd4318dbdd75e71d7a67
 F ext/fts5/test/fts5vocab.test 7ed80d9af1ddaaa1637da05e406327b5aac250848bc604c1c1cc667908b87760
-F ext/fts5/test/fts5vocab2.test 681980e92e031c9f3fe8d9c149189e876c108da2fb0fb3a25bd8a9b94bff8f68
+F ext/fts5/test/fts5vocab2.test 1b1f0059f762ffb404213d35dac013e010621f08128589b6ec7bec59d9a710f3
 F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85
 F ext/fts5/tool/fts5txt2db.tcl c0d43c8590656f8240e622b00957b3a0facc49482411a9fdc2870b45c0c82f9f
 F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093
@@ -457,7 +466,7 @@ F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69
 F ext/rbu/sqlite3rbu.c d4ddf8f0e93772556e452a6c2814063cf47efb760a0834391a9d0cd9859fa4b9
 F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304
 F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055
-F ext/recover/dbdata.c fc7147a68422cbbbaa481ee92ae1752cc25f5a24302bece1c70dcb76345bd736
+F ext/recover/dbdata.c b7746c2ec801b453840831311be4b31f8c8f9dd97791060a69bbf12392c78949
 F ext/recover/recover1.test c484d01502239f11b61f23c1cee9f5dd19fa17617f8974e42e74d64639c524cf
 F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a
 F ext/recover/recoverbuild.test c74170e0f7b02456af41838afeb5353fdb985a48cc2331d661bbabbca7c6b8e3
@@ -565,7 +574,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb
 F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c
-F ext/wasm/GNUmakefile 0e362f3fc04eab6628cbe4f1e35f4ab4a200881f6b5f753b27fb45eabeddd9d2
+F ext/wasm/GNUmakefile 57439eec2b8b4d4074e5d861e93aab3f32bb0f44339a2b472c23f4e638b7e8a3
 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576
 F ext/wasm/README.md a8a2962c3aebdf8d2104a9102e336c5554e78fc6072746e5daf9c61514e7d193
 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff
@@ -593,10 +602,12 @@ F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f
 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 595953994aa3ae2287c889c4da39ab3d6f17b6461ecf4bec334b7a3faafddb02
 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 46c4afa6c50d7369252c104f274ad977a97e91ccfafc38b400fe36e90bdda88e
 F ext/wasm/api/sqlite3-wasm.c d0e09eb5ed3743c00294e30019e591c3aa150572ae7ffe8a8994568a7377589f
-F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f
+F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 569d4e859968e65f55dec5fac0b879828a23c8b179162cc7812fec19f844dd21
 F ext/wasm/api/sqlite3-worker1.c-pp.js a541112aa51e16705f13a99bb943c64efe178aa28c86704a955f8fd9afe4ba37
+F ext/wasm/batch-runner-sahpool.html e9a38fdeb36a13eac7b50241dfe7ae066fe3f51f5c0b0151e7baee5fce0d07a7
+F ext/wasm/batch-runner-sahpool.js 54a3ac228e6c4703fe72fb65c897e19156263a51fe9b7e21d2834a45e876aabd
 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8
-F ext/wasm/batch-runner.js 0dad6a02ad796f1003d3b7048947d275c4d6277f63767b8e685c27df8fdac93e
+F ext/wasm/batch-runner.js 05ec254f5dbfe605146d9640b3db17d6ef8c3fbef6aa8396051ca72bb5884e3f
 F ext/wasm/c-pp.c 6d80d8569d85713effe8b0818a3cf51dc779e3f0bf8dc88771b8998552ee25b4
 F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51
 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15
@@ -604,7 +615,7 @@ F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a7
 F ext/wasm/common/whwasmutil.js 4c64594eecc7af4ae64259e95a71ba2a7edf118881aaff0bba86d0c7164e78e4
 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed
 F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508
-F ext/wasm/demo-123.js 38aa8faec4d0ace1c973bc8a7a1533584463ebeecd4c420daa7d9687beeb9cb5
+F ext/wasm/demo-123.js c7b3cca50c55841c381a9ca4f9396e5bbdc6114273d0b10a43e378e32e7be5bf
 F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e
 F ext/wasm/demo-jsstorage.js 44e3ae7ec2483b6c511384c3c290beb6f305c721186bcf5398ca4e00004a06b8
 F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98ab22f5786620b3354ed15f
@@ -618,8 +629,8 @@ F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d695
 F ext/wasm/fiddle/fiddle-worker.js e0153f9af6500805c6f09c0b3cfdb7d857e9d6863dbee9d50d1628fccf5f4b4d
 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715
 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2
-F ext/wasm/index-dist.html 22379774f0ad4edcaaa8cf9c674c82e794cc557719a8addabed74eb8069d412e
-F ext/wasm/index.html 4e7847b909f4ae0da8c829b150b79454050e53b3658431f138636257729cd42b
+F ext/wasm/index-dist.html e91d76e4581185238fd3d42ed86ec600f7023ed3e3a944c5c356f25304bf1263
+F ext/wasm/index.html b31ce41c0da476d5ffcef23069b9d3415b419d65af5779096ebcfbcbade453a9
 F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54
 F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb
 F ext/wasm/module-symbols.html dc476b403369b26a1a23773e13b80f41b9a49f0825e81435fe3600a7cfbbe337
@@ -627,8 +638,8 @@ F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d
 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63
 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d
 F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe
-F ext/wasm/speedtest1-worker.html e33e2064bda572c0c3ebaec7306c35aa758d9d27e245d67e807f8cc4a9351cc5
-F ext/wasm/speedtest1-worker.js 315d26198c46be7c85e26fda15d80ef882424276abde25ffd8b026fb02a35d8c
+F ext/wasm/speedtest1-worker.html 864b65ed78ce24847a348c180e7f267621a02ca027068a1863ec1c90187c1852
+F ext/wasm/speedtest1-worker.js 4d2ea70a3c24e05bdca78025202841f33d298c4fa9541a0070c3228661f89ecd
 F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da
 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
 F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
@@ -666,8 +677,8 @@ F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645
 F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522
 F src/btree.c f3b09c5414de3a11db73e11e1d66f4c5e53c9e89876ff3b531a887ab656ca303 x
 F src/btree.h 03e3356f5208bcab8eed4e094240fdac4a7f9f5ddf5e91045ce589f67d47c240
-F src/btreeInt.h ef12a72b708677e48d6bc8dcd66fed25434740568b89e2cfa368093cfc5b9d15
-F src/build.c 189e4517d67f09f0a3e0d8e1faa6e2ef0c2e95f6ac82e33c912cb7efa2a359cc
+F src/btreeInt.h 3e2589726c4f105e653461814f65857465da68be1fac688de340c43b873f4062
+F src/build.c 386eadecabe2e99a3783eb802ca01e665f8e0c2af0e0aab28161fd7def219a9d
 F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 23331529e654be40ca97d171cbbffe9b3d4c71cc53b78fe5501230675952da8b
@@ -675,20 +686,20 @@ F src/date.c 3b8d02977d160e128469de38493b4085f7c5cf4073193459909a6af3cf6d7c91
 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782
 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43
 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500
-F src/expr.c 88629faed0b576b7ffa3d82ce44cbcee4ed476a2bf1ea4e1d6bf1260e03b19cb
+F src/expr.c 05278def9c186b5875d6903ea26148c7461b9ce0344f0fd7be9a0dfea0a4538a
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00
 F src/func.c 472f6dcfa39cf54f89a6aec76c79c225fb880a6c14469c15d361331662b9bf43
-F src/global.c 29f56a330ed9d1b5cd9b79ac0ca36f97ac3afc730ff8bfa987b0db9e559d684d
+F src/global.c 765a0656d6cbf043cb272ff0ae38f39cc46713539ffe6793258ed3eb4b188b52
 F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220
 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c a647820e1e82fd516df80328ef3372209006bf99b7008d156411c8cd0d4d9fc4
+F src/json.c cd270629b4b99d065522b45c3d2b5185491b1ab3544843c6f780aa50755d54dc
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36
-F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9
+F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4
 F src/malloc.c f016922435dc7d1f1f5083a03338a3e91f8c67ce2c5bdcfa4cdef62e612f5fcc
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2
@@ -702,14 +713,14 @@ F src/mutex.c 1b4c7e5e3621b510e0c18397210be27cd54c8084141144fbbafd003fde948e88
 F src/mutex.h a7b2293c48db5f27007c3bdb21d438873637d12658f5a0bf8ad025bb96803c4a
 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4
 F src/mutex_unix.c f7ee5a2061a4c11815a2bf4fc0e2bfa6fb8d9dc89390eb613ca0cec32fc9a3d1
-F src/mutex_w32.c 38b56d0bc8d54c17c20cbaaad3719b0c36b92fd07a7e34360d0c6a18d5589912
+F src/mutex_w32.c 28f8d480387db5b2ef5248705dd4e19db0cfc12c3ba426695a7d2c45c48e6885
 F src/notify.c 57c2d1a2805d6dee32acd5d250d928ab94e02d76369ae057dee7d445fd64e878
 F src/os.c 509452169d5ea739723e213b8e2481cf0e587f0e88579a912d200db5269f5f6d
 F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63
 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06
 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a
 F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107
-F src/os_unix.c 0a33005e6426702c7e76f3d451f296c088693a95b2be28ba9ef59c8d8529ce6b
+F src/os_unix.c 97bdcd43315da7aaec9fea2da1ff7c9de458f93dd363e073f2742403a7f2e011
 F src/os_win.c 4a50a154aeebc66a1f8fb79c1ff6dd5fe3d005556533361e0d460d41cb6a45a8
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 987ab3a2cd9065d62e9955474470ff733445e2357432a67e3d0f5a8f9313e334
@@ -718,19 +729,19 @@ F src/parse.y 020d80386eb216ec9520549106353c517d2bbc89be28752ffdca649a9eaf56ec
 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75
 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5
 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00
-F src/pragma.c b3b4ad9c0298d63098a067acca613c21a5f56b4d176d5842922bcd0b07b7164e
+F src/pragma.c b5b4cff830575e6188cd56a295a57448d2b9dbc53f0dae58e22b97354cda3781
 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
-F src/prepare.c bde74add20fc0e8ce0c4e937a1f70a36d17413afe4f71d3e103f5cb74b17c8d9
-F src/printf.c 9da63b9ae1c14789bcae12840f5d800fd9302500cd2d62733fac77f0041b4750
+F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c
+F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c d017bad7ba8e778617701a0e986fdeb393d67d6afa84fb28ef4e8b8ad2acf916
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
-F src/select.c 503331aca8785254a7bf3d74ab338a99118fa297e1184a4dde33b3cdf7a9d341
-F src/shell.c.in 297625a1ba6ea1c08bc2ea1b838b646cad309b62bf08df0e379355629404f140
-F src/sqlite.h.in 4f841d3d117b830ee5ee45e8d89ceff1195f3ebb72d041ace8d116ba4c103b35
+F src/select.c 9f55c9f3307b9888f62abe709eec245e98ff217bd14c044f93d72810bb7dc445
+F src/shell.c.in 0cd2ef4b3c814dded5436625ab664d9a973cbc4266a1768e3aa4cbf11bb41eca
+F src/sqlite.h.in adcc7dbfeea1e69d6d487139a7e90db8a48fe998f3f5bb0f85c683e6a6fa68ca
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
-F src/sqliteInt.h 6b82eb99a9d2887e873fb29e56befb7c50cf4624df615d23a28f071dc8abf5f6
+F src/sqliteInt.h 615524fff083abf9353aa4a3a8adca2d32b3e61e014e62b1a5335c454a386783
 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6
 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -795,11 +806,11 @@ F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242
 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
 F src/util.c b22cc9f203a8c0b9ee5338a67f8860347d14845864c10248bebe84518a781677
 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104
-F src/vdbe.c 44bccc33ff8864b4b937be4d67ebb244e4f4e5ebc6d44832298954df6b7d6a78
-F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0
+F src/vdbe.c f73bead140670fac1aa4227188827ada52387a5fe0ccff0dd5af2a906754d904
+F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb
 F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c
 F src/vdbeapi.c 8f57d60c89da0b60e6d4e272358c511f6bae4e24330bdb11f8b42f986d1bf21b
-F src/vdbeaux.c f3997b5956c8d97bd2fc3392db42caecddfa6549e9df82e0a7e5804653ca475a
+F src/vdbeaux.c c5a471b34e9c4cfc0295a3e10734fd197670ffaebcb742f284c8e17e8026ceea
 F src/vdbeblob.c 13f9287b55b6356b4b1845410382d6bede203ceb29ef69388a4a3d007ffacbe5
 F src/vdbemem.c 0012d5f01cc866833847c2f3ae4c318ac53a1cb3d28acad9c35e688039464cf0
 F src/vdbesort.c 237840ca1947511fa59bd4e18b9eeae93f2af2468c34d2427b059f896230a547
@@ -807,10 +818,10 @@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf8
 F src/vdbevtab.c 2143db7db0ceed69b21422581f434baffc507a08d831565193a7a02882a1b6a7
 F src/vtab.c 154725ebecd3bc02f7fbd7ad3974334f73fff76e02a964e828e48a7c5fb7efff
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c bba7db5dae3ffe2c6b9c173fc10be4b570b125e985cb5b95a6c22716213adde4
+F src/wal.c e5247a3406531b705b44630e9ccf9ca0e5c74955ef19c06fbb146d765c500c20
 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452
 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2
-F src/where.c 1fdc69ce1333e9bd6d7d3df9fa5af1373a3f5bfdd52108d1dbc0ca85a55f777e
+F src/where.c dad4b472b8dc886c0962409e70978568c06b5108bcf3dcd4938a9305a3540440
 F src/whereInt.h 4b38c5889514e3aead3f27d0ee9a26e47c3f150efc59e2a8b4e3bc8835e4d7a1
 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1
 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00
@@ -824,7 +835,7 @@ F test/aggnested.test ce85a6af7d59c3109e35c5f03b2cd11da1a9b1417371e2f942102d0f0d
 F test/aggorderby.test e6b98dbbf3ababa96892435d387de2dcf602ef02c2b848d2d817473066f154ba
 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87
 F test/all.test 2ecb8bbd52416642e41c9081182a8df05d42c75637afd4488aace78cc4b69e13
-F test/alter.test 403a7f8842457044a994d0ffb42963d6e84fcfbf5e8f54556063b25d966cd454
+F test/alter.test 3c00eff1e2036b9f93e9cd0f3d3e63750ac87ecb5bc71b9d7bd07cbf2ac4c494
 F test/alter2.test a966ccfcddf9ce0a4e0e6ff1aca9e6e7948e0e242cd7e43fc091948521807687
 F test/alter3.test ffc4ab29ce78a3517a66afd69b2730667e3471622509c283b2bd4c46f680fba3
 F test/alter4.test 716caa071dd8a3c6d57225778d15d3c3cbf5e34b2e84ae44199aeb2bbf50a707
@@ -1218,7 +1229,7 @@ F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c
 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634
 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830
 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2
-F test/fuzzcheck.c 69b8549e112fb815931a8c14c7955a0c407ae91a79356eecb82458384f2cb989
+F test/fuzzcheck.c 442b0b02146bb905c47c4ee1388949e7fd9d27acf22d9535e9abf216e582b956
 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517
 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f
 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba
@@ -1319,16 +1330,17 @@ F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd7
 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa
 F test/json/README.md 63e3e589e1df8fd3cc1588ba1faaff659214003f8b77a15af5c6452b35e30ee2
 F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f
-F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b72447a147a462c
+F test/json/json-q1-b.txt 1e180fe6491efab307e318b22879e3a736ac9a96539bbde7911a13ee5b33abc7
 F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307
 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x
-F test/json101.test 2420fbed1d2dddb9562f8b9d0ac540849b99deba74e869560f94f674ef24171c
+F test/json101.test 70587d7d35ef9e2126364ba70f0c951f70827cfbd28649d779ff3df7e8f87547
 F test/json102.test 557a46e16df1aa9bdbc4076a71a45814ea0e7503d6621d87d42a8c04cbc2b0ef
 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe
 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1
-F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d
-F test/json501.test c419deb835b70c1a2c8532936927bcc1146730328edd2052276715bfd209724d
-F test/json502.test 98c38e3c4573841028a1381dfb81d4c3f9b105d39668167da10d055e503f6d0b
+F test/json105.test e64a8d73677fbae67886642cd5076e2ef3efe89f8483b87595cf9c030216c9bd
+F test/json501.test ab168a12eb6eb14d479f8c1cdae3ac062fd5a4679f17f976e96f1af518408330
+F test/json502.test 3c697e506fc38ccb455b49660b21b6e62e08ede0f2d0c869a7d171e17809093c
+F test/jsonb01.test cace70765b36a36aec9a85a41ea65667d3bbf647d4400ddc3ac76f8fe7d94f90
 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff
 F test/kvtest.c 6e0228409ea7ca0497dad503fbd109badb5e59545d131014b6aaac68b56f484a
 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63
@@ -1565,7 +1577,7 @@ F test/sharedA.test 64bdd21216dda2c6a3bd3475348ccdc108160f34682c97f2f51c19fc0e21
 F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8ee707
 F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939
 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304
-F test/shell1.test 2191c892d8256b1b6cccb4fca894cfc2e748450cf422e6ec80a320a0d538b6df
+F test/shell1.test c7127a5e780ffc9e14c476773127fdf292c6db226529c44c1676f37b3793123f
 F test/shell2.test 35226c070a8c7f64fd016dfac2a0db2a40f709b3131f61daacd9dad61536c9cb
 F test/shell3.test 91febeac0412812bf6370abb8ed72700e32bf8f9878849414518f662dfd55e8a
 F test/shell4.test 9abd0c12a7e20a4c49e84d5be208d2124fa6c09e728f56f1f4bee0f02853935f
@@ -1651,7 +1663,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163
 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc
 F test/tester.tcl 68454ef88508c196d19e8694daa27bff7107a91857799eaa12f417188ae53ede
-F test/testrunner.tcl 8a6721213bce1cfd3b33e1588cc6431143d96b98819206bf91f5a205fbb150d4
+F test/testrunner.tcl e18d71f2e797da808ba6d31335e504ed6b2791581b89287a72b697a2f31b1ea1
 F test/testrunner_data.tcl e4d5017290a6d5c11785e36cc94c67d8bb950c8cdc2dbe4c1db2a3a583812560
 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899
 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502
@@ -1817,7 +1829,7 @@ F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7
 F test/tpch01.test 4479008f85f6f8f25f7ab2cb305d665752b4727fa28a8df3d8e0ad46520c62ff
 F test/trace.test a659a9862957f4789e37a92b3bf6d2caf5c86b02cdeefc41e850ae53acf6992a
 F test/trace2.test f5cb67ad3bc09e0c58e8cca78dfd0b5639259983
-F test/trace3.test ae2004df24b585fed9046cc0bae4601762bc6fc4aa321d475f1350bba5047f31
+F test/trace3.test 4f418ed30d15d9d17dcf13a17f0bd99a92e3038e038798e35db7525f82f4c281
 F test/trans.test 45f6f9ab6f66a7b5744f1caac06b558f95da62501916906cf55586a896f9f439
 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76
 F test/trans3.test 91a100e5412b488e22a655fe423a14c26403ab94
@@ -1968,7 +1980,7 @@ F test/whereC.test cae295158703cb3fc23bf1a108a9ab730efff0f6
 F test/whereD.test c1c335e914e28b122e000e9310f02d2be83e1c9dbca2e29f46bd732703944d1b
 F test/whereE.test 7a727b5d5b6bc8fa4cef5206e90cc0363e55ca7f0566f6fbad0206e43170f59e
 F test/whereF.test 926b65519608e3f2aa28720822b9154fb5c7b13519dd78194f434a511ab3dac5
-F test/whereG.test b2a479f425f7d0a432df7e842e8484560908ef703fe0fd407888ff85e7097238
+F test/whereG.test 649d5ad02a87a76ec2ac8de9441e2c83a4dd0f29e459a31215c0533788c6bf07
 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2
 F test/whereI.test c4bb7e2ca56d49bd8ab5c7bd085b8b83e353922b46904d68aefb3c7468643581
 F test/whereJ.test fc05e374cc9f2dc204148d6c06822c380ad388895fe97a6d335b94a26a08aecf
@@ -2141,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a79ff8e58fcaf718a6fb78e145117f1d6d40d133f31e9752bb9c6d484850a27b
-R 6ab8908817380ee469413c76bca0c832
+P be1864eac4eb75cc30bf98f73092c8608467f4bd956240df6a0cbea9f1e09e85
+R 6e221a94cd5525fe9f8a7e94ea467b60
 U drh
-Z a103fcc805c4c7ff278e886cba60f9d7
+Z 2ea9e875faf360824d045f495f8867aa
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-e76d48137ea823b7810dc8c3b70eb21adabdd6cfbac36050c85d1375e94be1de
+c96ebb086feb89341565cc52b970ae7799ce1327fe1ad4fc790f1b0dcaa6e229

src/btreeInt.h

@@ -64,7 +64,7 @@
 **     22       1     Min embedded payload fraction (must be 32)
 **     23       1     Min leaf payload fraction (must be 32)
 **     24       4     File change counter
-**     28       4     Reserved for future use
+**     28       4     The size of the database in pages
 **     32       4     First freelist page
 **     36       4     Number of freelist pages in the file
 **     40      60     15 4-byte meta values passed to higher layers

src/build.c

@@ -720,7 +720,7 @@ void sqlite3ColumnSetExpr(
 */
 Expr *sqlite3ColumnExpr(Table *pTab, Column *pCol){
   if( pCol->iDflt==0 ) return 0;
-  if( NEVER(!IsOrdinaryTable(pTab)) ) return 0;
+  if( !IsOrdinaryTable(pTab) ) return 0;
   if( NEVER(pTab->u.tab.pDfltList==0) ) return 0;
   if( NEVER(pTab->u.tab.pDfltList->nExpr<pCol->iDflt) ) return 0;
   return pTab->u.tab.pDfltList->a[pCol->iDflt-1].pExpr;
@@ -873,6 +873,9 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){
   if( db->pnBytesFreed==0 && (--pTable->nTabRef)>0 ) return;
   deleteTable(db, pTable);
 }
+void sqlite3DeleteTableGeneric(sqlite3 *db, void *pTable){
+  sqlite3DeleteTable(db, (Table*)pTable);
+}
 
 
 /*
@@ -1410,7 +1413,8 @@ void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){
 /*
 ** Clean up the data structures associated with the RETURNING clause.
 */
-static void sqlite3DeleteReturning(sqlite3 *db, Returning *pRet){
+static void sqlite3DeleteReturning(sqlite3 *db, void *pArg){
+  Returning *pRet = (Returning*)pArg;
   Hash *pHash;
   pHash = &(db->aDb[1].pSchema->trigHash);
   sqlite3HashInsert(pHash, pRet->zName, 0);
@@ -1452,8 +1456,7 @@ void sqlite3AddReturning(Parse *pParse, ExprList *pList){
   pParse->u1.pReturning = pRet;
   pRet->pParse = pParse;
   pRet->pReturnEL = pList;
-  sqlite3ParserAddCleanup(pParse,
-     (void(*)(sqlite3*,void*))sqlite3DeleteReturning, pRet);
+  sqlite3ParserAddCleanup(pParse, sqlite3DeleteReturning, pRet);
   testcase( pParse->earlyCleanup );
   if( db->mallocFailed ) return;
   sqlite3_snprintf(sizeof(pRet->zName), pRet->zName,
@@ -5692,4 +5695,7 @@ void sqlite3WithDelete(sqlite3 *db, With *pWith){
     sqlite3DbFree(db, pWith);
   }
 }
+void sqlite3WithDeleteGeneric(sqlite3 *db, void *pWith){
+  sqlite3WithDelete(db, (With*)pWith);
+}
 #endif /* !defined(SQLITE_OMIT_CTE) */

src/expr.c

@@ -1223,9 +1223,7 @@ void sqlite3ExprAddFunctionOrderBy(
   assert( ExprUseXList(pExpr) );
   if( pExpr->x.pList==0 || NEVER(pExpr->x.pList->nExpr==0) ){
     /* Ignore ORDER BY on zero-argument aggregates */
-    sqlite3ParserAddCleanup(pParse,
-        (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
-        pOrderBy);
+    sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, pOrderBy);
     return;
   }
   if( IsWindowFunc(pExpr) ){
@@ -1406,6 +1404,9 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){
 void sqlite3ExprDelete(sqlite3 *db, Expr *p){
   if( p ) sqlite3ExprDeleteNN(db, p);
 }
+void sqlite3ExprDeleteGeneric(sqlite3 *db, void *p){
+  if( ALWAYS(p) ) sqlite3ExprDeleteNN(db, (Expr*)p);
+}
 
 /*
 ** Clear both elements of an OnOrUsing object
@@ -1431,9 +1432,7 @@ void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){
 ** pExpr to the pParse->pConstExpr list with a register number of 0.
 */
 void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){
-  sqlite3ParserAddCleanup(pParse,
-    (void(*)(sqlite3*,void*))sqlite3ExprDelete,
-    pExpr);
+  sqlite3ParserAddCleanup(pParse, sqlite3ExprDeleteGeneric, pExpr);
 }
 
 /* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the
@@ -2239,6 +2238,9 @@ static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){
 void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){
   if( pList ) exprListDeleteNN(db, pList);
 }
+void sqlite3ExprListDeleteGeneric(sqlite3 *db, void *pList){
+  if( ALWAYS(pList) ) exprListDeleteNN(db, (ExprList*)pList);
+}
 
 /*
 ** Return the bitwise-OR of all Expr.flags fields in the given
@@ -6043,8 +6045,8 @@ int sqlite3ExprListCompare(const ExprList *pA, const ExprList *pB, int iTab){
 */
 int sqlite3ExprCompareSkip(Expr *pA,Expr *pB, int iTab){
   return sqlite3ExprCompare(0,
-             sqlite3ExprSkipCollateAndLikely(pA),
-             sqlite3ExprSkipCollateAndLikely(pB),
+             sqlite3ExprSkipCollate(pA),
+             sqlite3ExprSkipCollate(pB),
              iTab);
 }
 

src/global.c

@@ -244,6 +244,9 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = {
    0,                         /* bSmallMalloc */
    1,                         /* bExtraSchemaChecks */
    sizeof(LONGDOUBLE_TYPE)>8, /* bUseLongDouble */
+#ifdef SQLITE_DEBUG
+   0,                         /* bJsonSelfcheck */
+#endif
    0x7ffffffe,                /* mxStrlen */
    0,                         /* neverCorrupt */
    SQLITE_DEFAULT_LOOKASIDE,  /* szLookaside, nLookaside */

src/main.c

@@ -4659,6 +4659,28 @@ int sqlite3_test_control(int op, ...){
       break;
     }
 #endif
+
+    /* sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &onOff);
+    **
+    ** Activate or deactivate validation of JSONB that is generated from
+    ** text.  Off by default, as the validation is slow.  Validation is
+    ** only available if compiled using SQLITE_DEBUG.
+    **
+    ** If onOff is initially 1, then turn it on.  If onOff is initially
+    ** off, turn it off.  If onOff is initially -1, then change onOff
+    ** to be the current setting.
+    */
+    case SQLITE_TESTCTRL_JSON_SELFCHECK: {
+#if defined(SQLITE_DEBUG)
+      int *pOnOff = va_arg(ap, int*);
+      if( *pOnOff<0 ){
+        *pOnOff = sqlite3Config.bJsonSelfcheck;
+      }else{
+        sqlite3Config.bJsonSelfcheck = (u8)((*pOnOff)&0xff);
+      }
+#endif
+      break;
+    }
   }
   va_end(ap);
 #endif /* SQLITE_UNTESTABLE */

src/mutex_w32.c

@@ -87,7 +87,7 @@ void sqlite3MemoryBarrier(void){
   SQLITE_MEMORY_BARRIER;
 #elif defined(__GNUC__)
   __sync_synchronize();
-#elif MSVC_VERSION>=1300
+#elif MSVC_VERSION>=1400
   _ReadWriteBarrier();
 #elif defined(MemoryBarrier)
   MemoryBarrier();

src/os_unix.c

@@ -4054,7 +4054,13 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){
 #ifdef SQLITE_ENABLE_SETLK_TIMEOUT
     case SQLITE_FCNTL_LOCK_TIMEOUT: {
       int iOld = pFile->iBusyTimeout;
+#if SQLITE_ENABLE_SETLK_TIMEOUT==1
       pFile->iBusyTimeout = *(int*)pArg;
+#elif SQLITE_ENABLE_SETLK_TIMEOUT==2
+      pFile->iBusyTimeout = !!(*(int*)pArg);
+#else
+# error "SQLITE_ENABLE_SETLK_TIMEOUT must be set to 1 or 2"
+#endif
       *(int*)pArg = iOld;
       return SQLITE_OK;
     }
@@ -4307,6 +4313,25 @@ static int unixGetpagesize(void){
 ** Either unixShmNode.pShmMutex must be held or unixShmNode.nRef==0 and
 ** unixMutexHeld() is true when reading or writing any other field
 ** in this structure.
+**
+** aLock[SQLITE_SHM_NLOCK]:
+**   This array records the various locks held by clients on each of the
+**   SQLITE_SHM_NLOCK slots. If the aLock[] entry is set to 0, then no
+**   locks are held by the process on this slot. If it is set to -1, then
+**   some client holds an EXCLUSIVE lock on the locking slot. If the aLock[]
+**   value is set to a positive value, then it is the number of shared 
+**   locks currently held on the slot.
+**
+** aMutex[SQLITE_SHM_NLOCK]:
+**   Normally, when SQLITE_ENABLE_SETLK_TIMEOUT is not defined, mutex 
+**   pShmMutex is used to protect the aLock[] array and the right to
+**   call fcntl() on unixShmNode.hShm to obtain or release locks.
+**
+**   If SQLITE_ENABLE_SETLK_TIMEOUT is defined though, we use an array
+**   of mutexes - one for each locking slot. To read or write locking
+**   slot aLock[iSlot], the caller must hold the corresponding mutex
+**   aMutex[iSlot]. Similarly, to call fcntl() to obtain or release a
+**   lock corresponding to slot iSlot, mutex aMutex[iSlot] must be held.
 */
 struct unixShmNode {
   unixInodeInfo *pInode;     /* unixInodeInfo that owns this SHM node */
@@ -4320,10 +4345,11 @@ struct unixShmNode {
   char **apRegion;           /* Array of mapped shared-memory regions */
   int nRef;                  /* Number of unixShm objects pointing to this */
   unixShm *pFirst;           /* All unixShm objects pointing to this */
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+  sqlite3_mutex *aMutex[SQLITE_SHM_NLOCK];
+#endif
   int aLock[SQLITE_SHM_NLOCK];  /* # shared locks on slot, -1==excl lock */
 #ifdef SQLITE_DEBUG
-  u8 exclMask;               /* Mask of exclusive locks held */
-  u8 sharedMask;             /* Mask of shared locks held */
   u8 nextShmId;              /* Next available unixShm.id value */
 #endif
 };
@@ -4406,16 +4432,29 @@ static int unixShmSystemLock(
   struct flock f;        /* The posix advisory locking structure */
   int rc = SQLITE_OK;    /* Result code form fcntl() */
 
-  /* Access to the unixShmNode object is serialized by the caller */
   pShmNode = pFile->pInode->pShmNode;
-  assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->pShmMutex) );
-  assert( pShmNode->nRef>0 || unixMutexHeld() );
+
+  /* Assert that the correct mutex or mutexes are held. */
+  if( pShmNode->nRef==0 ){
+    assert( ofst==UNIX_SHM_DMS && n==1 && unixMutexHeld() );
+  }else{
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+    int ii;
+    for(ii=ofst-UNIX_SHM_BASE; ii<ofst-UNIX_SHM_BASE+n; ii++){
+      assert( sqlite3_mutex_held(pShmNode->aMutex[ii]) );
+    }
+#else
+    assert( sqlite3_mutex_held(pShmNode->pShmMutex) );
+    assert( pShmNode->nRef>0 );
+#endif
+  }
 
   /* Shared locks never span more than one byte */
   assert( n==1 || lockType!=F_RDLCK );
 
   /* Locks are within range */
   assert( n>=1 && n<=SQLITE_SHM_NLOCK );
+  assert( ofst>=UNIX_SHM_BASE && ofst<=(UNIX_SHM_DMS+SQLITE_SHM_NLOCK) );
 
   if( pShmNode->hShm>=0 ){
     int res;
@@ -4426,7 +4465,7 @@ static int unixShmSystemLock(
     f.l_len = n;
     res = osSetPosixAdvisoryLock(pShmNode->hShm, &f, pFile);
     if( res==-1 ){
-#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+#if defined(SQLITE_ENABLE_SETLK_TIMEOUT) && SQLITE_ENABLE_SETLK_TIMEOUT==1
       rc = (pFile->iBusyTimeout ? SQLITE_BUSY_TIMEOUT : SQLITE_BUSY);
 #else
       rc = SQLITE_BUSY;
@@ -4434,39 +4473,28 @@ static int unixShmSystemLock(
     }
   }
 
-  /* Update the global lock state and do debug tracing */
+  /* Do debug tracing */
 #ifdef SQLITE_DEBUG
-  { u16 mask;
   OSTRACE(("SHM-LOCK "));
-  mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<<ofst);
   if( rc==SQLITE_OK ){
     if( lockType==F_UNLCK ){
-      OSTRACE(("unlock %d ok", ofst));
-      pShmNode->exclMask &= ~mask;
-      pShmNode->sharedMask &= ~mask;
+      OSTRACE(("unlock %d..%d ok\n", ofst, ofst+n-1));
     }else if( lockType==F_RDLCK ){
-      OSTRACE(("read-lock %d ok", ofst));
-      pShmNode->exclMask &= ~mask;
-      pShmNode->sharedMask |= mask;
+      OSTRACE(("read-lock %d..%d ok\n", ofst, ofst+n-1));
     }else{
       assert( lockType==F_WRLCK );
-      OSTRACE(("write-lock %d ok", ofst));
-      pShmNode->exclMask |= mask;
-      pShmNode->sharedMask &= ~mask;
+      OSTRACE(("write-lock %d..%d ok\n", ofst, ofst+n-1));
     }
   }else{
     if( lockType==F_UNLCK ){
-      OSTRACE(("unlock %d failed", ofst));
+      OSTRACE(("unlock %d..%d failed\n", ofst, ofst+n-1));
     }else if( lockType==F_RDLCK ){
-      OSTRACE(("read-lock failed"));
+      OSTRACE(("read-lock %d..%d failed\n", ofst, ofst+n-1));
     }else{
       assert( lockType==F_WRLCK );
-      OSTRACE(("write-lock %d failed", ofst));
+      OSTRACE(("write-lock %d..%d failed\n", ofst, ofst+n-1));
     }
   }
-  OSTRACE((" - afterwards %03x,%03x\n",
-           pShmNode->sharedMask, pShmNode->exclMask));
-  }
 #endif
 
   return rc;      
@@ -4503,6 +4531,11 @@ static void unixShmPurge(unixFile *pFd){
     int i;
     assert( p->pInode==pFd->pInode );
     sqlite3_mutex_free(p->pShmMutex);
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+    for(i=0; i<SQLITE_SHM_NLOCK; i++){
+      sqlite3_mutex_free(p->aMutex[i]);
+    }
+#endif
     for(i=0; i<p->nRegion; i+=nShmPerMap){
       if( p->hShm>=0 ){
         osMunmap(p->apRegion[i], p->szRegion);
@@ -4562,7 +4595,20 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){
       pShmNode->isUnlocked = 1;
       rc = SQLITE_READONLY_CANTINIT;
     }else{
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+      /* Do not use a blocking lock here. If the lock cannot be obtained
+      ** immediately, it means some other connection is truncating the
+      ** *-shm file. And after it has done so, it will not release its
+      ** lock, but only downgrade it to a shared lock. So no point in
+      ** blocking here. The call below to obtain the shared DMS lock may 
+      ** use a blocking lock. */
+      int iSaveTimeout = pDbFd->iBusyTimeout;
+      pDbFd->iBusyTimeout = 0;
+#endif
       rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1);
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+      pDbFd->iBusyTimeout = iSaveTimeout;
+#endif
       /* The first connection to attach must truncate the -shm file.  We
       ** truncate to 3 bytes (an arbitrary small number, less than the
       ** -shm header size) rather than 0 as a system debugging aid, to
@@ -4683,6 +4729,18 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
         rc = SQLITE_NOMEM_BKPT;
         goto shm_open_err;
       }
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+      {
+        int ii;
+        for(ii=0; ii<SQLITE_SHM_NLOCK; ii++){
+          pShmNode->aMutex[ii] = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST);
+          if( pShmNode->aMutex[ii]==0 ){
+            rc = SQLITE_NOMEM_BKPT;
+            goto shm_open_err;
+          }
+        }
+      }
+#endif
     }
 
     if( pInode->bProcessLock==0 ){
@@ -4904,9 +4962,11 @@ shmpage_out:
 */
 #ifdef SQLITE_DEBUG
 static int assertLockingArrayOk(unixShmNode *pShmNode){
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+  return 1;
+#else
   unixShm *pX;
   int aLock[SQLITE_SHM_NLOCK];
-  assert( sqlite3_mutex_held(pShmNode->pShmMutex) );
 
   memset(aLock, 0, sizeof(aLock));
   for(pX=pShmNode->pFirst; pX; pX=pX->pNext){
@@ -4924,13 +4984,14 @@ static int assertLockingArrayOk(unixShmNode *pShmNode){
 
   assert( 0==memcmp(pShmNode->aLock, aLock, sizeof(aLock)) );
   return (memcmp(pShmNode->aLock, aLock, sizeof(aLock))==0);
+#endif
 }
 #endif
 
 /*
 ** Change the lock state for a shared-memory segment.
 **
-** Note that the relationship between SHAREd and EXCLUSIVE locks is a little
+** Note that the relationship between SHARED and EXCLUSIVE locks is a little
 ** different here than in posix.  In xShmLock(), one can go from unlocked
 ** to shared and back or from unlocked to exclusive and back.  But one may
 ** not go from shared to exclusive or from exclusive to shared.
@@ -4945,7 +5006,7 @@ static int unixShmLock(
   unixShm *p;                           /* The shared memory being locked */
   unixShmNode *pShmNode;                /* The underlying file iNode */
   int rc = SQLITE_OK;                   /* Result code */
-  u16 mask;                             /* Mask of locks to take or release */
+  u16 mask = (1<<(ofst+n)) - (1<<ofst); /* Mask of locks to take or release */
   int *aLock;
 
   p = pDbFd->pShm;
@@ -4991,80 +5052,139 @@ static int unixShmLock(
   }
 #endif
 
-  mask = (1<<(ofst+n)) - (1<<ofst);
-  assert( n>1 || mask==(1<<ofst) );
-  sqlite3_mutex_enter(pShmNode->pShmMutex);
-  assert( assertLockingArrayOk(pShmNode) );
-  if( flags & SQLITE_SHM_UNLOCK ){
-    if( (p->exclMask|p->sharedMask) & mask ){
-      int ii;
-      int bUnlock = 1;
+  /* Check if there is any work to do. There are three cases:
+  **
+  **    a) An unlock operation where there are locks to unlock,
+  **    b) An shared lock where the requested lock is not already held
+  **    c) An exclusive lock where the requested lock is not already held
+  **
+  ** The SQLite core never requests an exclusive lock that it already holds.
+  ** This is assert()ed below.
+  */
+  assert( flags!=(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK) 
+       || 0==(p->exclMask & mask) 
+  );
+  if( ((flags & SQLITE_SHM_UNLOCK) && ((p->exclMask|p->sharedMask) & mask))
+   || (flags==(SQLITE_SHM_SHARED|SQLITE_SHM_LOCK) && 0==(p->sharedMask & mask))
+   || (flags==(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK))
+  ){
 
-      for(ii=ofst; ii<ofst+n; ii++){
-        if( aLock[ii]>((p->sharedMask & (1<<ii)) ? 1 : 0) ){
-          bUnlock = 0;
-        }
-      }
-
-      if( bUnlock ){
-        rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n);
-        if( rc==SQLITE_OK ){
-          memset(&aLock[ofst], 0, sizeof(int)*n);
-        }
-      }else if( ALWAYS(p->sharedMask & (1<<ofst)) ){
-        assert( n==1 && aLock[ofst]>1 );
-        aLock[ofst]--;
-      }
-
-      /* Undo the local locks */
-      if( rc==SQLITE_OK ){
-        p->exclMask &= ~mask;
-        p->sharedMask &= ~mask;
+    /* Take the required mutexes. In SETLK_TIMEOUT mode (blocking locks), if
+    ** this is an attempt on an exclusive lock use sqlite3_mutex_try(). If any
+    ** other thread is holding this mutex, then it is either holding or about
+    ** to hold a lock exclusive to the one being requested, and we may
+    ** therefore return SQLITE_BUSY to the caller.
+    **
+    ** Doing this prevents some deadlock scenarios. For example, thread 1 may
+    ** be a checkpointer blocked waiting on the WRITER lock. And thread 2
+    ** may be a normal SQL client upgrading to a write transaction. In this
+    ** case thread 2 does a non-blocking request for the WRITER lock. But -
+    ** if it were to use sqlite3_mutex_enter() then it would effectively
+    ** become a (doomed) blocking request, as thread 2 would block until thread
+    ** 1 obtained WRITER and released the mutex. Since thread 2 already holds
+    ** a lock on a read-locking slot at this point, this breaks the
+    ** anti-deadlock rules (see above).  */
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+    int iMutex;
+    for(iMutex=ofst; iMutex<ofst+n; iMutex++){
+      if( flags==(SQLITE_SHM_LOCK|SQLITE_SHM_EXCLUSIVE) ){
+        rc = sqlite3_mutex_try(pShmNode->aMutex[iMutex]);
+        if( rc!=SQLITE_OK ) break;
+      }else{
+        sqlite3_mutex_enter(pShmNode->aMutex[iMutex]);
       }
     }
-  }else if( flags & SQLITE_SHM_SHARED ){
-    assert( n==1 );
-    assert( (p->exclMask & (1<<ofst))==0 );
-    if( (p->sharedMask & mask)==0 ){
-      if( aLock[ofst]<0 ){
-        rc = SQLITE_BUSY;
-      }else if( aLock[ofst]==0 ){
-        rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n);
-      }
+#else
+    sqlite3_mutex_enter(pShmNode->pShmMutex);
+#endif
 
-      /* Get the local shared locks */
-      if( rc==SQLITE_OK ){
-        p->sharedMask |= mask;
-        aLock[ofst]++;
-      }
-    }
-  }else{
-    /* Make sure no sibling connections hold locks that will block this
-    ** lock.  If any do, return SQLITE_BUSY right away.  */
-    int ii;
-    for(ii=ofst; ii<ofst+n; ii++){
-      assert( (p->sharedMask & mask)==0 );
-      if( ALWAYS((p->exclMask & (1<<ii))==0) && aLock[ii] ){
-        rc = SQLITE_BUSY;
-        break;
-      }
-    }
-
-    /* Get the exclusive locks at the system level. Then if successful
-    ** also update the in-memory values. */
     if( rc==SQLITE_OK ){
-      rc = unixShmSystemLock(pDbFd, F_WRLCK, ofst+UNIX_SHM_BASE, n);
-      if( rc==SQLITE_OK ){
+      if( flags & SQLITE_SHM_UNLOCK ){
+        /* Case (a) - unlock.  */
+        int bUnlock = 1;
+        assert( (p->exclMask & p->sharedMask)==0 );
+        assert( !(flags & SQLITE_SHM_EXCLUSIVE) || (p->exclMask & mask)==mask );
+        assert( !(flags & SQLITE_SHM_SHARED) || (p->sharedMask & mask)==mask );
+  
+        /* If this is a SHARED lock being unlocked, it is possible that other
+        ** clients within this process are holding the same SHARED lock. In
+        ** this case, set bUnlock to 0 so that the posix lock is not removed
+        ** from the file-descriptor below.  */
+        if( flags & SQLITE_SHM_SHARED ){
+          assert( n==1 );
+          assert( aLock[ofst]>=1 );
+          if( aLock[ofst]>1 ){
+            bUnlock = 0;
+            aLock[ofst]--;
+            p->sharedMask &= ~mask;
+          }
+        }
+  
+        if( bUnlock ){
+          rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n);
+          if( rc==SQLITE_OK ){
+            memset(&aLock[ofst], 0, sizeof(int)*n);
+            p->sharedMask &= ~mask;
+            p->exclMask &= ~mask;
+          }
+        }
+      }else if( flags & SQLITE_SHM_SHARED ){
+        /* Case (b) - a shared lock.  */
+  
+        if( aLock[ofst]<0 ){
+          /* An exclusive lock is held by some other connection. BUSY. */ 
+          rc = SQLITE_BUSY;
+        }else if( aLock[ofst]==0 ){
+          rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n);
+        }
+  
+        /* Get the local shared locks */
+        if( rc==SQLITE_OK ){
+          p->sharedMask |= mask;
+          aLock[ofst]++;
+        }
+      }else{
+        /* Case (c) - an exclusive lock.  */
+        int ii;
+  
+        assert( flags==(SQLITE_SHM_LOCK|SQLITE_SHM_EXCLUSIVE) );
         assert( (p->sharedMask & mask)==0 );
-        p->exclMask |= mask;
+        assert( (p->exclMask & mask)==0 );
+  
+        /* Make sure no sibling connections hold locks that will block this
+        ** lock.  If any do, return SQLITE_BUSY right away.  */
         for(ii=ofst; ii<ofst+n; ii++){
-          aLock[ii] = -1;
+          if( aLock[ii] ){
+            rc = SQLITE_BUSY;
+            break;
+          }
+        }
+  
+        /* Get the exclusive locks at the system level. Then if successful
+        ** also update the in-memory values. */
+        if( rc==SQLITE_OK ){
+          rc = unixShmSystemLock(pDbFd, F_WRLCK, ofst+UNIX_SHM_BASE, n);
+          if( rc==SQLITE_OK ){
+            p->exclMask |= mask;
+            for(ii=ofst; ii<ofst+n; ii++){
+              aLock[ii] = -1;
+            }
+          }
         }
       }
+      assert( assertLockingArrayOk(pShmNode) );
     }
+
+    /* Drop the mutexes acquired above. */
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+    for(iMutex--; iMutex>=ofst; iMutex--){
+      sqlite3_mutex_leave(pShmNode->aMutex[iMutex]);
+    }
+#else
+    sqlite3_mutex_leave(pShmNode->pShmMutex);
+#endif
   }
-  assert( assertLockingArrayOk(pShmNode) );
-  sqlite3_mutex_leave(pShmNode->pShmMutex);
+
   OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x\n",
            p->id, osGetpid(0), p->sharedMask, p->exclMask));
   return rc;

src/pragma.c

@@ -1779,7 +1779,8 @@ void sqlite3Pragma(
           if( pVTab->pModule->iVersion<4 ) continue;
           if( pVTab->pModule->xIntegrity==0 ) continue;
           sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick);
-          sqlite3VdbeAppendP4(v, pTab, P4_TABLE);
+          pTab->nTabRef++;
+          sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF);
           a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v);
           integrityCheckResultRow(v);
           sqlite3VdbeJumpHere(v, a1);

src/prepare.c

@@ -868,6 +868,7 @@ static int sqlite3LockAndPrepare(
   assert( (rc&db->errMask)==rc );
   db->busyHandler.nBusy = 0;
   sqlite3_mutex_leave(db->mutex);
+  assert( rc==SQLITE_OK || (*ppStmt)==0 );
   return rc;
 }
 

src/printf.c

@@ -1369,7 +1369,7 @@ void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){
 
 
 /*****************************************************************************
-** Reference counted string storage
+** Reference counted string/blob storage
 *****************************************************************************/
 
 /*

src/select.c

@@ -184,6 +184,9 @@ Select *sqlite3SelectNew(
 void sqlite3SelectDelete(sqlite3 *db, Select *p){
   if( OK_IF_ALWAYS_TRUE(p) ) clearSelect(db, p, 1);
 }
+void sqlite3SelectDeleteGeneric(sqlite3 *db, void *p){
+  if( ALWAYS(p) ) clearSelect(db, (Select*)p, 1);
+}
 
 /*
 ** Return a pointer to the right-most SELECT statement in a compound.
@@ -2319,7 +2322,8 @@ void sqlite3SubqueryColumnTypes(
   NameContext sNC;
 
   assert( pSelect!=0 );
-  assert( (pSelect->selFlags & SF_Resolved)!=0 );
+  testcase( (pSelect->selFlags & SF_Resolved)==0 );
+  assert( (pSelect->selFlags & SF_Resolved)!=0 || IN_RENAME_OBJECT );
   assert( pTab->nCol==pSelect->pEList->nExpr || pParse->nErr>0 );
   assert( aff==SQLITE_AFF_NONE || aff==SQLITE_AFF_BLOB );
   if( db->mallocFailed || IN_RENAME_OBJECT ) return;
@@ -3203,9 +3207,7 @@ multi_select_end:
   pDest->iSdst = dest.iSdst;
   pDest->nSdst = dest.nSdst;
   if( pDelete ){
-    sqlite3ParserAddCleanup(pParse,
-        (void(*)(sqlite3*,void*))sqlite3SelectDelete,
-        pDelete);
+    sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pDelete);
   }
   return rc;
 }
@@ -3756,8 +3758,7 @@ static int multiSelectOrderBy(
   /* Make arrangements to free the 2nd and subsequent arms of the compound
   ** after the parse has finished */
   if( pSplit->pPrior ){
-    sqlite3ParserAddCleanup(pParse,
-       (void(*)(sqlite3*,void*))sqlite3SelectDelete, pSplit->pPrior);
+    sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pSplit->pPrior);
   }
   pSplit->pPrior = pPrior;
   pPrior->pNext = pSplit;
@@ -4578,9 +4579,7 @@ static int flattenSubquery(
     Table *pTabToDel = pSubitem->pTab;
     if( pTabToDel->nTabRef==1 ){
       Parse *pToplevel = sqlite3ParseToplevel(pParse);
-      sqlite3ParserAddCleanup(pToplevel,
-         (void(*)(sqlite3*,void*))sqlite3DeleteTable,
-         pTabToDel);
+      sqlite3ParserAddCleanup(pToplevel, sqlite3DeleteTableGeneric, pTabToDel);
       testcase( pToplevel->earlyCleanup );
     }else{
       pTabToDel->nTabRef--;
@@ -5627,8 +5626,7 @@ static struct Cte *searchWith(
 With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){
   if( pWith ){
     if( bFree ){
-      pWith = (With*)sqlite3ParserAddCleanup(pParse,
-                      (void(*)(sqlite3*,void*))sqlite3WithDelete,
+      pWith = (With*)sqlite3ParserAddCleanup(pParse, sqlite3WithDeleteGeneric,
                       pWith);
       if( pWith==0 ) return 0;
     }
@@ -6372,10 +6370,11 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){
   SrcList *pTabList;
   SrcItem *pFrom;
 
-  assert( p->selFlags & SF_Resolved );
   if( p->selFlags & SF_HasTypeInfo ) return;
   p->selFlags |= SF_HasTypeInfo;
   pParse = pWalker->pParse;
+  testcase( (p->selFlags & SF_Resolved)==0 );
+  assert( (p->selFlags & SF_Resolved) || IN_RENAME_OBJECT );
   pTabList = p->pSrc;
   for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){
     Table *pTab = pFrom->pTab;
@@ -7035,7 +7034,8 @@ static SrcItem *isSelfJoinView(
 /*
 ** Deallocate a single AggInfo object
 */
-static void agginfoFree(sqlite3 *db, AggInfo *p){
+static void agginfoFree(sqlite3 *db, void *pArg){
+  AggInfo *p = (AggInfo*)pArg;
   sqlite3DbFree(db, p->aCol);
   sqlite3DbFree(db, p->aFunc);
   sqlite3DbFreeNN(db, p);
@@ -7289,9 +7289,8 @@ int sqlite3Select(
         sqlite3TreeViewExprList(0, p->pOrderBy, 0, "ORDERBY");
       }
 #endif   
-      sqlite3ParserAddCleanup(pParse,
-        (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
-        p->pOrderBy);
+      sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric,
+                              p->pOrderBy);
       testcase( pParse->earlyCleanup );
       p->pOrderBy = 0;
     }
@@ -7483,9 +7482,8 @@ int sqlite3Select(
     ){
       TREETRACE(0x800,pParse,p,
                 ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1));
-      sqlite3ParserAddCleanup(pParse,
-         (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
-         pSub->pOrderBy);
+      sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric,
+                              pSub->pOrderBy);
       pSub->pOrderBy = 0;
     }
 
@@ -8014,8 +8012,7 @@ int sqlite3Select(
     */
     pAggInfo = sqlite3DbMallocZero(db, sizeof(*pAggInfo) );
     if( pAggInfo ){
-      sqlite3ParserAddCleanup(pParse,
-          (void(*)(sqlite3*,void*))agginfoFree, pAggInfo);
+      sqlite3ParserAddCleanup(pParse, agginfoFree, pAggInfo);
       testcase( pParse->earlyCleanup );
     }
     if( db->mallocFailed ){

src/sqlite.h.in

@@ -3954,15 +3954,17 @@ void sqlite3_free_filename(sqlite3_filename);
 ** </ul>
 **
 ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language
-** text that describes the error, as either UTF-8 or UTF-16 respectively.
+** text that describes the error, as either UTF-8 or UTF-16 respectively,
+** or NULL if no error message is available.
 ** (See how SQLite handles [invalid UTF] for exceptions to this rule.)
 ** ^(Memory to hold the error message string is managed internally.
 ** The application does not need to worry about freeing the result.
 ** However, the error string might be overwritten or deallocated by
 ** subsequent calls to other SQLite interface functions.)^
 **
-** ^The sqlite3_errstr() interface returns the English-language text
-** that describes the [result code], as UTF-8.
+** ^The sqlite3_errstr(E) interface returns the English-language text
+** that describes the [result code] E, as UTF-8, or NULL if E is not an
+** result code for which a text error message is available.
 ** ^(Memory to hold the error message string is managed internally
 ** and must not be freed by the application)^.
 **
@@ -8298,6 +8300,7 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_ASSERT                  12
 #define SQLITE_TESTCTRL_ALWAYS                  13
 #define SQLITE_TESTCTRL_RESERVE                 14  /* NOT USED */
+#define SQLITE_TESTCTRL_JSON_SELFCHECK          14
 #define SQLITE_TESTCTRL_OPTIMIZATIONS           15
 #define SQLITE_TESTCTRL_ISKEYWORD               16  /* NOT USED */
 #define SQLITE_TESTCTRL_SCRATCHMALLOC           17  /* NOT USED */

src/sqliteInt.h

@@ -4116,6 +4116,9 @@ struct sqlite3_str {
 **
 **   3.  Make a (read-only) copy of a read-only RCStr string using
 **       sqlite3RCStrRef().
+**
+** "String" is in the name, but an RCStr object can also be used to hold
+** binary data.
 */
 struct RCStr {
   u64 nRCRef;            /* Number of references */
@@ -4174,6 +4177,9 @@ struct Sqlite3Config {
   u8 bSmallMalloc;                  /* Avoid large memory allocations if true */
   u8 bExtraSchemaChecks;            /* Verify type,name,tbl_name in schema */
   u8 bUseLongDouble;                /* Make use of long double */
+#ifdef SQLITE_DEBUG
+  u8 bJsonSelfcheck;                /* Double-check JSON parsing */
+#endif
   int mxStrlen;                     /* Maximum string length */
   int neverCorrupt;                 /* Database is always well-formed */
   int szLookaside;                  /* Default lookaside buffer size */
@@ -4800,6 +4806,7 @@ void sqlite3ExprOrderByAggregateError(Parse*,Expr*);
 void sqlite3ExprFunctionUsable(Parse*,const Expr*,const FuncDef*);
 void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32);
 void sqlite3ExprDelete(sqlite3*, Expr*);
+void sqlite3ExprDeleteGeneric(sqlite3*,void*);
 void sqlite3ExprDeferredDelete(Parse*, Expr*);
 void sqlite3ExprUnmapAndDelete(Parse*, Expr*);
 ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*);
@@ -4809,6 +4816,7 @@ void sqlite3ExprListSetSortOrder(ExprList*,int,int);
 void sqlite3ExprListSetName(Parse*,ExprList*,const Token*,int);
 void sqlite3ExprListSetSpan(Parse*,ExprList*,const char*,const char*);
 void sqlite3ExprListDelete(sqlite3*, ExprList*);
+void sqlite3ExprListDeleteGeneric(sqlite3*,void*);
 u32 sqlite3ExprListFlags(const ExprList*);
 int sqlite3IndexHasDuplicateRootPage(Index*);
 int sqlite3Init(sqlite3*, char**);
@@ -4899,6 +4907,7 @@ void sqlite3CreateView(Parse*,Token*,Token*,Token*,ExprList*,Select*,int,int);
 void sqlite3DropTable(Parse*, SrcList*, int, int);
 void sqlite3CodeDropTable(Parse*, Table*, int, int);
 void sqlite3DeleteTable(sqlite3*, Table*);
+void sqlite3DeleteTableGeneric(sqlite3*, void*);
 void sqlite3FreeIndex(sqlite3*, Index*);
 #ifndef SQLITE_OMIT_AUTOINCREMENT
   void sqlite3AutoincrementBegin(Parse *pParse);
@@ -4935,6 +4944,7 @@ int sqlite3Select(Parse*, Select*, SelectDest*);
 Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*,
                          Expr*,ExprList*,u32,Expr*);
 void sqlite3SelectDelete(sqlite3*, Select*);
+void sqlite3SelectDeleteGeneric(sqlite3*,void*);
 Table *sqlite3SrcListLookup(Parse*, SrcList*);
 int sqlite3IsReadOnly(Parse*, Table*, Trigger*);
 void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
@@ -5507,6 +5517,7 @@ const char *sqlite3JournalModename(int);
   void sqlite3CteDelete(sqlite3*,Cte*);
   With *sqlite3WithAdd(Parse*,With*,Cte*);
   void sqlite3WithDelete(sqlite3*,With*);
+  void sqlite3WithDeleteGeneric(sqlite3*,void*);
   With *sqlite3WithPush(Parse*, With*, u8);
 #else
 # define sqlite3CteNew(P,T,E,S)   ((void*)0)

src/vdbe.c

@@ -132,11 +132,12 @@ int sqlite3_found_count = 0;
 **   sqlite3CantopenError(lineno)
 */
 static void test_trace_breakpoint(int pc, Op *pOp, Vdbe *v){
-  static int n = 0;
+  static u64 n = 0;
   (void)pc;
   (void)pOp;
   (void)v;
   n++;
+  if( n==LARGEST_UINT64 ) abort(); /* So that n is used, preventing a warning */
 }
 #endif
 
@@ -8179,24 +8180,23 @@ case OP_VCheck: {             /* out2 */
 
   pOut = &aMem[pOp->p2];
   sqlite3VdbeMemSetNull(pOut);  /* Innocent until proven guilty */
-  assert( pOp->p4type==P4_TABLE );
+  assert( pOp->p4type==P4_TABLEREF );
   pTab = pOp->p4.pTab;
   assert( pTab!=0 );
+  assert( pTab->nTabRef>0 );
   assert( IsVirtual(pTab) );
-  assert( pTab->u.vtab.p!=0 );
+  if( pTab->u.vtab.p==0 ) break;
   pVtab = pTab->u.vtab.p->pVtab;
   assert( pVtab!=0 );
   pModule = pVtab->pModule;
   assert( pModule!=0 );
   assert( pModule->iVersion>=4 );
   assert( pModule->xIntegrity!=0 );
-  pTab->nTabRef++;
   sqlite3VtabLock(pTab->u.vtab.p);
   assert( pOp->p1>=0 && pOp->p1<db->nDb );
   rc = pModule->xIntegrity(pVtab, db->aDb[pOp->p1].zDbSName, pTab->zName,
                            pOp->p3, &zErr);
   sqlite3VtabUnlock(pTab->u.vtab.p);
-  sqlite3DeleteTable(db, pTab);
   if( rc ){
     sqlite3_free(zErr);
     goto abort_due_to_error;

src/vdbe.h

@@ -126,6 +126,7 @@ typedef struct VdbeOpList VdbeOpList;
 #define P4_INT64      (-13) /* P4 is a 64-bit signed integer */
 #define P4_INTARRAY   (-14) /* P4 is a vector of 32-bit integers */
 #define P4_FUNCCTX    (-15) /* P4 is a pointer to an sqlite3_context object */
+#define P4_TABLEREF   (-16) /* Like P4_TABLE, but reference counted */
 
 /* Error message codes for OP_Halt */
 #define P5_ConstraintNotNull 1

src/vdbeaux.c

@@ -211,10 +211,11 @@ static int growOpArray(Vdbe *v, int nOp){
 **   sqlite3CantopenError(lineno)
 */
 static void test_addop_breakpoint(int pc, Op *pOp){
-  static int n = 0;
+  static u64 n = 0;
   (void)pc;
   (void)pOp;
   n++;
+  if( n==LARGEST_UINT64 ) abort(); /* so that n is used, preventing a warning */
 }
 #endif
 
@@ -1399,6 +1400,10 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){
       if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4);
       break;
     }
+    case P4_TABLEREF: {
+      if( db->pnBytesFreed==0 ) sqlite3DeleteTable(db, (Table*)p4);
+      break;
+    }
   }
 }
 

src/wal.c

@@ -2004,6 +2004,19 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){
 }
 
 #ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+
+
+/*
+** Attempt to enable blocking locks that block for nMs ms. Return 1 if 
+** blocking locks are successfully enabled, or 0 otherwise.
+*/
+static int walEnableBlockingMs(Wal *pWal, int nMs){
+  int rc = sqlite3OsFileControl(
+      pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&nMs
+  );
+  return (rc==SQLITE_OK);
+}
+
 /*
 ** Attempt to enable blocking locks. Blocking locks are enabled only if (a)
 ** they are supported by the VFS, and (b) the database handle is configured
@@ -2015,11 +2028,7 @@ static int walEnableBlocking(Wal *pWal){
   if( pWal->db ){
     int tmout = pWal->db->busyTimeout;
     if( tmout ){
-      int rc;
-      rc = sqlite3OsFileControl(
-          pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&tmout
-      );
-      res = (rc==SQLITE_OK);
+      res = walEnableBlockingMs(pWal, tmout);
     }
   }
   return res;
@@ -2068,20 +2077,10 @@ void sqlite3WalDb(Wal *pWal, sqlite3 *db){
   pWal->db = db;
 }
 
-/*
-** Take an exclusive WRITE lock. Blocking if so configured.
-*/
-static int walLockWriter(Wal *pWal){
-  int rc;
-  walEnableBlocking(pWal);
-  rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1);
-  walDisableBlocking(pWal);
-  return rc;
-}
 #else
 # define walEnableBlocking(x) 0
 # define walDisableBlocking(x)
-# define walLockWriter(pWal) walLockExclusive((pWal), WAL_WRITE_LOCK, 1)
+# define walEnableBlockingMs(pWal, ms) 0
 # define sqlite3WalDb(pWal, db)
 #endif   /* ifdef SQLITE_ENABLE_SETLK_TIMEOUT */
 
@@ -2682,7 +2681,9 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
       }
     }else{
       int bWriteLock = pWal->writeLock;
-      if( bWriteLock || SQLITE_OK==(rc = walLockWriter(pWal)) ){
+      if( bWriteLock 
+       || SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) 
+      ){
         pWal->writeLock = 1;
         if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){
           badHdr = walIndexTryHdr(pWal, pChanged);
@@ -2690,7 +2691,8 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
             /* If the wal-index header is still malformed even while holding
             ** a WRITE lock, it can only mean that the header is corrupted and
             ** needs to be reconstructed.  So run recovery to do exactly that.
-            */
+            ** Disable blocking locks first.  */
+            walDisableBlocking(pWal);
             rc = walIndexRecover(pWal);
             *pChanged = 1;
           }
@@ -2957,6 +2959,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
   int i;                          /* Loop counter */
   int rc = SQLITE_OK;             /* Return code  */
   u32 mxFrame;                    /* Wal frame to lock to */
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+  int nBlockTmout = 0;
+#endif
 
   assert( pWal->readLock<0 );     /* Not currently locked */
 
@@ -2987,6 +2992,19 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
       return SQLITE_PROTOCOL;
     }
     if( cnt>=10 ) nDelay = (cnt-9)*(cnt-9)*39;
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+    /* In SQLITE_ENABLE_SETLK_TIMEOUT builds, configure the file-descriptor
+    ** to block for locks for approximately nDelay us. This affects three
+    ** locks: (a) the shared lock taken on the DMS slot in os_unix.c (if
+    ** using os_unix.c), (b) the WRITER lock taken in walIndexReadHdr() if the
+    ** first attempted read fails, and (c) the shared lock taken on the DMS
+    ** slot in os_unix.c. All three of these locks are attempted from within 
+    ** the call to walIndexReadHdr() below.  */
+    nBlockTmout = (nDelay+998) / 1000;
+    if( !useWal && walEnableBlockingMs(pWal, nBlockTmout) ){
+      nDelay = 1;
+    }
+#endif
     sqlite3OsSleep(pWal->pVfs, nDelay);
   }
 
@@ -2995,6 +3013,10 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
     if( pWal->bShmUnreliable==0 ){
       rc = walIndexReadHdr(pWal, pChanged);
     }
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+    walDisableBlocking(pWal);
+    if( rc==SQLITE_BUSY_TIMEOUT ) rc = SQLITE_BUSY;
+#endif
     if( rc==SQLITE_BUSY ){
       /* If there is not a recovery running in another thread or process
       ** then convert BUSY errors to WAL_RETRY.  If recovery is known to
@@ -3109,9 +3131,12 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
     return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT;
   }
 
+  (void)walEnableBlockingMs(pWal, nBlockTmout);
   rc = walLockShared(pWal, WAL_READ_LOCK(mxI));
+  walDisableBlocking(pWal);
   if( rc ){
-    return rc==SQLITE_BUSY ? WAL_RETRY : rc;
+    assert( (rc&0xFF)!=SQLITE_BUSY||rc==SQLITE_BUSY||rc==SQLITE_BUSY_TIMEOUT );
+    return (rc&0xFF)==SQLITE_BUSY ? WAL_RETRY : rc;
   }
   /* Now that the read-lock has been obtained, check that neither the
   ** value in the aReadMark[] array or the contents of the wal-index
@@ -4204,10 +4229,9 @@ int sqlite3WalCheckpoint(
   if( pWal->readOnly ) return SQLITE_READONLY;
   WALTRACE(("WAL%p: checkpoint begins\n", pWal));
 
-  /* Enable blocking locks, if possible. If blocking locks are successfully
-  ** enabled, set xBusy2=0 so that the busy-handler is never invoked. */
+  /* Enable blocking locks, if possible. */
   sqlite3WalDb(pWal, db);
-  (void)walEnableBlocking(pWal);
+  if( xBusy2 ) (void)walEnableBlocking(pWal);
 
   /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive
   ** "checkpoint" lock on the database file.

src/where.c

@@ -2020,7 +2020,8 @@ static int whereRangeScanEst(
           ** sample, then assume they are 4x more selective.  This brings
           ** the estimated selectivity more in line with what it would be
           ** if estimated without the use of STAT4 tables. */
-          if( iLwrIdx==iUprIdx ) nNew -= 20;  assert( 20==sqlite3LogEst(4) );
+          if( iLwrIdx==iUprIdx ){ nNew -= 20; }
+          assert( 20==sqlite3LogEst(4) );
         }else{
           nNew = 10;        assert( 10==sqlite3LogEst(2) );
         }

test/alter.test

@@ -954,4 +954,33 @@ do_catchsql_test alter-20.3 {
   INSERT INTO t1(a) VALUES(45);
 } {1 {cannot store BLOB value in TEXT column t1.b}}
 
+# 2023-11-17 dbsqlfuzz e0900262dadd5c78c2226ad6a435c7f0255be2cd
+# Assertion fault associated with ALTER TABLE and an
+# aggregate ORDER BY within an unknown aggregate function.
+#
+reset_db
+do_execsql_test alter-21.1 {
+  CREATE TABLE t1(a,b,c,d);
+  CREATE TABLE t2(a,b,c,d,x);
+  CREATE TRIGGER r1 AFTER INSERT ON t2 BEGIN
+    SELECT unknown_function(a ORDER BY (SELECT group_concat(DISTINCT a ORDER BY a) FROM t1)) FROM t1;
+  END;
+  ALTER TABLE t2 RENAME TO e;
+} {}
+do_execsql_test alter-21.2 {
+  SELECT name, type FROM sqlite_schema ORDER BY name;
+} {e table r1 trigger t1 table}
+do_execsql_test alter-21.3 {
+  DROP TRIGGER r1;
+  CREATE TRIGGER r2 AFTER INSERT ON e BEGIN
+    SELECT unknown_function(a ORDER BY (SELECT group_concat(a ORDER BY a) FROM (SELECT b FROM t1))) FROM t1;
+  END;
+  ALTER TABLE e RENAME TO t99;
+}
+do_execsql_test alter-21.4 {
+  SELECT name, type FROM sqlite_schema ORDER BY name;
+} {r2 trigger t1 table t99 table}
+
+
+
 finish_test

test/fuzzcheck.c

@@ -1267,6 +1267,8 @@ int runCombinedDbSqlInput(
   }
   sqlite3_limit(cx.db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH, 100);
   sqlite3_hard_heap_limit64(heapLimit);
+  rc = 1;
+  sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &rc);
 
   if( nDb>=20 && aDb[18]==2 && aDb[19]==2 ){
     aDb[18] = aDb[19] = 1;

test/json/json-q1-b.txt

@@ -1,6 +1,6 @@
 .mode qbox
 .timer on
-.param set $label '$.q87'
+.param set $label 'q87'
 SELECT rowid, x->>$label FROM data1 WHERE x->>$label IS NOT NULL;
 
 CREATE TEMP TABLE t2(x JSON TEXT);
@@ -14,8 +14,7 @@ WITH RECURSIVE
   ),
   c2(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c2 WHERE n<5)
 INSERT INTO t2(x)
-  SELECT jsonb(json_object('a',n,'b',n*2,'c',y,'d',3,'e',5,'f',6))
-    FROM array1, c2;
+  SELECT jsonb_object('a',n,'b',n*2,'c',y,'d',3,'e',5,'f',6) FROM array1, c2;
 CREATE INDEX t2x1 ON t2(x->>'a');
 CREATE INDEX t2x2 ON t2(x->>'b');
 CREATE INDEX t2x3 ON t2(x->>'e');

test/json101.test

@@ -324,7 +324,7 @@ do_execsql_test json101-5.2 {
   SELECT id, json_valid(json), json_type(json), '|' FROM j2 ORDER BY id;
 } {1 1 object | 2 1 object | 3 1 array |}
 do_execsql_test json101-5.2b {
-  SELECT id, json_valid(json), json_type(json), '|' FROM j2b ORDER BY id;
+  SELECT id, json_valid(json,5), json_type(json), '|' FROM j2b ORDER BY id;
 } {1 1 object | 2 1 object | 3 1 array |}
 
 ifcapable !vtab {
@@ -925,16 +925,16 @@ do_execsql_test json101-14.170 {
 #
 do_execsql_test json101-15.100 {
   SELECT * FROM JSON_EACH('{"a":1, "b":2}');
-} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+} {a 1 integer 1 1 {} {$.a} {$} b 2 integer 2 5 {} {$.b} {$}}
 do_execsql_test json101-15.110 {
   SELECT xyz.* FROM JSON_EACH('{"a":1, "b":2}') AS xyz;
-} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+} {a 1 integer 1 1 {} {$.a} {$} b 2 integer 2 5 {} {$.b} {$}}
 do_execsql_test json101-15.120 {
   SELECT * FROM (JSON_EACH('{"a":1, "b":2}'));
-} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+} {a 1 integer 1 1 {} {$.a} {$} b 2 integer 2 5 {} {$.b} {$}}
 do_execsql_test json101-15.130 {
   SELECT xyz.* FROM (JSON_EACH('{"a":1, "b":2}')) AS xyz;
-} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+} {a 1 integer 1 1 {} {$.a} {$} b 2 integer 2 5 {} {$.b} {$}}
 
 # 2019-11-10
 # Mailing list bug report on the handling of surrogate pairs
@@ -974,7 +974,7 @@ do_execsql_test json101-18.4 {
 } {6}
 do_catchsql_test json101-18.5 {
   SELECT json_extract('{"":8}', '$.');
-} {1 {JSON path error near ''}}
+} {1 {bad JSON path: '$.'}}
 
 # 2022-08-29 https://sqlite.org/forum/forumpost/9b9e4716c0d7bbd1
 # This is not a problem specifically with JSON functions.  It is
@@ -1139,4 +1139,29 @@ do_execsql_test json101-23.2 {
     FROM (SELECT json_set('[]','$[#]',0,'$[#]',1) AS j);
 } {{[0,1]} 0 1}
 
+# Insert/Set/Replace where the path specifies substructure that
+# does not yet exist
+#
+proc tx x {return [string map [list ( \173 ) \175 ' \042 < \133 > \135] $x]}
+foreach {id start path ins set repl} {
+  1 {{}}       {$.a.b.c}     ('a':('b':('c':9)))      ('a':('b':('c':9)))      ()
+  2 {{a:4}}    {$.a.b.c}     ('a':4)                  ('a':4)                  ('a':4)
+  3 {{a:{}}}   {$.a.b.c}     ('a':('b':('c':9)))      ('a':('b':('c':9)))      ('a':())
+  4 {[0,1,2]}  {$[3].a[0].b} <0,1,2,('a':<('b':9)>)>  <0,1,2,('a':<('b':9)>)>  <0,1,2>
+  5 {[0,1,2]}  {$[1].a[0].b} <0,1,2>                  <0,1,2>                  <0,1,2>
+  6 {[0,{},2]} {$[1].a[0].b} <0,('a':<('b':9)>),2>    <0,('a':<('b':9)>),2>    <0,(),2>
+  7 {[0,1,2]}  {$[3][0].b}   <0,1,2,<('b':9)>>        <0,1,2,<('b':9)>>        <0,1,2>
+  8 {[0,1,2]}  {$[1][0].b}   <0,1,2>                  <0,1,2>                  <0,1,2>
+} {
+  do_execsql_test json101-24.$id.insert {
+    SELECT json_insert($start,$path,9);
+  } [list [tx $ins]]
+  do_execsql_test json101-24.$id.set {
+    SELECT json_set($start,$path,9);
+  } [list [tx $set]]
+  do_execsql_test json101-24.$id.replace {
+    SELECT json_replace($start,$path,9);
+  } [list [tx $repl]]
+}
+
 finish_test

test/json105.test

@@ -96,18 +96,18 @@ json_replace_test 80 {'$.b[#-1]','AAA','$.b[#-1]','BBB'} \
 
 do_catchsql_test json105-6.10 {
   SELECT json_extract(j, '$.b[#-]') FROM t1;
-} {1 {JSON path error near '[#-]'}}
+} {1 {bad JSON path: '$.b[#-]'}}
 do_catchsql_test json105-6.20 {
   SELECT json_extract(j, '$.b[#9]') FROM t1;
-} {1 {JSON path error near '[#9]'}}
+} {1 {bad JSON path: '$.b[#9]'}}
 do_catchsql_test json105-6.30 {
   SELECT json_extract(j, '$.b[#+2]') FROM t1;
-} {1 {JSON path error near '[#+2]'}}
+} {1 {bad JSON path: '$.b[#+2]'}}
 do_catchsql_test json105-6.40 {
   SELECT json_extract(j, '$.b[#-1') FROM t1;
-} {1 {JSON path error near '[#-1'}}
+} {1 {bad JSON path: '$.b[#-1'}}
 do_catchsql_test json105-6.50 {
   SELECT json_extract(j, '$.b[#-1x]') FROM t1;
-} {1 {JSON path error near '[#-1x]'}}
+} {1 {bad JSON path: '$.b[#-1x]'}}
 
 finish_test

test/json501.test

@@ -252,13 +252,13 @@ do_execsql_test 8.11 {
 
 do_execsql_test 9.1 {
   WITH c(x) AS (VALUES('{x: +Infinity}')) SELECT x->>'x', json(x) FROM c;
-} {Inf {{"x":9.0e999}}}
+} {Inf {{"x":9e999}}}
 do_execsql_test 9.2 {
   WITH c(x) AS (VALUES('{x: -Infinity}')) SELECT x->>'x', json(x) FROM c;
-} {-Inf {{"x":-9.0e999}}}
+} {-Inf {{"x":-9e999}}}
 do_execsql_test 9.3 {
   WITH c(x) AS (VALUES('{x: Infinity}')) SELECT x->>'x', json(x) FROM c;
-} {Inf {{"x":9.0e999}}}
+} {Inf {{"x":9e999}}}
 do_execsql_test 9.4 {
   WITH c(x) AS (VALUES('{x: NaN}')) SELECT x->>'x', json(x) FROM c;
 } {{} {{"x":null}}}

test/json502.test

@@ -36,5 +36,30 @@ do_catchsql_test 2.3 {
   SELECT '{a:null,{"h":[1,[1,2,3]],"j":"abc"}:true}'->'$h[#-1]';
 } {1 {malformed JSON}}
 
+# Verify that escaped label names are compared correctly.
+# 
+do_execsql_test 3.1 {
+  SELECT '{"a\x62c":123}' ->> 'abc';
+} 123
+do_execsql_test 3.2 {
+  SELECT '{"abc":123}' ->> 'a\x62c';
+} 123
+
+db null null
+do_execsql_test 3.3 {
+  DROP TABLE t1;
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES(json_insert('{}','$.a\',111,'$."b\\"',222));
+  INSERT INTO t1 VALUES(jsonb_insert('{}','$.a\',111,'$."b\\"',222));
+  SELECT x->'$.a\', x->'$.a\\', x->'$."a\\"', x->'$."b\\"' FROM t1;
+} {111 null 111 222 111 null 111 222}
+
+do_execsql_test 3.4 {
+  SELECT json_patch('{"a\x62c":123}','{"ab\x63":456}') ->> 'abc';
+} 456
+
+do_execsql_test 4.1 {
+  SELECT * FROM json_tree('{"\u0017":1}','$."\x17"');
+} {{\x17} 1 integer 1 1 null {$."\x17"} {$}}
 
 finish_test

test/jsonb01.test

@@ -0,0 +1,49 @@
+# 2023-11-15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# Test cases for JSONB
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_execsql_test jsonb01-1.1 {
+  CREATE TABLE t1(x JSON BLOB);
+  INSERT INTO t1 VALUES(jsonb('{a:5,b:{x:10,y:11},c:[1,2,3,4]}'));
+}
+foreach {id path res} {
+  1 {$.a}    {{{"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+  2 {$.b}    {{{"a":5,"c":[1,2,3,4]}}}
+  3 {$.c}    {{{"a":5,"b":{"x":10,"y":11}}}}
+  4 {$.d}    {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+  5 {$.b.x}  {{{"a":5,"b":{"y":11},"c":[1,2,3,4]}}}
+  6 {$.b.y}  {{{"a":5,"b":{"x":10},"c":[1,2,3,4]}}}
+  7 {$.c[0]}  {{{"a":5,"b":{"x":10,"y":11},"c":[2,3,4]}}}
+  8 {$.c[1]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,3,4]}}}
+  9 {$.c[2]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,4]}}}
+ 10 {$.c[3]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3]}}}
+ 11 {$.c[4]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+ 12 {$.c[#]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+ 13 {$.c[#-1]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3]}}}
+ 14 {$.c[#-2]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,4]}}}
+ 15 {$.c[#-3]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,3,4]}}}
+ 16 {$.c[#-4]}  {{{"a":5,"b":{"x":10,"y":11},"c":[2,3,4]}}}
+ 17 {$.c[#-5]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+ 18 {$.c[#-6]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+} {
+  do_execsql_test jsonb01-1.2.$id.1 {
+    SELECT json(jsonb_remove(x,$path)) FROM t1;
+  } $res
+  do_execsql_test jsonb01-1.2.$id.2 {
+    SELECT json_remove(x,$path) FROM t1;
+  } $res
+}
+
+finish_test

test/shell1.test

@@ -496,11 +496,16 @@ do_test shell1-3.14.3 {
 
 # .output FILENAME       Send output to FILENAME
 do_test shell1-3.15.1 {
-  catchcmd "test.db" ".output"
-} {0 {}}
+  catchcmd "test.db" ".output
+.print x"
+} {0 x}
 do_test shell1-3.15.2 {
-  catchcmd "test.db" ".output FOO"
-} {0 {}}
+  catchcmd "test.db" ".output FOO
+.print x
+.output
+SELECT readfile('FOO');"
+} {0 {x
+}}
 do_test shell1-3.15.3 {
   # too many arguments
   catchcmd "test.db" ".output FOO BAD"
@@ -1067,7 +1072,7 @@ do_test shell1-5.0 {
       #
       set escapes [list \
           \a \\a \b \\b \t \\t \n \\n \v \\v \f \\f \r \\r \
-          " " "\" \"" \" \\\" ' \"'\" \\ \\\\]
+          " " "\" \"" \" \\\" \\ \\\\]
     } else {
       #
       # NOTE: On Unix, we need to escape most of the whitespace characters

test/testrunner.tcl

@@ -148,6 +148,8 @@ set TRG(cmdline) $argv
 set TRG(reporttime) 2000
 set TRG(fuzztest) 0                 ;# is the fuzztest option present.
 set TRG(zipvfs) ""                  ;# -zipvfs option, if any
+set TRG(buildonly) 0                ;# True if --buildonly option 
+set TRG(dryrun) 0                   ;# True if --dryrun option 
 
 switch -nocase -glob -- $tcl_platform(os) {
   *darwin* {
@@ -427,6 +429,10 @@ for {set ii 0} {$ii < [llength $argv]} {incr ii} {
       incr ii
       set TRG(zipvfs) [file normalize [lindex $argv $ii]]
       if {$isLast} { usage }
+    } elseif {($n>2 && [string match "$a*" --buildonly]) || $a=="-b"} {
+      set TRG(buildonly) 1
+    } elseif {($n>2 && [string match "$a*" --dryrun]) || $a=="-d"} {
+      set TRG(dryrun) 1
     } else {
       usage
     }
@@ -752,6 +758,20 @@ proc add_zipvfs_jobs {} {
   set ::env(SQLITE_TEST_DIR) $::testdir
 }
 
+# Used to add jobs for "mdevtest" and "sdevtest".
+#
+proc add_devtest_jobs {lBld patternlist} {
+  global TRG
+
+  foreach b $lBld {
+    set bld [add_build_job $b $TRG(testfixture)]
+    add_tcl_jobs $bld veryquick $patternlist
+    if {$patternlist==""} {
+      add_fuzztest_jobs $b
+    }
+  }
+}
+
 proc add_jobs_from_cmdline {patternlist} {
   global TRG
 
@@ -775,33 +795,28 @@ proc add_jobs_from_cmdline {patternlist} {
     }
 
     mdevtest {
-      foreach b [list All-O0 All-Debug] {
-        set bld [add_build_job $b $TRG(testfixture)]
-        add_tcl_jobs $bld veryquick ""
-        add_fuzztest_jobs $b
-      }
+      add_devtest_jobs {All-O0 All-Debug} [lrange $patternlist 1 end]
     }
 
     sdevtest {
-      foreach b [list All-Sanitize All-Debug] {
-        set bld [add_build_job $b $TRG(testfixture)]
-        add_tcl_jobs $bld veryquick ""
-        add_fuzztest_jobs $b
-      }
+      add_devtest_jobs {All-Sanitize All-Debug} [lrange $patternlist 1 end]
     }
 
     release {
+      set patternlist [lrange $patternlist 1 end]
       foreach b [trd_builds $TRG(platform)] {
         set bld [add_build_job $b $TRG(testfixture)]
         foreach c [trd_configs $TRG(platform) $b] {
-          add_tcl_jobs $bld $c ""
+          add_tcl_jobs $bld $c $patternlist
         }
 
-        foreach e [trd_extras $TRG(platform) $b] {
-          if {$e=="fuzztest"} {
-            add_fuzztest_jobs $b
-          } else {
-            add_make_job $bld $e
+        if {$patternlist==""} {
+          foreach e [trd_extras $TRG(platform) $b] {
+            if {$e=="fuzztest"} {
+              add_fuzztest_jobs $b
+            } else {
+              add_make_job $bld $e
+            }
           }
         }
       }
@@ -834,6 +849,17 @@ proc make_new_testset {} {
 
 }
 
+proc mark_job_as_finished {jobid output state endtm} {
+  r_write_db {
+    trdb eval {
+      UPDATE jobs 
+        SET output=$output, state=$state, endtime=$endtm
+        WHERE jobid=$jobid;
+      UPDATE jobs SET state='ready' WHERE depid=$jobid;
+    }
+  }
+}
+
 proc script_input_ready {fd iJob jobid} {
   global TRG
   global O
@@ -868,15 +894,7 @@ proc script_input_ready {fd iJob jobid} {
     puts $TRG(log) "### $job(displayname) ${jobtm}ms ($state)"
     puts $TRG(log) [string trim $O($iJob)]
 
-    r_write_db {
-      set output $O($iJob)
-      trdb eval {
-        UPDATE jobs 
-          SET output=$output, state=$state, endtime=$tm
-          WHERE jobid=$jobid;
-        UPDATE jobs SET state='ready' WHERE depid=$jobid;
-      }
-    }
+    mark_job_as_finished $jobid $O($iJob) $state $tm
 
     dirs_freeDir $iJob
     launch_some_jobs
@@ -928,16 +946,28 @@ proc launch_another_job {iJob} {
     close $fd
   }
 
-  set pwd [pwd]
-  cd $dir
-  set fd [open $TRG(run) w]
-  puts $fd $job(cmd) 
-  close $fd
-  set fd [open "|$TRG(runcmd) 2>@1" r]
-  cd $pwd
+  if { $TRG(dryrun) } {
 
-  fconfigure $fd -blocking false
-  fileevent $fd readable [list script_input_ready $fd $iJob $job(jobid)]
+    mark_job_as_finished $job(jobid) "" done 0
+    dirs_freeDir $iJob
+    if {$job(build)!=""} {
+      puts $TRG(log) "(cd $dir ; $job(cmd) )"
+    } else {
+      puts $TRG(log) "$job(cmd)"
+    }
+
+  } else {
+    set pwd [pwd]
+    cd $dir
+    set fd [open $TRG(run) w]
+    puts $fd $job(cmd) 
+    close $fd
+    set fd [open "|$TRG(runcmd) 2>@1" r]
+    cd $pwd
+
+    fconfigure $fd -blocking false
+    fileevent $fd readable [list script_input_ready $fd $iJob $job(jobid)]
+  }
 
   return 1
 }
@@ -1035,6 +1065,16 @@ proc run_testset {} {
   puts "Test log is $TRG(logname)"
 }
 
+# Handle the --buildonly option, if it was specified.
+#
+proc handle_buildonly {} {
+  global TRG
+  if {$TRG(buildonly)} {
+    r_write_db {
+      trdb eval { DELETE FROM jobs WHERE displaytype!='bld' }
+    }
+  }
+}
 
 sqlite3 trdb $TRG(dbname)
 trdb timeout $TRG(timeout)
@@ -1043,6 +1083,8 @@ if {$TRG(nJob)>1} {
   puts "splitting work across $TRG(nJob) jobs"
 }
 puts "built testset in [expr $tm/1000]ms.."
+
+handle_buildonly
 run_testset
 trdb close
 #puts [pwd]

test/trace3.test

@@ -132,14 +132,27 @@ do_test trace3-4.3 {
   list $stmt [expr {$ns >= 0 && $ns <= 9999999}]; # less than 0.010 seconds
 } {/^-?\d+ 1$/}
 do_test trace3-4.4 {
-  set ::stmtlist(record) {}
-  db trace_v2 trace_v2_record 2
-  execsql {
-    SELECT a, b FROM t1 ORDER BY a;
+  set cnt 0
+  while {1} {
+    set ::stmtlist(record) {}
+    db trace_v2 trace_v2_record 2
+    execsql {
+      SELECT a, b FROM t1 ORDER BY a;
+    }
+    set stmt [lindex [lindex $::stmtlist(record) 0] 0]
+    set ns [lindex [lindex $::stmtlist(record) 0] 1]
+    if {$ns<0 || $ns>9999999} {  #less than 0.010 seconds
+      incr cnt
+      if {$cnt>3} {
+        set res "time out of bounds.  Expected less than 99999999.  Got $ns"
+        break
+      }
+    } else {
+      set res 1
+      break
+    }
   }
-  set stmt [lindex [lindex $::stmtlist(record) 0] 0]
-  set ns [lindex [lindex $::stmtlist(record) 0] 1]
-  list $stmt [expr {$ns >= 0 && $ns <= 9999999}]; # less than 0.010 seconds
+  list $stmt $res
 } {/^-?\d+ 1$/}
 
 do_test trace3-5.1 {

test/whereG.test

@@ -311,6 +311,20 @@ do_execsql_test 8.9 {
 do_execsql_test 8.10 {
   SELECT * FROM t0 WHERE likelihood(t0.rowid <= '0', 0.5);
 } {}
+# Forum https://sqlite.org/forum/forumpost/45ec3d9788
+reset_db
+do_execsql_test 8.11 {
+  CREATE TABLE t1(c0 INT);
+  INSERT INTO t1(c0) VALUES (NULL);
+  CREATE INDEX i46 ON t1(CAST( (c0 IS TRUE) AS TEXT));
+  CREATE VIEW v0(c2) AS SELECT CAST( (c0 IS TRUE) AS TEXT ) FROM t1;
+}
+do_execsql_test 8.12 {
+  SELECT quote(c0), quote(c2) FROM t1, v0 WHERE  (0 < LIKELY(v0.c2));
+} {NULL '0'} 
+do_execsql_test 8.13 {
+  SELECT quote(c0), quote(c2) FROM t1, v0 WHERE  (0 < LIKELY(v0.c2)) IS TRUE;
+} {NULL '0'}
 
 # 2019-12-31: assertion fault discovered by Yongheng's fuzzer.
 # Harmless memIsValid() due to the code generators failure to