Add NEVER to two unreachable branches in JSON.

FossilOrigin-Name: c96ebb086feb89341565cc52b970ae7799ce1327fe1ad4fc790f1b0dcaa6e229
Improvements to JSONB validation - catch more cases where the input does
2023-12-12 17:13:10 +00:00 · 2023-12-12 14:33:52 +00:00 · 2023-12-12 02:31:12 +00:00 · 2023-12-11 21:00:55 +00:00 · 2023-12-11 20:44:21 +00:00 · 2023-12-11 20:19:10 +00:00
69 changed files with 7641 additions and 2527 deletions
@@ -159,7 +159,7 @@ extension and only later escaped to the wild as an independent library.)

 Test scripts and programs are found in the **test/** subdirectory.
 Additional test code is found in other source repositories.
-See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for
+See [How SQLite Is Tested](https://www.sqlite.org/testing.html) for
 additional information.

 The **ext/** subdirectory contains code for extensions.  The
@@ -183,7 +183,7 @@ manually-edited files and automatically-generated files.

 The SQLite interface is defined by the **sqlite3.h** header file, which is
 generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION.  The
-[Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
+[Tcl script](https://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
 The manifest.uuid file contains the SHA3 hash of the particular check-in
 and is used to generate the SQLITE\_SOURCE\_ID macro.  The VERSION file
 contains the current SQLite version number.  The sqlite3.h header is really
@@ -250,14 +250,14 @@ individual source file exceeds 32K lines in length.
 ## How It All Fits Together

 SQLite is modular in design.
-See the [architectural description](http://www.sqlite.org/arch.html)
+See the [architectural description](https://www.sqlite.org/arch.html)
 for details. Other documents that are useful in
 (helping to understand how SQLite works include the
-[file format](http://www.sqlite.org/fileformat2.html) description,
-the [virtual machine](http://www.sqlite.org/opcode.html) that runs
+[file format](https://www.sqlite.org/fileformat2.html) description,
+the [virtual machine](https://www.sqlite.org/opcode.html) that runs
 prepared statements, the description of
-[how transactions work](http://www.sqlite.org/atomiccommit.html), and
-the [overview of the query planner](http://www.sqlite.org/optoverview.html).
+[how transactions work](https://www.sqlite.org/atomiccommit.html), and
+the [overview of the query planner](https://www.sqlite.org/optoverview.html).

 Years of effort have gone into optimizing SQLite, both
 for small size and high performance.  And optimizations tend to result in
@@ -353,7 +353,7 @@ hidden by also modifying the makefiles.

 ## Contacts

-The main SQLite website is [http:/sqlite.org/](http://sqlite.org/)
+The main SQLite website is [https://sqlite.org/](https://sqlite.org/)
 with geographically distributed backups at
-[http://www2.sqlite.org/](http://www2.sqlite.org) and
-[http://www3.sqlite.org/](http://www3.sqlite.org).
+[https://www2.sqlite.org/](https://www2.sqlite.org) and
+[https://www3.sqlite.org/](https://www3.sqlite.org).
@@ -2,6 +2,26 @@

 # The testrunner.tcl Script

+<ul type=none>
+  <li> 1. <a href=#overview>Overview</a>
+  <li> 2. <a href=#binary_tests>Binary Tests</a>
+<ul type=none>
+  <li> 2.1. <a href=#organization_tests>Organization of Tcl Tests</a>
+  <li> 2.2. <a href=#run_tests>Commands to Run Tests</a>
+  <li> 2.3. <a href=#binary_test_failures>Investigating Binary Test Failures</a>
+</ul>
+  <li> 3. <a href=#source_code_tests>Source Tests</a>
+<ul type=none>
+  <li> 3.1. <a href=#commands_to_run_tests>Commands to Run SQLite Tests</a>
+  <li> 3.2. <a href=#zipvfs_tests>Running ZipVFS Tests</a>
+  <li> 3.3. <a href=#source_code_test_failures>Investigating Source Code Test Failures</a>
+</ul>
+  <li> 4. <a href=#testrunner_options>Extra testrunner.tcl Options</a>
+# 4. Extra testrunner.tcl Options
+  <li> 5. <a href=#cpu_cores>Controlling CPU Core Utilization</a>
+</ul>
+
+<a name=overview></a>
 # 1. Overview

 testrunner.tcl is a Tcl script used to run multiple SQLite tests using 
@@ -44,6 +64,7 @@ Sometimes testrunner.tcl uses the [testfixture] binary that it is run with
 to run tests (see "Binary Tests" below). Sometimes it builds testfixture and
 other binaries in specific configurations to test (see "Source Tests").

+<a name=binary_tests></a>
 # 2. Binary Tests

 The commands described in this section all run various combinations of the Tcl
@@ -61,6 +82,7 @@ these tests is therefore:
 The following sub-sections describe the various options that can be
 passed to testrunner.tcl to test binary testfixture builds.

+<a name=organization_tests></a>
 ## 2.1. Organization of Tcl Tests

 Tcl tests are stored in files that match the pattern *\*.test*. They are
@@ -91,6 +113,7 @@ Running **all** tests is to run all tests in the full test set, plus a dozen
 or so permutations. The specific permutations that are run as part of "all"
 are defined in file *testrunner_data.tcl*.

+<a name=run_tests></a>
 ## 2.2. Commands to Run Tests

 To run the "veryquick" test set, use either of the following:
@@ -114,6 +137,12 @@ a specified pattern (e.g. all tests that start with "fts5"), either of:
  ./testfixture $TESTDIR/testrunner.tcl 'fts5*'
 ```

+Strictly speaking, for a test to be run the pattern must match the script
+filename, not including the directory, using the rules of Tcl's 
+\[string match\] command. Except that before the matching is done, any "%"
+characters specified as part of the pattern are transformed to "\*".
+
+
 To run "all" tests (full + permutations):

 ```
@@ -141,6 +170,7 @@ Or, if the failure occured as part of a permutation:

 TODO: An example instead of "$PERMUTATION" and $PATH\_TO\_SCRIPT?

+<a name=source_code_tests></a>
 # 3. Source Code Tests

 The commands described in this section invoke the C compiler to build 
@@ -159,7 +189,8 @@ shell that supports SQLite 3.31.1 or newer via "package require sqlite3".

 TODO: ./configure + Makefile.msc build systems.

-## Commands to Run SQLite Tests
+<a name=commands_to_run_tests></a>
+## 3.1. Commands to Run SQLite Tests

 The **mdevtest** command is equivalent to running the veryquick tests and
 the [make fuzztest] target once for each of two --enable-all builds - one 
@@ -201,7 +232,18 @@ of the specific tests run.
  tclsh $TESTDIR/testrunner.tcl release
 ```

-## Running ZipVFS Tests
+As with <a href=#source code tests>source code tests</a>, one or more patterns
+may be appended to any of the above commands (mdevtest, sdevtest or release).
+In that case only Tcl tests (no fuzz or other tests) that match the specified
+pattern are run. For example, to run the just the Tcl rtree tests in all 
+builds and configurations supported by "release":
+
+```
+  tclsh $TESTDIR/testrunner.tcl release rtree%
+```
+
+<a name=zipvfs_tests></a>
+## 3.2. Running ZipVFS Tests

 testrunner.tcl can build a zipvfs-enabled testfixture and use it to run
 tests from the Zipvfs project with the following command:
@@ -217,7 +259,8 @@ test both SQLite and Zipvfs with a single command:
  tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS mdevtest
 ```

-## Investigating Source Code Test Failures
+<a name=source_code_test_failures></a>
+## 3.3. Investigating Source Code Test Failures

 Investigating a test failure that occurs during source code testing is a
 two step process:
@@ -244,9 +287,31 @@ target to build. This may be used either to run a [make] command test directly,
 or else to build a testfixture (or testfixture.exe) binary with which to
 run a Tcl test script, as <a href=#binary_test_failures>described above</a>.

+<a name=testrunner_options></a>
+# 4. Extra testrunner.tcl Options

+The testrunner.tcl script options in this section may be used with both source
+code and binary tests.

-# 4. Controlling CPU Core Utilization
+The **--buildonly** option instructs testrunner.tcl just to build the binaries
+required by a test, not to run any actual tests. For example:
+
+```
+  # Build binaries required by release test.
+  tclsh $TESTDIR/testrunner.tcl --buildonly release"
+```
+
+The **--dryrun** option prevents testrunner.tcl from building any binaries
+or running any tests. Instead, it just writes the shell commands that it
+would normally execute into the testrunner.log file. Example:
+
+```
+  # Log the shell commmands that make up the mdevtest test.
+  tclsh $TESTDIR/testrunner.tcl --dryrun mdevtest"
+```
+
+<a name=cpu_cores></a>
+# 5. Controlling CPU Core Utilization

 When running either binary or source code tests, testrunner.tcl reports the
 number of jobs it intends to use to stdout. e.g.
@@ -277,8 +342,3 @@ testrunner.log and testrunner.db files:



-
-
-
-
-
@@ -128,7 +128,7 @@ static short streamOfConsole(FILE *pf, /* out */ PerStreamTags *ppst){
 # if CIO_WIN_WC_XLATE
 /* Define console modes for use with the Windows Console API. */
 #  define SHELL_CONI_MODE \
-  (ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT \
+  (ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | 0x80 \
  | ENABLE_QUICK_EDIT_MODE | ENABLE_EXTENDED_FLAGS | ENABLE_PROCESSED_INPUT)
 #  define SHELL_CONO_MODE (ENABLE_PROCESSED_OUTPUT | ENABLE_WRAP_AT_EOL_OUTPUT \
  | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
@@ -549,10 +549,11 @@ zSkipValidUtf8(const char *z, int nAccept, long ccm){
 #endif /*!(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE))*/

 #ifndef SQLITE_CIO_NO_TRANSLATE
+# ifdef CONSIO_SPUTB
 SQLITE_INTERNAL_LINKAGE int
 fPutbUtf8(FILE *pfO, const char *cBuf, int nAccept){
  assert(pfO!=0);
-# if CIO_WIN_WC_XLATE
+#  if CIO_WIN_WC_XLATE
  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
  PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
  if( pstReachesConsole(ppst) ){
@@ -562,12 +563,13 @@ fPutbUtf8(FILE *pfO, const char *cBuf, int nAccept){
    if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
    return rv;
  }else {
-# endif
+#  endif
    return (int)fwrite(cBuf, 1, nAccept, pfO);
-# if CIO_WIN_WC_XLATE
+#  if CIO_WIN_WC_XLATE
  }
-# endif
+#  endif
 }
+# endif

 SQLITE_INTERNAL_LINKAGE int
 oPutbUtf8(const char *cBuf, int nAccept){
@@ -607,56 +609,56 @@ ePutbUtf8(const char *cBuf, int nAccept){
 }
 # endif /* defined(CONSIO_EPUTB) */

-# if CIO_WIN_WC_XLATE
-
-
-/* Read up to two UTF-16 words.
-** Return 0 at EOF, 1 for ordinary UTF-16, or 2 for UTF-16 surrogate pair.
-** Buffer at *pwc must have at least 2 WCHAR slots remaining.
-*/
-static DWORD readConsoleCharacter(PerStreamTags *ppst, WCHAR *pwc){
-  DWORD ncr = 0;
-  BOOL bRC = ReadConsoleW(ppst->hx, pwc, 1, &ncr, 0);
-  if( !bRC || ncr == 0 ) return 0;
-  assert(ncr==1);
-  if( (pwc[0] & 0xF800)==0xD800 ){
-    DWORD ncrx;
-    /* WHAR just read leads a UTF-16 surrogate pair. Grab its mate. */
-    bRC = ReadConsoleW(ppst->hx, pwc+1, 1, &ncrx, 0);
-    if( !bRC || ncrx == 0 ) return 1;
-    ncr += ncrx;
-  }
-  return ncr;
-}
-# endif
-
 SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){
  if( pfIn==0 ) pfIn = stdin;
 # if CIO_WIN_WC_XLATE
  if( pfIn == consoleInfo.pstSetup[0].pf
      && (consoleInfo.sacSetup & SAC_InConsole)!=0 ){
 #  if CIO_WIN_WC_XLATE==1
-    WCHAR wcBuf[2];
+#   define SHELL_GULP 150 /* Count of WCHARS to be gulped at a time */
+    WCHAR wcBuf[SHELL_GULP+1];
    int lend = 0, noc = 0;
    if( ncMax > 0 ) cBuf[0] = 0;
-    while( noc < ncMax-4-1 && !lend ){
-      /* Have room for at least 1 more UTF-8 character and a 0-terminator. */
-      DWORD ncr = readConsoleCharacter(&consoleInfo.pstSetup[0], wcBuf);
-      WCHAR wc = wcBuf[0];
-      if( ncr==0 ) break;
-      assert(ncr <= 2);
-      if( wc>=L'\x80' || ncr>1 ){
-        int nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,ncr,cBuf+noc,4,0,0);
-        noc += nmb;
-      }else{
-        switch( wc ){
-        case L'\n': lend = 1; break;
-        case L'\x1a': lend = 1; continue;
-        case L'\r': continue;
-        default: break;
-        }
-        cBuf[noc++] = (char)wc;
+    while( noc < ncMax-8-1 && !lend ){
+      /* There is room for at least 2 more characters and a 0-terminator. */
+      int na = (ncMax > SHELL_GULP*4+1 + noc)? SHELL_GULP : (ncMax-1 - noc)/4;
+#   undef SHELL_GULP
+      DWORD nbr = 0;
+      BOOL bRC = ReadConsoleW(consoleInfo.pstSetup[0].hx, wcBuf, na, &nbr, 0);
+      if( bRC && nbr>0 && (wcBuf[nbr-1]&0xF800)==0xD800 ){
+        /* Last WHAR read is first of a UTF-16 surrogate pair. Grab its mate. */
+        DWORD nbrx;
+        bRC &= ReadConsoleW(consoleInfo.pstSetup[0].hx, wcBuf+nbr, 1, &nbrx, 0);
+        if( bRC ) nbr += nbrx;
      }
+      if( !bRC || (noc==0 && nbr==0) ) return 0;
+      if( nbr > 0 ){
+        int nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,nbr,0,0,0,0);
+        if( nmb != 0 && noc+nmb <= ncMax ){
+          int iseg = noc;
+          nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,nbr,cBuf+noc,nmb,0,0);
+          noc += nmb;
+          /* Fixup line-ends as coded by Windows for CR (or "Enter".)
+          ** This is done without regard for any setMode{Text,Binary}()
+          ** call that might have been done on the interactive input.
+          */
+          if( noc > 0 ){
+            if( cBuf[noc-1]=='\n' ){
+              lend = 1;
+              if( noc > 1 && cBuf[noc-2]=='\r' ) cBuf[--noc-1] = '\n';
+            }
+          }
+          /* Check for ^Z (anywhere in line) too, to act as EOF. */
+          while( iseg < noc ){
+            if( cBuf[iseg]=='\x1a' ){
+              noc = iseg; /* Chop ^Z and anything following. */
+              lend = 1; /* Counts as end of line too. */
+              break;
+            }
+            ++iseg;
+          }
+        }else break; /* Drop apparent garbage in. (Could assert.) */
+      }else break;
    }
    /* If got nothing, (after ^Z chop), must be at end-of-file. */
    if( noc > 0 ){
@@ -163,9 +163,11 @@ SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z);
 **
 ** Returns the number of accepted char values.
 */
+#ifdef CONSIO_SPUTB
 SQLITE_INTERNAL_LINKAGE int
 fPutbUtf8(FILE *pfOut, const char *cBuf, int nAccept);
 /* Like fPutbUtf8 except stream is always the designated output. */
+#endif
 SQLITE_INTERNAL_LINKAGE int
 oPutbUtf8(const char *cBuf, int nAccept);
 /* Like fPutbUtf8 except stream is always the designated error. */
@@ -179,10 +181,11 @@ ePutbUtf8(const char *cBuf, int nAccept);
 ** from the console on platforms that require same. Defers to the
 ** C library fgets() when input is not from the console. Newline
 ** translation may be done as set by set{Binary,Text}Mode(). As a
-** convenience, pfIn==NULL is treated as stdin. Argument ncMax
-** must be 5 or greater, otherwise 0 (EOF indicator) is returned.
+** convenience, pfIn==NULL is treated as stdin.
 */
 SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn);
+/* Like fGetsUtf8 except stream is always the designated input. */
+/* SQLITE_INTERNAL_LINKAGE char* iGetsUtf8(char *cBuf, int ncMax); */

 #endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */

@@ -223,10 +223,12 @@ proc main {data} {
    Fts5ExtensionApi {
      set struct [get_fts5_struct $data "^struct Fts5ExtensionApi" "^.;"]
      set map [list]
+      set lKey [list]
      foreach {k v} [get_struct_members $data] {
        if {[string match x* $k]==0} continue
-        lappend map $k "<a href=#$k>$k</a>"
+        lappend lKey $k
      }
+      foreach k [lsort -decr $lKey] { lappend map $k "<a href=#$k>$k</a>" }
      output [string map $map $struct]
    }

@@ -261,9 +261,34 @@ struct Fts5PhraseIter {
 **
 ** xPhraseNextColumn()
 **   See xPhraseFirstColumn above.
+**
+** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)
+**   This is used to access token iToken of phrase iPhrase of the current
+**   query. Before returning, output parameter *ppToken is set to point
+**   to a buffer containing the requested token, and *pnToken to the
+**   size of this buffer in bytes.
+**
+**   The output text is not a copy of the query text that specified the
+**   token. It is the output of the tokenizer module. For tokendata=1
+**   tables, this includes any embedded 0x00 and trailing data.
+**
+** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)
+**   This is used to access token iToken of phrase hit iIdx within the
+**   current row. Output variable (*ppToken) is set to point to a buffer
+**   containing the matching document token, and (*pnToken) to the size
+**   of that buffer in bytes. This API is not available if the specified 
+**   token matches a prefix query term. In that case both output variables 
+**   are always set to 0.
+**
+**   The output text is not a copy of the document text that was tokenized.
+**   It is the output of the tokenizer module. For tokendata=1 tables, this 
+**   includes any embedded 0x00 and trailing data.
+**
+**   This API can be quite slow if used with an FTS5 table created with the
+**   "detail=none" or "detail=column" option.
 */
 struct Fts5ExtensionApi {
-  int iVersion;                   /* Currently always set to 2 */
+  int iVersion;                   /* Currently always set to 3 */

  void *(*xUserData)(Fts5Context*);

@@ -298,6 +323,13 @@ struct Fts5ExtensionApi {

  int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
  void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
+
+  /* Below this point are iVersion>=3 only */
+  int (*xQueryToken)(Fts5Context*, 
+      int iPhrase, int iToken, 
+      const char **ppToken, int *pnToken
+  );
+  int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
 };

 /* 
@@ -196,6 +196,7 @@ struct Fts5Config {
  char *zContent;                 /* content table */ 
  char *zContentRowid;            /* "content_rowid=" option value */ 
  int bColumnsize;                /* "columnsize=" option value (dflt==1) */
+  int bTokendata;                 /* "tokendata=" option value (dflt==0) */
  int eDetail;                    /* FTS5_DETAIL_XXX value */
  char *zContentExprlist;
  Fts5Tokenizer *pTok;
@@ -384,17 +385,19 @@ struct Fts5IndexIter {
 /*
 ** Values used as part of the flags argument passed to IndexQuery().
 */
-#define FTS5INDEX_QUERY_PREFIX     0x0001   /* Prefix query */
-#define FTS5INDEX_QUERY_DESC       0x0002   /* Docs in descending rowid order */
-#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004   /* Do not use prefix index */
-#define FTS5INDEX_QUERY_SCAN       0x0008   /* Scan query (fts5vocab) */
+#define FTS5INDEX_QUERY_PREFIX      0x0001  /* Prefix query */
+#define FTS5INDEX_QUERY_DESC        0x0002  /* Docs in descending rowid order */
+#define FTS5INDEX_QUERY_TEST_NOIDX  0x0004  /* Do not use prefix index */
+#define FTS5INDEX_QUERY_SCAN        0x0008  /* Scan query (fts5vocab) */

 /* The following are used internally by the fts5_index.c module. They are
 ** defined here only to make it easier to avoid clashes with the flags
 ** above. */
-#define FTS5INDEX_QUERY_SKIPEMPTY  0x0010
-#define FTS5INDEX_QUERY_NOOUTPUT   0x0020
-#define FTS5INDEX_QUERY_SKIPHASH   0x0040
+#define FTS5INDEX_QUERY_SKIPEMPTY   0x0010
+#define FTS5INDEX_QUERY_NOOUTPUT    0x0020
+#define FTS5INDEX_QUERY_SKIPHASH    0x0040
+#define FTS5INDEX_QUERY_NOTOKENDATA 0x0080
+#define FTS5INDEX_QUERY_SCANONETERM 0x0100

 /*
 ** Create/destroy an Fts5Index object.
@@ -463,6 +466,10 @@ void *sqlite3Fts5StructureRef(Fts5Index*);
 void sqlite3Fts5StructureRelease(void*);
 int sqlite3Fts5StructureTest(Fts5Index*, void*);

+/*
+** Used by xInstToken():
+*/
+int sqlite3Fts5IterToken(Fts5IndexIter*, i64, int, int, const char**, int*);

 /*
 ** Insert or remove data to or from the index. Each time a document is 
@@ -540,6 +547,13 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin);
 int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid);

+void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*);
+
+/* Used to populate hash tables for xInstToken in detail=none/column mode. */
+int sqlite3Fts5IndexIterWriteTokendata(
+    Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff
+);
+
 /*
 ** End of interface to code in fts5_index.c.
 **************************************************************************/
@@ -645,6 +659,7 @@ void sqlite3Fts5HashScanNext(Fts5Hash*);
 int sqlite3Fts5HashScanEof(Fts5Hash*);
 void sqlite3Fts5HashScanEntry(Fts5Hash *,
  const char **pzTerm,            /* OUT: term (nul-terminated) */
+  int *pnTerm,                    /* OUT: Size of term in bytes */
  const u8 **ppDoclist,           /* OUT: pointer to doclist */
  int *pnDoclist                  /* OUT: size of doclist in bytes */
 );
@@ -771,6 +786,10 @@ int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);

 int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);

+int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*);
+int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*);
+void sqlite3Fts5ExprClearTokens(Fts5Expr*);
+
 /*******************************************
 ** The fts5_expr.c API above this point is used by the other hand-written
 ** C code in this module. The interfaces below this point are called by
@@ -398,6 +398,16 @@ static int fts5ConfigParseSpecial(
    return rc;
  }

+  if( sqlite3_strnicmp("tokendata", zCmd, nCmd)==0 ){
+    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
+      *pzErr = sqlite3_mprintf("malformed tokendata=... directive");
+      rc = SQLITE_ERROR;
+    }else{
+      pConfig->bTokendata = (zArg[0]=='1');
+    }
+    return rc;
+  }
+
  *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
  return SQLITE_ERROR;
 }
@@ -100,7 +100,9 @@ struct Fts5ExprNode {
 struct Fts5ExprTerm {
  u8 bPrefix;                     /* True for a prefix term */
  u8 bFirst;                      /* True if token must be first in column */
-  char *zTerm;                    /* nul-terminated term */
+  char *pTerm;                    /* Term data */
+  int nQueryTerm;                 /* Effective size of term in bytes */
+  int nFullTerm;                  /* Size of term in bytes incl. tokendata */
  Fts5IndexIter *pIter;           /* Iterator for this term */
  Fts5ExprTerm *pSynonym;         /* Pointer to first in list of synonyms */
 };
@@ -967,7 +969,7 @@ static int fts5ExprNearInitAll(
            p->pIter = 0;
          }
          rc = sqlite3Fts5IndexQuery(
-              pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
+              pExpr->pIndex, p->pTerm, p->nQueryTerm,
              (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
              (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
              pNear->pColset,
@@ -1604,7 +1606,7 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
      Fts5ExprTerm *pSyn;
      Fts5ExprTerm *pNext;
      Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
-      sqlite3_free(pTerm->zTerm);
+      sqlite3_free(pTerm->pTerm);
      sqlite3Fts5IterClose(pTerm->pIter);
      for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){
        pNext = pSyn->pSynonym;
@@ -1702,6 +1704,7 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
 typedef struct TokenCtx TokenCtx;
 struct TokenCtx {
  Fts5ExprPhrase *pPhrase;
+  Fts5Config *pConfig;
  int rc;
 };

@@ -1735,8 +1738,10 @@ static int fts5ParseTokenize(
      rc = SQLITE_NOMEM;
    }else{
      memset(pSyn, 0, (size_t)nByte);
-      pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
-      memcpy(pSyn->zTerm, pToken, nToken);
+      pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
+      pSyn->nFullTerm = pSyn->nQueryTerm = nToken;
+      if( pCtx->pConfig->bTokendata ) pSyn->nQueryTerm = strlen(pSyn->pTerm);
+      memcpy(pSyn->pTerm, pToken, nToken);
      pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
      pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn;
    }
@@ -1761,7 +1766,11 @@ static int fts5ParseTokenize(
    if( rc==SQLITE_OK ){
      pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
      memset(pTerm, 0, sizeof(Fts5ExprTerm));
-      pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
+      pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
+      pTerm->nFullTerm = pTerm->nQueryTerm = nToken;
+      if( pCtx->pConfig->bTokendata && rc==SQLITE_OK ){ 
+        pTerm->nQueryTerm = strlen(pTerm->pTerm);
+      }
    }
  }

@@ -1828,6 +1837,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(

  memset(&sCtx, 0, sizeof(TokenCtx));
  sCtx.pPhrase = pAppend;
+  sCtx.pConfig = pConfig;

  rc = fts5ParseStringFromToken(pToken, &z);
  if( rc==SQLITE_OK ){
@@ -1877,8 +1887,7 @@ int sqlite3Fts5ExprClonePhrase(
  int rc = SQLITE_OK;             /* Return code */
  Fts5ExprPhrase *pOrig;          /* The phrase extracted from pExpr */
  Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
-  TokenCtx sCtx = {0,0};          /* Context object for fts5ParseTokenize */
-
+  TokenCtx sCtx = {0,0,0};        /* Context object for fts5ParseTokenize */
  pOrig = pExpr->apExprPhrase[iPhrase];
  pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
  if( rc==SQLITE_OK ){
@@ -1909,13 +1918,12 @@ int sqlite3Fts5ExprClonePhrase(

  if( pOrig->nTerm ){
    int i;                          /* Used to iterate through phrase terms */
+    sCtx.pConfig = pExpr->pConfig;
    for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
      int tflags = 0;
      Fts5ExprTerm *p;
      for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
-        const char *zTerm = p->zTerm;
-        rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
-            0, 0);
+        rc = fts5ParseTokenize((void*)&sCtx, tflags, p->pTerm,p->nFullTerm,0,0);
        tflags = FTS5_TOKEN_COLOCATED;
      }
      if( rc==SQLITE_OK ){
@@ -2296,11 +2304,13 @@ static Fts5ExprNode *fts5ParsePhraseToAnd(
        if( parseGrowPhraseArray(pParse) ){
          fts5ExprPhraseFree(pPhrase);
        }else{
+          Fts5ExprTerm *p = &pNear->apPhrase[0]->aTerm[ii];
+          Fts5ExprTerm *pTo = &pPhrase->aTerm[0];
          pParse->apPhrase[pParse->nPhrase++] = pPhrase;
          pPhrase->nTerm = 1;
-          pPhrase->aTerm[0].zTerm = sqlite3Fts5Strndup(
-              &pParse->rc, pNear->apPhrase[0]->aTerm[ii].zTerm, -1
-          );
+          pTo->pTerm = sqlite3Fts5Strndup(&pParse->rc, p->pTerm, p->nFullTerm);
+          pTo->nQueryTerm = p->nQueryTerm;
+          pTo->nFullTerm = p->nFullTerm;
          pRet->apChild[ii] = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 
              0, 0, sqlite3Fts5ParseNearset(pParse, 0, pPhrase)
          );
@@ -2485,16 +2495,17 @@ static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){

  /* Determine the maximum amount of space required. */
  for(p=pTerm; p; p=p->pSynonym){
-    nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2;
+    nByte += pTerm->nQueryTerm * 2 + 3 + 2;
  }
  zQuoted = sqlite3_malloc64(nByte);

  if( zQuoted ){
    int i = 0;
    for(p=pTerm; p; p=p->pSynonym){
-      char *zIn = p->zTerm;
+      char *zIn = p->pTerm;
+      char *zEnd = &zIn[p->nQueryTerm];
      zQuoted[i++] = '"';
-      while( *zIn ){
+      while( zIn<zEnd ){
        if( *zIn=='"' ) zQuoted[i++] = '"';
        zQuoted[i++] = *zIn++;
      }
@@ -2572,8 +2583,10 @@ static char *fts5ExprPrintTcl(

      zRet = fts5PrintfAppend(zRet, " {");
      for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){
-        char *zTerm = pPhrase->aTerm[iTerm].zTerm;
-        zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm);
+        Fts5ExprTerm *p = &pPhrase->aTerm[iTerm];
+        zRet = fts5PrintfAppend(zRet, "%s%.*s", iTerm==0?"":" ", 
+            p->nQueryTerm, p->pTerm
+        );
        if( pPhrase->aTerm[iTerm].bPrefix ){
          zRet = fts5PrintfAppend(zRet, "*");
        }
@@ -2974,6 +2987,17 @@ static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){
  return 0;
 }

+/*
+** pToken is a buffer nToken bytes in size that may or may not contain
+** an embedded 0x00 byte. If it does, return the number of bytes in
+** the buffer before the 0x00. If it does not, return nToken.
+*/
+static int fts5QueryTerm(const char *pToken, int nToken){
+  int ii;
+  for(ii=0; ii<nToken && pToken[ii]; ii++){}
+  return ii;
+}
+
 static int fts5ExprPopulatePoslistsCb(
  void *pCtx,                /* Copy of 2nd argument to xTokenize() */
  int tflags,                /* Mask of FTS5_TOKEN_* flags */
@@ -2985,22 +3009,33 @@ static int fts5ExprPopulatePoslistsCb(
  Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
  Fts5Expr *pExpr = p->pExpr;
  int i;
+  int nQuery = nToken;
+  i64 iRowid = pExpr->pRoot->iRowid;

  UNUSED_PARAM2(iUnused1, iUnused2);

-  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
+  if( nQuery>FTS5_MAX_TOKEN_SIZE ) nQuery = FTS5_MAX_TOKEN_SIZE;
+  if( pExpr->pConfig->bTokendata ){
+    nQuery = fts5QueryTerm(pToken, nQuery);
+  }
  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
  for(i=0; i<pExpr->nPhrase; i++){
-    Fts5ExprTerm *pTerm;
+    Fts5ExprTerm *pT;
    if( p->aPopulator[i].bOk==0 ) continue;
-    for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
-      int nTerm = (int)strlen(pTerm->zTerm);
-      if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
-       && memcmp(pTerm->zTerm, pToken, nTerm)==0
+    for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){
+      if( (pT->nQueryTerm==nQuery || (pT->nQueryTerm<nQuery && pT->bPrefix))
+       && memcmp(pT->pTerm, pToken, pT->nQueryTerm)==0
      ){
        int rc = sqlite3Fts5PoslistWriterAppend(
            &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
        );
+        if( rc==SQLITE_OK && pExpr->pConfig->bTokendata && !pT->bPrefix ){
+          int iCol = p->iOff>>32;
+          int iTokOff = p->iOff & 0x7FFFFFFF;
+          rc = sqlite3Fts5IndexIterWriteTokendata(
+              pT->pIter, pToken, nToken, iRowid, iCol, iTokOff
+          );
+        }
        if( rc ) return rc;
        break;
      }
@@ -3135,3 +3170,81 @@ int sqlite3Fts5ExprPhraseCollist(

  return rc;
 }
+
+/*
+** Does the work of the fts5_api.xQueryToken() API method.
+*/
+int sqlite3Fts5ExprQueryToken(
+  Fts5Expr *pExpr, 
+  int iPhrase, 
+  int iToken, 
+  const char **ppOut, 
+  int *pnOut
+){
+  Fts5ExprPhrase *pPhrase = 0;
+
+  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
+    return SQLITE_RANGE;
+  }
+  pPhrase = pExpr->apExprPhrase[iPhrase];
+  if( iToken<0 || iToken>=pPhrase->nTerm ){
+    return SQLITE_RANGE;
+  }
+
+  *ppOut = pPhrase->aTerm[iToken].pTerm;
+  *pnOut = pPhrase->aTerm[iToken].nFullTerm;
+  return SQLITE_OK;
+}
+
+/*
+** Does the work of the fts5_api.xInstToken() API method.
+*/
+int sqlite3Fts5ExprInstToken(
+  Fts5Expr *pExpr, 
+  i64 iRowid,
+  int iPhrase, 
+  int iCol, 
+  int iOff, 
+  int iToken, 
+  const char **ppOut, 
+  int *pnOut
+){
+  Fts5ExprPhrase *pPhrase = 0;
+  Fts5ExprTerm *pTerm = 0;
+  int rc = SQLITE_OK;
+
+  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
+    return SQLITE_RANGE;
+  }
+  pPhrase = pExpr->apExprPhrase[iPhrase];
+  if( iToken<0 || iToken>=pPhrase->nTerm ){
+    return SQLITE_RANGE;
+  }
+  pTerm = &pPhrase->aTerm[iToken];
+  if( pTerm->bPrefix==0 ){
+    if( pExpr->pConfig->bTokendata ){
+      rc = sqlite3Fts5IterToken(
+          pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut
+      );
+    }else{
+      *ppOut = pTerm->pTerm;
+      *pnOut = pTerm->nFullTerm;
+    }
+  }
+  return rc;
+}
+
+/*
+** Clear the token mappings for all Fts5IndexIter objects mannaged by 
+** the expression passed as the only argument.
+*/
+void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){
+  int ii;
+  for(ii=0; ii<pExpr->nPhrase; ii++){
+    Fts5ExprTerm *pT;
+    for(pT=&pExpr->apExprPhrase[ii]->aTerm[0]; pT; pT=pT->pSynonym){
+      sqlite3Fts5IndexIterClearTokendata(pT->pIter);
+    }
+  }
+}
+
@@ -36,10 +36,15 @@ struct Fts5Hash {

 /*
 ** Each entry in the hash table is represented by an object of the 
-** following type. Each object, its key (a nul-terminated string) and 
-** its current data are stored in a single memory allocation. The 
-** key immediately follows the object in memory. The position list
-** data immediately follows the key data in memory.
+** following type. Each object, its key, and its current data are stored 
+** in a single memory allocation. The key immediately follows the object 
+** in memory. The position list data immediately follows the key data 
+** in memory.
+**
+** The key is Fts5HashEntry.nKey bytes in size. It consists of a single
+** byte identifying the index (either the main term index or a prefix-index),
+** followed by the term data. For example: "0token". There is no 
+** nul-terminator - in this case nKey=6.
 **
 ** The data that follows the key is in a similar, but not identical format
 ** to the doclist data stored in the database. It is:
@@ -174,8 +179,7 @@ static int fts5HashResize(Fts5Hash *pHash){
      unsigned int iHash;
      Fts5HashEntry *p = apOld[i];
      apOld[i] = p->pHashNext;
-      iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p),
-                          (int)strlen(fts5EntryKey(p)));
+      iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), p->nKey);
      p->pHashNext = apNew[iHash];
      apNew[iHash] = p;
    }
@@ -259,7 +263,7 @@ int sqlite3Fts5HashWrite(
  for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
    char *zKey = fts5EntryKey(p);
    if( zKey[0]==bByte 
-     && p->nKey==nToken
+     && p->nKey==nToken+1
     && memcmp(&zKey[1], pToken, nToken)==0 
    ){
      break;
@@ -289,9 +293,9 @@ int sqlite3Fts5HashWrite(
    zKey[0] = bByte;
    memcpy(&zKey[1], pToken, nToken);
    assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) );
-    p->nKey = nToken;
+    p->nKey = nToken+1;
    zKey[nToken+1] = '\0';
-    p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry);
+    p->nData = nToken+1 + sizeof(Fts5HashEntry);
    p->pHashNext = pHash->aSlot[iHash];
    pHash->aSlot[iHash] = p;
    pHash->nEntry++;
@@ -408,12 +412,17 @@ static Fts5HashEntry *fts5HashEntryMerge(
      *ppOut = p1;
      p1 = 0;
    }else{
-      int i = 0;
      char *zKey1 = fts5EntryKey(p1);
      char *zKey2 = fts5EntryKey(p2);
-      while( zKey1[i]==zKey2[i] ) i++;
+      int nMin = MIN(p1->nKey, p2->nKey);

-      if( ((u8)zKey1[i])>((u8)zKey2[i]) ){
+      int cmp = memcmp(zKey1, zKey2, nMin);
+      if( cmp==0 ){
+        cmp = p1->nKey - p2->nKey;
+      }
+      assert( cmp!=0 );
+
+      if( cmp>0 ){
        /* p2 is smaller */
        *ppOut = p2;
        ppOut = &p2->pScanNext;
@@ -455,7 +464,7 @@ static int fts5HashEntrySort(
    Fts5HashEntry *pIter;
    for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
      if( pTerm==0 
-       || (pIter->nKey+1>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm))
+       || (pIter->nKey>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm))
      ){
        Fts5HashEntry *pEntry = pIter;
        pEntry->pScanNext = 0;
@@ -494,12 +503,11 @@ int sqlite3Fts5HashQuery(

  for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
    zKey = fts5EntryKey(p);
-    assert( p->nKey+1==(int)strlen(zKey) );
-    if( nTerm==p->nKey+1 && memcmp(zKey, pTerm, nTerm)==0 ) break;
+    if( nTerm==p->nKey && memcmp(zKey, pTerm, nTerm)==0 ) break;
  }

  if( p ){
-    int nHashPre = sizeof(Fts5HashEntry) + nTerm + 1;
+    int nHashPre = sizeof(Fts5HashEntry) + nTerm;
    int nList = p->nData - nHashPre;
    u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10));
    if( pRet ){
@@ -560,19 +568,22 @@ int sqlite3Fts5HashScanEof(Fts5Hash *p){
 void sqlite3Fts5HashScanEntry(
  Fts5Hash *pHash,
  const char **pzTerm,            /* OUT: term (nul-terminated) */
+  int *pnTerm,                    /* OUT: Size of term in bytes */
  const u8 **ppDoclist,           /* OUT: pointer to doclist */
  int *pnDoclist                  /* OUT: size of doclist in bytes */
 ){
  Fts5HashEntry *p;
  if( (p = pHash->pScan) ){
    char *zKey = fts5EntryKey(p);
-    int nTerm = (int)strlen(zKey);
+    int nTerm = p->nKey;
    fts5HashAddPoslistSize(pHash, p, 0);
    *pzTerm = zKey;
-    *ppDoclist = (const u8*)&zKey[nTerm+1];
-    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
+    *pnTerm = nTerm;
+    *ppDoclist = (const u8*)&zKey[nTerm];
+    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm);
  }else{
    *pzTerm = 0;
+    *pnTerm = 0;
    *ppDoclist = 0;
    *pnDoclist = 0;
  }
@@ -118,7 +118,7 @@ struct Fts5FullTable {
  Fts5Global *pGlobal;            /* Global (connection wide) data */
  Fts5Cursor *pSortCsr;           /* Sort data from this cursor */
  int iSavepoint;                 /* Successful xSavepoint()+1 */
-  int bInSavepoint;
+  
 #ifdef SQLITE_DEBUG
  struct Fts5TransactionState ts;
 #endif
@@ -656,12 +656,15 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
  }
  idxStr[iIdxStr] = '\0';

-  /* Set idxFlags flags for the ORDER BY clause */
+  /* Set idxFlags flags for the ORDER BY clause
+  **
+  ** Note that tokendata=1 tables cannot currently handle "ORDER BY rowid DESC".
+  */
  if( pInfo->nOrderBy==1 ){
    int iSort = pInfo->aOrderBy[0].iColumn;
    if( iSort==(pConfig->nCol+1) && bSeenMatch ){
      idxFlags |= FTS5_BI_ORDER_RANK;
-    }else if( iSort==-1 ){
+    }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){
      idxFlags |= FTS5_BI_ORDER_ROWID;
    }
    if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){
@@ -913,6 +916,16 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
  );
  assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) );

+  /* If this cursor uses FTS5_PLAN_MATCH and this is a tokendata=1 table,
+  ** clear any token mappings accumulated at the fts5_index.c level. In
+  ** other cases, specifically FTS5_PLAN_SOURCE and FTS5_PLAN_SORTED_MATCH,
+  ** we need to retain the mappings for the entire query.  */
+  if( pCsr->ePlan==FTS5_PLAN_MATCH 
+   && ((Fts5Table*)pCursor->pVtab)->pConfig->bTokendata 
+  ){
+    sqlite3Fts5ExprClearTokens(pCsr->pExpr);
+  }
+
  if( pCsr->ePlan<3 ){
    int bSkip = 0;
    if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
@@ -2063,12 +2076,6 @@ static int fts5ApiInst(
  ){
    if( iIdx<0 || iIdx>=pCsr->nInstCount ){
      rc = SQLITE_RANGE;
-#if 0
-    }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){
-      *piPhrase = pCsr->aInst[iIdx*3];
-      *piCol = pCsr->aInst[iIdx*3 + 2];
-      *piOff = -1;
-#endif
    }else{
      *piPhrase = pCsr->aInst[iIdx*3];
      *piCol = pCsr->aInst[iIdx*3 + 1];
@@ -2323,13 +2330,56 @@ static int fts5ApiPhraseFirstColumn(
  return rc;
 }

+/*
+** xQueryToken() API implemenetation.
+*/
+static int fts5ApiQueryToken(
+  Fts5Context* pCtx, 
+  int iPhrase, 
+  int iToken, 
+  const char **ppOut, 
+  int *pnOut
+){
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
+  return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut);
+}
+
+/*
+** xInstToken() API implemenetation.
+*/
+static int fts5ApiInstToken(
+  Fts5Context *pCtx,
+  int iIdx, 
+  int iToken,
+  const char **ppOut, int *pnOut
+){
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
+  int rc = SQLITE_OK;
+  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 
+   || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) 
+  ){
+    if( iIdx<0 || iIdx>=pCsr->nInstCount ){
+      rc = SQLITE_RANGE;
+    }else{
+      int iPhrase = pCsr->aInst[iIdx*3];
+      int iCol = pCsr->aInst[iIdx*3 + 1];
+      int iOff = pCsr->aInst[iIdx*3 + 2];
+      i64 iRowid = fts5CursorRowid(pCsr);
+      rc = sqlite3Fts5ExprInstToken(
+          pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut
+      );
+    }
+  }
+  return rc;
+}
+

 static int fts5ApiQueryPhrase(Fts5Context*, int, void*, 
    int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
 );

 static const Fts5ExtensionApi sFts5Api = {
-  2,                            /* iVersion */
+  3,                            /* iVersion */
  fts5ApiUserData,
  fts5ApiColumnCount,
  fts5ApiRowCount,
@@ -2349,6 +2399,8 @@ static const Fts5ExtensionApi sFts5Api = {
  fts5ApiPhraseNext,
  fts5ApiPhraseFirstColumn,
  fts5ApiPhraseNextColumn,
+  fts5ApiQueryToken,
+  fts5ApiInstToken
 };

 /*
@@ -2615,9 +2667,7 @@ static int fts5RenameMethod(
 ){
  int rc;
  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
-  pTab->bInSavepoint = 1;
  rc = sqlite3Fts5StorageRename(pTab->pStorage, zName);
-  pTab->bInSavepoint = 0;
  return rc;
 }

@@ -2634,26 +2684,12 @@ int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
 static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
  int rc = SQLITE_OK;
-  char *zSql = 0;
+
  fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
-
-  if( pTab->bInSavepoint==0 ){
-    zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')",
-        pTab->p.pConfig->zDb, pTab->p.pConfig->zName, pTab->p.pConfig->zName
-    );
-    if( zSql ){
-      pTab->bInSavepoint = 1;
-      rc = sqlite3_exec(pTab->p.pConfig->db, zSql, 0, 0, 0);
-      pTab->bInSavepoint = 0;
-      sqlite3_free(zSql);
-    }else{
-      rc = SQLITE_NOMEM;
-    }
-    if( rc==SQLITE_OK ){
-      pTab->iSavepoint = iSavepoint+1;
-    }
+  rc = sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
+  if( rc==SQLITE_OK ){
+    pTab->iSavepoint = iSavepoint+1;
  }
-
  return rc;
 }

@@ -2914,7 +2950,7 @@ static int fts5ShadowName(const char *zName){
 ** if anything is found amiss.  Return a NULL pointer if everything is
 ** OK.
 */
-static int fts5Integrity(
+static int fts5IntegrityMethod(
  sqlite3_vtab *pVtab,    /* the FTS5 virtual table to check */
  const char *zSchema,    /* Name of schema in which this table lives */
  const char *zTabname,   /* Name of the table itself */
@@ -2972,7 +3008,7 @@ static int fts5Init(sqlite3 *db){
    /* xRelease      */ fts5ReleaseMethod,
    /* xRollbackTo   */ fts5RollbackToMethod,
    /* xShadowName   */ fts5ShadowName,
-    /* xIntegrity    */ fts5Integrity
+    /* xIntegrity    */ fts5IntegrityMethod
  };

  int rc;
@@ -244,6 +244,9 @@ static int SQLITE_TCLAPI xF5tApi(
    { "xGetAuxdataInt",    1, "CLEAR" },              /* 15 */
    { "xPhraseForeach",    4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */
    { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */
+
+    { "xQueryToken",       2, "IPHRASE ITERM" },      /* 18 */
+    { "xInstToken",        2, "IDX ITERM" },          /* 19 */
    { 0, 0, 0}
  };

@@ -500,6 +503,38 @@ static int SQLITE_TCLAPI xF5tApi(
      break;
    }

+    CASE(18, "xQueryToken") {
+      const char *pTerm = 0;
+      int nTerm = 0;
+      int iPhrase = 0;
+      int iTerm = 0;
+
+      if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
+      if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
+      rc = p->pApi->xQueryToken(p->pFts, iPhrase, iTerm, &pTerm, &nTerm);
+      if( rc==SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
+      }
+
+      break;
+    }
+
+    CASE(19, "xInstToken") {
+      const char *pTerm = 0;
+      int nTerm = 0;
+      int iIdx = 0;
+      int iTerm = 0;
+
+      if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ) return TCL_ERROR;
+      if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
+      rc = p->pApi->xInstToken(p->pFts, iIdx, iTerm, &pTerm, &nTerm);
+      if( rc==SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
+      }
+
+      break;
+    }
+
    default: 
      assert( 0 );
      break;
@@ -1117,6 +1152,176 @@ static int SQLITE_TCLAPI f5tRegisterTok(
  return TCL_OK;
 }

+typedef struct OriginTextCtx OriginTextCtx;
+struct OriginTextCtx {
+  sqlite3 *db;
+  fts5_api *pApi;
+};
+
+typedef struct OriginTextTokenizer OriginTextTokenizer;
+struct OriginTextTokenizer {
+  Fts5Tokenizer *pTok;            /* Underlying tokenizer object */
+  fts5_tokenizer tokapi;          /* API implementation for pTok */
+};
+
+/*
+** Delete the OriginTextCtx object indicated by the only argument.
+*/
+static void f5tOrigintextTokenizerDelete(void *pCtx){
+  OriginTextCtx *p = (OriginTextCtx*)pCtx;
+  ckfree(p);
+}
+
+static int f5tOrigintextCreate(
+  void *pCtx, 
+  const char **azArg, 
+  int nArg, 
+  Fts5Tokenizer **ppOut
+){
+  OriginTextCtx *p = (OriginTextCtx*)pCtx;
+  OriginTextTokenizer *pTok = 0;
+  void *pTokCtx = 0;
+  int rc = SQLITE_OK;
+
+  pTok = (OriginTextTokenizer*)sqlite3_malloc(sizeof(OriginTextTokenizer));
+  if( pTok==0 ){
+    rc = SQLITE_NOMEM;
+  }else if( nArg<1 ){
+    rc = SQLITE_ERROR;
+  }else{
+    /* Locate the underlying tokenizer */
+    rc = p->pApi->xFindTokenizer(p->pApi, azArg[0], &pTokCtx, &pTok->tokapi);
+  }
+
+  /* Create the new tokenizer instance */
+  if( rc==SQLITE_OK ){
+    rc = pTok->tokapi.xCreate(pTokCtx, &azArg[1], nArg-1, &pTok->pTok);
+  }
+
+  if( rc!=SQLITE_OK ){
+    sqlite3_free(pTok);
+    pTok = 0;
+  }
+  *ppOut = (Fts5Tokenizer*)pTok;
+  return rc;
+}
+
+static void f5tOrigintextDelete(Fts5Tokenizer *pTokenizer){
+  OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
+  if( p->pTok ){
+    p->tokapi.xDelete(p->pTok);
+  }
+  sqlite3_free(p);
+}
+
+typedef struct OriginTextCb OriginTextCb;
+struct OriginTextCb {
+  void *pCtx;
+  const char *pText; 
+  int nText;
+  int (*xToken)(void *, int, const char *, int, int, int);
+
+  char *aBuf;                     /* Buffer to use */
+  int nBuf;                       /* Allocated size of aBuf[] */
+};
+
+static int xOriginToken(
+  void *pCtx,                     /* Copy of 2nd argument to xTokenize() */
+  int tflags,                     /* Mask of FTS5_TOKEN_* flags */
+  const char *pToken,             /* Pointer to buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStart,                     /* Byte offset of token within input text */
+  int iEnd                        /* Byte offset of end of token within input */
+){
+  OriginTextCb *p = (OriginTextCb*)pCtx;
+  int ret = 0;
+
+  if( nToken==(iEnd-iStart) && 0==memcmp(pToken, &p->pText[iStart], nToken) ){
+    /* Token exactly matches document text. Pass it through as is. */
+    ret = p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd);
+  }else{
+    int nReq = nToken + 1 + (iEnd-iStart);
+    if( nReq>p->nBuf ){
+      sqlite3_free(p->aBuf);
+      p->aBuf = sqlite3_malloc(nReq*2);
+      if( p->aBuf==0 ) return SQLITE_NOMEM;
+      p->nBuf = nReq*2;
+    }
+
+    memcpy(p->aBuf, pToken, nToken);
+    p->aBuf[nToken] = '\0';
+    memcpy(&p->aBuf[nToken+1], &p->pText[iStart], iEnd-iStart);
+    ret = p->xToken(p->pCtx, tflags, p->aBuf, nReq, iStart, iEnd);
+  }
+
+  return ret;
+}
+
+
+static int f5tOrigintextTokenize(
+  Fts5Tokenizer *pTokenizer, 
+  void *pCtx,
+  int flags,                      /* Mask of FTS5_TOKENIZE_* flags */
+  const char *pText, int nText, 
+  int (*xToken)(void *, int, const char *, int, int, int)
+){
+  OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
+  OriginTextCb cb;
+  int ret;
+
+  memset(&cb, 0, sizeof(cb));
+  cb.pCtx = pCtx;
+  cb.pText = pText;
+  cb.nText = nText;
+  cb.xToken = xToken;
+
+  ret = p->tokapi.xTokenize(p->pTok,(void*)&cb,flags,pText,nText,xOriginToken);
+  sqlite3_free(cb.aBuf);
+  return ret;
+}
+
+/*
+**      sqlite3_fts5_register_origintext DB
+**
+** Description...
+*/
+static int SQLITE_TCLAPI f5tRegisterOriginText(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3 *db = 0;
+  fts5_api *pApi = 0;
+  int rc;
+  fts5_tokenizer tok = {0, 0, 0};
+  OriginTextCtx *pCtx = 0;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }
+  if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR;
+
+  pCtx = (OriginTextCtx*)ckalloc(sizeof(OriginTextCtx));
+  pCtx->db = db;
+  pCtx->pApi = pApi;
+
+  tok.xCreate = f5tOrigintextCreate;
+  tok.xDelete = f5tOrigintextDelete;
+  tok.xTokenize = f5tOrigintextTokenize;
+  rc = pApi->xCreateTokenizer(
+      pApi, "origintext", (void*)pCtx, &tok, f5tOrigintextTokenizerDelete
+  );
+
+  Tcl_ResetResult(interp);
+  if( rc!=SQLITE_OK ){
+    Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
+    return TCL_ERROR;
+  }
+  return TCL_OK;
+}
+
 /*
 ** Entry point.
 */
@@ -1133,7 +1338,8 @@ int Fts5tcl_Init(Tcl_Interp *interp){
    { "sqlite3_fts5_may_be_corrupt",     f5tMayBeCorrupt, 0 },
    { "sqlite3_fts5_token_hash",         f5tTokenHash, 0 },
    { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 },
-    { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 }
+    { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 },
+    { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 }
  };
  int i;
  F5tTokenizerContext *pContext;
@@ -629,7 +629,7 @@ static int fts5VocabFilterMethod(
  if( pEq ){
    zTerm = (const char *)sqlite3_value_text(pEq);
    nTerm = sqlite3_value_bytes(pEq);
-    f = 0;
+    f = FTS5INDEX_QUERY_NOTOKENDATA;
  }else{
    if( pGe ){
      zTerm = (const char *)sqlite3_value_text(pGe);
@@ -438,6 +438,20 @@ proc detail_is_none {} { detail_check ; expr {$::detail == "none"} }
 proc detail_is_col {}  { detail_check ; expr {$::detail == "col" } }
 proc detail_is_full {} { detail_check ; expr {$::detail == "full"} }

+proc foreach_tokenizer_mode {prefix script} {
+  set saved $::testprefix
+  foreach {d mapping} {
+    ""              {}
+    "-origintext"   {, tokenize="origintext unicode61", tokendata=1}
+  } {
+    set s [string map [list %TOKENIZER% $mapping] $script]
+    set ::testprefix "$prefix$d"
+    reset_db
+    sqlite3_fts5_register_origintext db
+    uplevel $s
+  }
+  set ::testprefix $saved
+}

 #-------------------------------------------------------------------------
 # Convert a poslist of the type returned by fts5_test_poslist() to a 
@@ -22,6 +22,7 @@ ifcapable !fts5 {
 }

 foreach_detail_mode $::testprefix {
+foreach_tokenizer_mode $::testprefix {

 do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
@@ -44,7 +45,7 @@ do_execsql_test 1.1 {
 #

 do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
 }
 do_execsql_test 2.1 {
  INSERT INTO t1 VALUES('a b c', 'd e f');
@@ -73,8 +74,9 @@ do_execsql_test 2.4 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 3.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
 }
 foreach {i x y} {
   1  {g f d b f} {h h e i a}
@@ -97,8 +99,9 @@ foreach {i x y} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 4.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }
 foreach {i x y} {
@@ -121,8 +124,9 @@ foreach {i x y} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 5.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }
 foreach {i x y} {
@@ -145,8 +149,9 @@ foreach {i x y} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 6.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }

@@ -181,6 +186,7 @@ do_execsql_test 6.6 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 expr srand(0)
 do_execsql_test 7.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x,y,z);
@@ -222,6 +228,7 @@ for {set i 1} {$i <= 10} {incr i} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 8.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3");
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
@@ -236,6 +243,7 @@ do_execsql_test 8.1 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db

 expr srand(0)

@@ -280,8 +288,9 @@ for {set i 1} {$i <= 10} {incr i} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 10.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
 }
 set d10 {
   1  {g f d b f} {h h e i a}
@@ -314,19 +323,19 @@ do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
 #-------------------------------------------------------------------------
 #
 do_catchsql_test 11.1 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL% %TOKENIZER%);
 } {1 {reserved fts5 column name: rank}}
 do_catchsql_test 11.2 {
-  CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL% %TOKENIZER%);
 } {1 {reserved fts5 table name: rank}}
 do_catchsql_test 11.3 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL% %TOKENIZER%);
 } {1 {reserved fts5 column name: rowid}}

 #-------------------------------------------------------------------------
 #
 do_execsql_test 12.1 {
-  CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
 } {}

 do_catchsql_test 12.2 {
@@ -341,8 +350,9 @@ do_test 12.3 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 13.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
 } {}

@@ -365,8 +375,9 @@ do_execsql_test 13.6 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 14.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  WITH d(x,y) AS (
    SELECT NULL, 'xyz xyz xyz xyz xyz xyz'
@@ -449,8 +460,9 @@ do_catchsql_test 16.2 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 17.1 {
-  CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO b2 VALUES('a');
  INSERT INTO b2 VALUES('b');
  INSERT INTO b2 VALUES('c');
@@ -466,8 +478,9 @@ do_test 17.2 {

 if {[string match n* %DETAIL%]==0} {
  reset_db
+  sqlite3_fts5_register_origintext db
  do_execsql_test 17.3 {
-    CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%);
+    CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
    INSERT INTO c2 VALUES('x x x', 'x x x');
    SELECT rowid FROM c2 WHERE c2 MATCH 'y:x';
  } {1}
@@ -476,8 +489,9 @@ if {[string match n* %DETAIL%]==0} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 17.1 {
-  CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO uio VALUES(NULL);
  INSERT INTO uio SELECT NULL FROM uio;
  INSERT INTO uio SELECT NULL FROM uio;
@@ -524,8 +538,8 @@ do_execsql_test 17.9 {
 #--------------------------------------------------------------------
 #
 do_execsql_test 18.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
-  CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1 VALUES('abc*', NULL);
  INSERT INTO t2 VALUES(1, 'abcdefg');
 }
@@ -540,8 +554,9 @@ do_execsql_test 18.3 {
 # fts5 table in the temp schema.
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 19.0 {
-  CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1 VALUES('x y z');
  INSERT INTO t1 VALUES('w x 1');
  SELECT rowid FROM t1 WHERE t1 MATCH 'x';
@@ -551,8 +566,9 @@ do_execsql_test 19.0 {
 # Test that 6 and 7 byte varints can be read.
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 20.0 {
-  CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL% %TOKENIZER%);
 }
 set ::ids [list \
  0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43]
@@ -570,7 +586,7 @@ do_test 20.1 {
 #
 do_execsql_test 21.0 {
  CREATE TEMP TABLE t8(a, b);
-  CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL% %TOKENIZER%);
 }

 do_execsql_test 21.1 {
@@ -581,7 +597,7 @@ do_execsql_test 21.1 {
 }

 do_execsql_test 22.0 {
-  CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t9(rowid, x) VALUES(2, 'bbb');
  BEGIN;
    INSERT INTO t9(rowid, x) VALUES(1, 'aaa');
@@ -596,7 +612,7 @@ do_execsql_test 22.1 {

 #-------------------------------------------------------------------------
 do_execsql_test 23.0 {
-  CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  CREATE TABLE t11(x);
 }
 do_execsql_test 23.1 {
@@ -608,7 +624,7 @@ do_execsql_test 23.2 {

 #-------------------------------------------------------------------------
 do_execsql_test 24.0 {
-  CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t12 VALUES('aaaa');
 }
 do_execsql_test 24.1 {
@@ -618,6 +634,9 @@ do_execsql_test 24.1 {
    INSERT INTO t12 VALUES('aaaa');
  END;
 }
+execsql_pp {
+  SELECT rowid, hex(block) FROM t12_data
+}
 do_execsql_test 24.2 {
  INSERT INTO t12(t12) VALUES('integrity-check');
 }
@@ -627,7 +646,7 @@ do_execsql_test 24.3 {

 #-------------------------------------------------------------------------
 do_execsql_test 25.0 {
-  CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
 }
 do_execsql_test 25.1 {
  BEGIN;
@@ -638,6 +657,7 @@ SELECT * FROM t13('BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
 }


+}
 }

 expand_all_sql db
@@ -880,6 +880,347 @@ do_catchsql_test 5.4 {
  UPDATE t1 SET content=randomblob(500);
 } {1 {database disk image is malformed}}

+#-------------------------------------------------------------------------
+reset_db
+do_test 6.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+.open --hexdb
+| size 32768 pagesize 4096 filename crash-42fa37b694d45a.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07   .....@  ........
+|     96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d   ...............m
+|    112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00   .....N..........
+|   3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74   .........1tablet
+|   3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45   2t2.CREATE TABLE
+|   3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61    t2(x)V.......ta
+|   3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63   blet1_configt1_c
+|   3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42   onfig.CREATE TAB
+|   3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b   LE 't1_config'(k
+|   3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29    PRIMARY KEY, v)
+|   3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05    WITHOUT ROWID[.
+|   3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64   ..!!...tablet1_d
+|   3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65   ocsizet1_docsize
+|   3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74   .CREATE TABLE 't
+|   3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e   1_docsize'(id IN
+|   3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45   TEGER PRIMARY KE
+|   3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21   Y, sz BLOB)U...!
+|   3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65   !.wtablet1_conte
+|   3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45   ntt1_content.CRE
+|   3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f   ATE TABLE 't1_co
+|   3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45   ntent'(id INTEGE
+|   3824: 52 20 50 52 49 4d 41 52 49 20 4b 45 59 2c 20 63   R PRIMARI KEY, c
+|   3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65   0)i.......-table
+|   3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45   t1_idxt1_idx.CRE
+|   3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64   ATE TABLE 't1_id
+|   3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20   x'(segid, term, 
+|   3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45   pgno, PRIMARY KE
+|   3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20   Y(segid, term)) 
+|   3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07   WITHOUT ROWIDU..
+|   3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61   ......tablet1_da
+|   3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45   tat1_data.CREATE
+|   3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27    TABLE 't1_data'
+|   4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d   (id INTEGER PRIM
+|   4016: 41 52 b9 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42   AR. KEY, block B
+|   4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c   LOB):......ctabl
+|   4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54   et1t1CREATE VIRT
+|   4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49   UAL TABLE t1 USI
+|   4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29   NG fts5(content)
+| page 2 offset 4096
+|      0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd f0 00   ................
+|     16: 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
+|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80   .............$..
+|   4032: 80 80 80 01 03 00 4e 00 10 00 1e 06 30 61 62 61   ......N.....0aba
+|   4048: 63 6c 01 02 02 04 02 66 74 02 5f 02 04 04 6e 64   cl.....ft._...nd
+|   4064: 6f 6e 02 02 02 04 0a 07 05 01 03 00 10 03 03 0f   on..............
+|   4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 11   ...$............
+| page 3 offset 8192
+|      0: 0a 00 00 00 01 0f 00 01 00 00 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
+| page 4 offset 12288
+|      0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00   ................
+|   4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00   .....abandon....
+|   4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b   .abaft.....aback
+| page 5 offset 16384
+|      0: 0d 00 00 00 03 0f ee 00 0f fa 0f 00 00 00 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03   ................
+|   4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01   ................
+| page 6 offset 20480
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| page 7 offset 24576
+|      0: 0d 00 00 10 03 0f d6 00 0f 00 00 00 00 00 00 00   ................
+|   4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c   ..........rebuil
+|   4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63   d...+integrity-c
+|   4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 00 00 00   heck....optim...
+| page 8 offset 28672
+|      0: 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
+| end crash-42fa37b694d45a.db
+}]} {}
+
+do_execsql_test 6.1 {
+  INSERT INTO t1(t1,rank) VALUES('secure-delete',1);
+}
+do_catchsql_test 6.2 {
+  UPDATE t1 SET content=randomblob(500) WHERE t1;
+} {1 {constraint failed}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_test 7.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+.open --hexdb
+| size 40960 pagesize 4096 filename crash-d8b4a99207c10b.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04   ................
+|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     96: 00 00 00 00 0d 00 00 00 0d 0b 62 00 0f 97 0f 40   ..........b....@
+|    112: 0e d5 0e 75 0e 18 0d c0 0d 66 0d 0f 0c a4 0c 44   ...u.....f.....D
+|    128: 0b ec 0b a7 0b 62 00 00 00 00 00 00 00 00 00 00   .....b..........
+|   2912: 00 00 43 0d 06 17 11 11 08 75 74 61 62 6c 65 74   ..C......utablet
+|   2928: 34 74 34 43 52 45 41 54 45 20 56 49 52 54 55 41   4t4CREATE VIRTUA
+|   2944: 4c 20 54 41 42 4c 45 20 74 34 20 55 53 49 4e 47   L TABLE t4 USING
+|   2960: 20 66 74 73 35 76 6f 63 61 62 28 27 74 32 27 2c    fts5vocab('t2',
+|   2976: 20 27 72 6f 77 27 29 43 0c 06 17 11 11 08 75 74    'row')C......ut
+|   2992: 61 62 6c 65 74 33 74 33 43 52 45 41 54 45 20 56   ablet3t3CREATE V
+|   3008: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 33 20   IRTUAL TABLE t3 
+|   3024: 55 53 49 4e 47 20 66 74 73 35 76 6f 63 61 62 28   USING fts5vocab(
+|   3040: 27 74 31 27 2c 20 27 72 6f 77 27 29 56 0b 06 17   't1', 'row')V...
+|   3056: 1f 1f 01 7d 74 61 62 6c 65 74 32 5f 63 6f 6e 66   ....tablet2_conf
+|   3072: 69 67 74 32 5f 63 6f 6e 66 69 67 0a 43 52 45 41   igt2_config.CREA
+|   3088: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 63 6f 6e   TE TABLE 't2_con
+|   3104: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b   fig'(k PRIMARY K
+|   3120: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52   EY, v) WITHOUT R
+|   3136: 4f 57 49 44 5e 0a 07 17 21 21 01 81 07 74 61 62   OWID^...!!...tab
+|   3152: 6c 65 74 32 5f 63 6f 6e 74 65 6e 74 74 32 5f 63   let2_contentt2_c
+|   3168: 6f 6e 74 65 6e 74 09 43 52 45 41 54 45 20 54 41   ontent.CREATE TA
+|   3184: 42 4c 45 20 27 74 32 5f 63 6f 6e 74 65 6e 74 27   BLE 't2_content'
+|   3200: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d   (id INTEGER PRIM
+|   3216: 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c   ARY KEY, c0, c1,
+|   3232: 20 63 32 29 69 09 07 17 19 19 01 81 2d 74 61 62    c2)i.......-tab
+|   3248: 6c 65 74 32 5f 69 64 78 74 32 5f 69 64 78 08 43   let2_idxt2_idx.C
+|   3264: 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 32 5f   REATE TABLE 't2_
+|   3280: 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d   idx'(segid, term
+|   3296: 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20   , pgno, PRIMARY 
+|   3312: 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29   KEY(segid, term)
+|   3328: 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55   ) WITHOUT ROWIDU
+|   3344: 08 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 32 5f   ........tablet2_
+|   3360: 64 61 74 61 74 32 5f 64 61 74 61 07 43 52 45 41   datat2_data.CREA
+|   3376: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 64 61 74   TE TABLE 't2_dat
+|   3392: 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52   a'(id INTEGER PR
+|   3408: 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b   IMARY KEY, block
+|   3424: 20 42 4c 4f 42 29 58 07 07 17 11 11 08 81 1d 74    BLOB)X........t
+|   3440: 61 62 6c 65 74 32 74 32 43 52 45 41 54 45 20 56   ablet2t2CREATE V
+|   3456: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 32 20   IRTUAL TABLE t2 
+|   3472: 55 53 49 4e 47 20 66 74 73 35 28 27 61 27 2c 5b   USING fts5('a',[
+|   3488: 62 5d 2c 22 63 22 2c 64 65 74 61 69 6c 3d 6e 6f   b],.c.,detail=no
+|   3504: 6e 65 2c 63 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29   ne,columnsize=0)
+|   3520: 56 06 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f   V.......tablet1_
+|   3536: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 06   configt1_config.
+|   3552: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
+|   3568: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41   _config'(k PRIMA
+|   3584: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f   RY KEY, v) WITHO
+|   3600: 55 54 20 52 4f 57 49 44 5b 05 07 17 21 21 01 81   UT ROWID[...!!..
+|   3616: 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65   .tablet1_docsize
+|   3632: 74 31 5f 64 6f 63 73 69 7a 65 05 43 52 45 41 54   t1_docsize.CREAT
+|   3648: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73   E TABLE 't1_docs
+|   3664: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20   ize'(id INTEGER 
+|   3680: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20   PRIMARY KEY, sz 
+|   3696: 42 4c 4f 42 29 5e 04 07 17 21 21 01 81 07 74 61   BLOB)^...!!...ta
+|   3712: 62 6c 65 74 31 5f 63 6f 6e 74 65 6e 74 74 31 5f   blet1_contentt1_
+|   3728: 63 6f 6e 74 65 6e 74 04 43 52 45 41 54 45 20 54   content.CREATE T
+|   3744: 41 42 4c 45 20 27 74 31 5f 63 6f 6e 74 65 6e 74   ABLE 't1_content
+|   3760: 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49   '(id INTEGER PRI
+|   3776: 4d 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31   MARY KEY, c0, c1
+|   3792: 2c 20 63 32 29 69 03 07 17 19 19 01 81 2d 74 61   , c2)i.......-ta
+|   3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03   blet1_idxt1_idx.
+|   3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
+|   3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72   _idx'(segid, ter
+|   3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59   m, pgno, PRIMARY
+|   3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d    KEY(segid, term
+|   3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44   )) WITHOUT ROWID
+|   3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31   U........tablet1
+|   3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45   _datat1_data.CRE
+|   3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61   ATE TABLE 't1_da
+|   3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50   ta'(id INTEGER P
+|   3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63   RIMARY KEY, bloc
+|   3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b   k BLOB)g.......;
+|   4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20   tablet1t1CREATE 
+|   4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31   VIRTUAL TABLE t1
+|   4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 62 20    USING fts5(a,b 
+|   4048: 75 6e 69 6e 64 65 78 65 64 2c 63 2c 74 6f 6b 65   unindexed,c,toke
+|   4064: 6e 69 7a 65 3d 22 70 6f 72 74 65 72 20 61 73 63   nize=.porter asc
+|   4080: 69 69 22 2c 74 6f 6b 65 6e 64 61 74 61 3d 31 29   ii.,tokendata=1)
+| page 2 offset 4096
+|      0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c   ..h............|
+|     16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00   .*..............
+|   3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00   .......0........
+|   3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80   ..........7.....
+|   3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01   ...t.....0a.....
+|   3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01   b.....c.....g...
+|   3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01   ....h.......i...
+|   3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00   ............*...
+|   3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80   ............%...
+|   3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02   .....P.....0g...
+|   3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02   .....h........i.
+|   4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 7f f1 03   ........7.......
+|   4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01   .t.....0a.....b.
+|   4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02   ....c.....g.....
+|   4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 04   ..h.......i.....
+|   4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00   ................
+|   4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
+| page 3 offset 8192
+|      0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c   ................
+|   4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02   ................
+| page 4 offset 12288
+|      0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00   ................
+|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
+|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
+|   4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69    h i.......g h i
+|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
+|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
+| page 5 offset 16384
+|      0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03   ................
+|   4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03   ................
+| page 6 offset 20480
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| page 7 offset 24576
+|      0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00   ................
+|   3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84   ..............A.
+|   4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61   ............4.0a
+|   4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01   .....b.....c....
+|   4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01   .d...e...f...g..
+|   4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06   ...h.....i......
+|   4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f   ................
+|   4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
+| page 8 offset 28672
+|      0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
+| page 9 offset 32768
+|      0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00   ................
+|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
+|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
+|   4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69    h i.......g h i
+|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
+|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
+| page 10 offset 36864
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| end crash-d8b4a99207c10b.db
+}]} {}
+
+do_catchsql_test 7.1 {
+  SELECT snippet(t1, -1, '.', '..', '[', ']'), 
+         highlight(t1, 2, '[', ']') 
+           FROM t1('g + h') 
+           WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank;
+} {1 {database disk image is malformed}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_test 8.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+.open --hexdb
+| size 20480 pagesize 4096 filename crash-d57c01958e48ab.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 05   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 04   ................
+|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     96: 00 00 00 00 0d 00 00 00 05 0e 10 00 0f 97 0f 40   ...............@
+|    112: 0e d5 0e 68 0e 10 01 00 00 00 00 00 00 00 00 00   ...h............
+|   3600: 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f   V.......tablet1_
+|   3616: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 05   configt1_config.
+|   3632: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
+|   3648: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41   _config'(k PRIMA
+|   3664: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f   RY KEY, v) WITHO
+|   3680: 55 54 20 52 4f 57 49 44 6b 04 07 17 21 21 01 81   UT ROWIDk...!!..
+|   3696: 21 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65   !tablet1_docsize
+|   3712: 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 45 41 54   t1_docsize.CREAT
+|   3728: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73   E TABLE 't1_docs
+|   3744: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20   ize'(id INTEGER 
+|   3760: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20   PRIMARY KEY, sz 
+|   3776: 42 4c 4f 42 2c 20 6f 72 69 67 69 6e 20 49 4e 54   BLOB, origin INT
+|   3792: 45 47 45 52 29 69 03 07 17 19 19 01 81 2d 74 61   EGER)i.......-ta
+|   3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03   blet1_idxt1_idx.
+|   3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
+|   3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72   _idx'(segid, ter
+|   3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59   m, pgno, PRIMARY
+|   3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d    KEY(segid, term
+|   3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44   )) WITHOUT ROWID
+|   3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31   U........tablet1
+|   3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45   _datat1_data.CRE
+|   3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61   ATE TABLE 't1_da
+|   3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50   ta'(id INTEGER P
+|   3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63   RIMARY KEY, bloc
+|   3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b   k BLOB)g.......;
+|   4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20   tablet1t1CREATE 
+|   4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31   VIRTUAL TABLE t1
+|   4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 20 62    USING fts5(a, b
+|   4048: 2c 20 63 6f 6e 74 65 6e 74 3d 27 27 2c 20 63 6f   , content='', co
+|   4064: 6e 74 65 6e 74 6c 65 73 73 5f 64 65 6c 65 74 65   ntentless_delete
+|   4080: 3d 31 2c 20 74 6f 6b 65 6e 64 61 74 61 3d 31 29   =1, tokendata=1)
+| page 2 offset 4096
+|      0: 0d 0f eb 00 03 0e 17 00 0f e2 0e 17 0e 31 00 00   .............1..
+|     16: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
+|   3600: 00 00 00 00 00 00 00 18 0a 03 00 36 00 00 00 00   ...........6....
+|   3616: ff 00 00 01 01 01 01 00 01 01 01 01 01 01 00 00   ................
+|   3632: 07 83 29 84 80 80 80 80 01 04 00 86 56 00 00 01   ..).........V...
+|   3648: 96 04 30 61 61 61 01 02 02 01 04 02 04 01 08 02   ..0aaa..........
+|   3664: 04 04 04 01 10 02 04 04 04 04 04 04 04 01 20 02   .............. .
+|   3680: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 01   ................
+|   3696: 40 02 04 04 04 04 04 04 04 04 04 04 04 04 04 04   @...............
+|   3712: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3728: 04 01 81 00 02 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3744: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3760: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3776: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3792: 04 04 04 04 02 02 62 63 01 06 01 01 02 01 03 62   ......bc.......b
+|   3808: 62 62 02 02 03 01 04 03 06 01 08 03 06 06 06 01   bb..............
+|   3824: 10 03 06 06 06 06 06 06 06 01 20 03 06 06 06 06   .......... .....
+|   3840: 06 06 06 06 06 06 06 06 06 06 06 01 40 03 06 06   ............@...
+|   3856: 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06   ................
+|   3872: 06 06 06 06 06 06 06 06 06 06 16 06 06 02 02 63   ...............c
+|   3888: 64 02 06 01 01 02 01 03 63 63 63 03 02 05 01 04   d.......ccc.....
+|   3904: 05 0a 01 08 05 0a 0a 0a 01 10 05 0a 0a 0a 0a 0a   ................
+|   3920: 0a 0a 01 20 05 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a   ... ............
+|   3936: 0a 0a 0a 0a 02 02 64 65 03 06 01 01 02 01 03 64   ......de.......d
+|   3952: 64 64 04 02 09 01 04 09 12 01 08 09 12 12 12 01   dd..............
+|   3968: 10 09 12 12 12 12 12 12 12 02 02 65 66 04 06 01   ...........ef...
+|   3984: 01 02 01 03 65 65 65 05 02 11 01 04 11 22 01 08   ....eee.........
+|   4000: 11 22 22 22 02 02 66 67 05 06 01 01 02 01 03 66   ......fg.......f
+|   4016: 56 66 06 02 21 01 04 21 42 02 02 67 68 06 06 01   Vf..!..!B..gh...
+|   4032: 01 02 cb 03 67 67 67 07 02 41 02 02 68 69 07 06   ....ggg..A..hi..
+|   4048: 01 01 02 04 81 13 09 50 09 2e 09 1c 09 12 09 0c   .......P........
+|   4064: 09 08 07 01 03 00 14 07 81 77 07 00 00 00 15 22   .........w......
+|   4080: 00 00 00 00 ff 00 00 01 00 00 00 00 00 00 05 0c   ................
+| page 3 offset 8192
+|      0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
+| page 4 offset 12288
+|      0: 0d 00 00 00 07 0f c8 00 0f f8 0f f0 0f e8 0f e0   ................
+|     16: 0f d8 0f d0 0f c8 00 00 00 00 00 00 00 00 00 00   ................
+|   4032: 00 00 00 00 00 00 00 00 06 07 04 00 10 09 7f 01   ................
+|   4048: 06 06 04 00 10 09 3f 01 06 05 04 00 10 09 1f 01   ......?.........
+|   4064: 06 04 04 00 10 09 0f 01 06 03 04 00 10 09 07 01   ................
+|   4080: 06 02 04 00 10 09 03 01 06 01 04 00 10 09 01 01   ................
+| page 5 offset 16384
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| end crash-d57c01958e48ab.db
+}]} {}
+
+do_catchsql_test 8.1 {
+  SELECT rowid FROM t1('a* NOT ý') ;
+} {0 {1 2 3 4 5 6 7}}

 sqlite3_fts5_may_be_corrupt 0
 finish_test
@@ -0,0 +1,141 @@
+# 2010 June 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultG
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+set ::testprefix fts5faultH
+
+sqlite3_fts5_register_origintext db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+
+  BEGIN;
+    INSERT INTO t1 VALUES('oNe tWo thRee');
+    INSERT INTO t1 VALUES('One Two Three');
+    INSERT INTO t1 VALUES('onE twO threE');
+  COMMIT;
+  BEGIN;
+    INSERT INTO t1 VALUES('one two three');
+    INSERT INTO t1 VALUES('one two three');
+    INSERT INTO t1 VALUES('one two three');
+  COMMIT;
+}
+
+do_faultsim_test 1 -faults oom* -prep { 
+} -body {
+  execsql {
+    SELECT rowid FROM t1('three');
+  }
+} -test {
+  faultsim_integrity_check
+  faultsim_test_result {0 {1 2 3 4 5 6}}
+}
+
+
+reset_db
+sqlite3_fts5_register_origintext db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+
+  BEGIN;
+    INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(12, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(13, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(14, 'bbb BBB bbb');
+    INSERT INTO t1(rowid, x) VALUES(15, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(16, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(17, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(18, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(19, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(20, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(21, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(22, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(23, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(24, 'aaa bbb BBB');
+  COMMIT;
+}
+
+do_faultsim_test 2 -faults oom* -prep { 
+} -body {
+  execsql {
+    SELECT rowid FROM t1('BBB AND AAA');
+  }
+} -test {
+  faultsim_integrity_check
+  faultsim_test_result {0 {10 24}}
+}
+
+reset_db
+sqlite3_fts5_register_origintext db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+
+  INSERT INTO t1(rowid, x) VALUES(9, 'bbb Bbb BBB');
+  BEGIN;
+    INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(11, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(12, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(13, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(14, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(15, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(16, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(17, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(18, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(19, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(20, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(21, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(22, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(23, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(24, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(25, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(26, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(27, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(28, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(29, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(30, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(31, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(32, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(33, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(34, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(35, 'aaa bbb BBB');
+  COMMIT;
+}
+
+do_faultsim_test 3 -faults oom* -prep { 
+} -body {
+  execsql {
+    SELECT rowid FROM t1('BBB AND AAA');
+  }
+} -test {
+  faultsim_integrity_check
+  faultsim_test_result {0 {10 35}}
+}
+
+
+finish_test
@@ -91,7 +91,6 @@ do_execsql_test 2.2.1 {
  INSERT INTO vt0(c0) VALUES ('xyz');
 }

-breakpoint
 do_execsql_test 2.2.2 {
  ALTER TABLE t0 RENAME TO t1;
 }
@@ -500,6 +499,21 @@ do_execsql_test 17.5 {
  SELECT c0 FROM t0 WHERE c0 GLOB '*faul*';
 } {assertionfaultproblem}

+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 18.0 {
+  BEGIN;
+    CREATE VIRTUAL TABLE t1 USING fts5(text);
+    ALTER TABLE t1 RENAME TO t2;
+}
+
+do_execsql_test 18.1 {
+    DROP TABLE t2;
+}
+
+do_execsql_test 18.2 {
+  COMMIT;
+}

 finish_test

@@ -0,0 +1,297 @@
+# 2014 Jan 08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", detail=%DETAIL%
+  );
+  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO ft VALUES('Hello world');
+}
+
+do_execsql_test 1.2 {
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+proc b {x} { string map [list "\0" "."] $x }
+db func b b
+
+do_execsql_test 1.3 {
+  select b(term) from vocab;
+} {
+  hello.Hello
+  world
+}
+
+do_execsql_test 1.4 {
+  SELECT rowid FROM ft('Hello');
+} {1}
+
+#-------------------------------------------------------------------------
+reset_db
+
+# Return a random integer between 0 and n-1.
+#
+proc random {n} {
+  expr {abs(int(rand()*$n))}
+}
+
+proc select_one {list} {
+  set n [llength $list]
+  lindex $list [random $n]
+}
+
+proc term {} {
+  set first_letter {
+    a b c d e f g h i j k l m n o p q r s t u v w x y z
+    A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
+  }
+
+  set term [select_one $first_letter]
+  append term [random 100]
+}
+
+proc document {} {
+  set nTerm [expr [random 5] + 5]
+  set doc ""
+  for {set ii 0} {$ii < $nTerm} {incr ii} {
+    lappend doc [term]
+  }
+  set doc
+}
+db func document document
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", detail=%DETAIL%
+  );
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 128);
+  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
+}
+
+do_test 2.1 {
+  for {set ii 0} {$ii < 500} {incr ii} {
+    execsql { INSERT INTO ft VALUES( document() ) }
+  }
+} {}
+
+do_execsql_test 2.2 {
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+do_execsql_test 2.3 {
+  INSERT INTO ft(ft, rank) VALUES('merge', 16);
+}
+
+do_execsql_test 2.4 {
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+do_execsql_test 2.5 {
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", detail=%DETAIL%
+  );
+  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
+
+  INSERT INTO ft(rowid, x) VALUES(1, 'hello');
+  INSERT INTO ft(rowid, x) VALUES(2, 'Hello');
+  INSERT INTO ft(rowid, x) VALUES(3, 'HELLO');
+}
+
+#proc b {x} { string map [list "\0" "."] $x }
+#db func b b
+#execsql_pp { SELECT b(term) FROM vocab }
+
+do_execsql_test 3.1.1 { SELECT rowid FROM ft('hello') } 1
+do_execsql_test 3.1.2 { SELECT rowid FROM ft('Hello') } 2
+do_execsql_test 3.1.3 { SELECT rowid FROM ft('HELLO') } 3
+
+do_execsql_test 3.2 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(x, 
+      tokenize="origintext unicode61", 
+      tokendata=1,
+      detail=%DETAIL%
+  );
+  CREATE VIRTUAL TABLE vocab2 USING fts5vocab(ft2, instance);
+
+  INSERT INTO ft2(rowid, x) VALUES(1, 'hello');
+  INSERT INTO ft2(rowid, x) VALUES(2, 'Hello');
+  INSERT INTO ft2(rowid, x) VALUES(3, 'HELLO');
+
+  INSERT INTO ft2(rowid, x) VALUES(10, 'helloooo');
+}
+
+#proc b {x} { string map [list "\0" "."] $x }
+#db func b b
+#execsql_pp { SELECT b(term) FROM vocab }
+
+do_execsql_test 3.3.1 { SELECT rowid FROM ft2('hello') } {1 2 3}
+do_execsql_test 3.3.2 { SELECT rowid FROM ft2('Hello') } {1 2 3}
+do_execsql_test 3.3.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3}
+
+do_execsql_test 3.3.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+sqlite3_fts5_register_origintext db
+proc querytoken {cmd iPhrase iToken} { 
+  set txt [$cmd xQueryToken $iPhrase $iToken]
+  string map [list "\0" "."] $txt
+}
+sqlite3_fts5_create_function db querytoken querytoken
+
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
+  );
+  INSERT INTO ft VALUES('one two three four');
+}
+
+do_execsql_test 4.1 {
+  SELECT rowid, querytoken(ft, 0, 0) FROM ft('TwO')
+} {1 two.TwO}
+do_execsql_test 4.2 {
+  SELECT rowid, querytoken(ft, 0, 0) FROM ft('one TWO ThreE')
+} {1 one}
+do_execsql_test 4.3 {
+  SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE')
+} {1 two.TWO}
+
+if {"%DETAIL%"=="full"} {
+  # Phrase queries are only supported for detail=full.
+  #
+  do_execsql_test 4.4 {
+    SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"')
+  } {1 three.ThreE}
+  do_catchsql_test 4.5 {
+    SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"')
+  } {1 SQLITE_RANGE}
+  do_catchsql_test 4.6 {
+    SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"')
+  } {1 SQLITE_RANGE}
+  do_catchsql_test 4.7 {
+    SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"')
+  } {1 SQLITE_RANGE}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+sqlite3_fts5_register_origintext db
+proc insttoken {cmd iIdx iToken} { 
+  set txt [$cmd xInstToken $iIdx $iToken]
+  string map [list "\0" "."] $txt
+}
+sqlite3_fts5_create_function db insttoken insttoken
+fts5_aux_test_functions db
+
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
+  );
+  INSERT INTO ft VALUES('one ONE One oNe oNE one');
+}
+
+do_execsql_test 5.1 {
+  SELECT insttoken(ft, 0, 0), 
+         insttoken(ft, 1, 0),
+         insttoken(ft, 2, 0),
+         insttoken(ft, 3, 0),
+         insttoken(ft, 4, 0),
+         insttoken(ft, 5, 0)
+  FROM ft('one');
+} {
+  one one.ONE one.One one.oNe one.oNE one
+}
+
+do_execsql_test 5.2 {
+  SELECT insttoken(ft, 1, 0) FROM ft('one');
+} {
+  one.ONE
+}
+
+do_execsql_test 5.3 {
+  SELECT fts5_test_poslist(ft) FROM ft('one');
+} {
+  {0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5}
+}
+
+#-------------------------------------------------------------------------
+# Test the xInstToken() API with:
+#
+#   * a non tokendata=1 table.
+#   * prefix queries.
+#
+reset_db
+sqlite3_fts5_register_origintext db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, y, tokenize='origintext unicode61', detail=%DETAIL%
+  );
+
+  INSERT INTO ft VALUES('One Two', 'Three two');
+  INSERT INTO ft VALUES('three Three', 'one One');
+}
+proc tokens {cmd} { 
+  set ret [list]
+  for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+    set txt [$cmd xInstToken $iTok 0]
+    set txt [string map [list "\0" "."] $txt]
+    lappend ret $txt
+  }
+  set ret
+}
+sqlite3_fts5_create_function db tokens tokens
+
+do_execsql_test 6.1 {
+  SELECT rowid, tokens(ft) FROM ft('One');
+} {1 one.One 2 one.One}
+
+do_execsql_test 6.2 {
+  SELECT rowid, tokens(ft) FROM ft('on*');
+} {1 {{}} 2 {{} {}}}
+
+do_execsql_test 6.3 {
+  SELECT rowid, tokens(ft) FROM ft('Three*');
+} {1 {{}} 2 {{}}}
+
+}
+
+finish_test
+
@@ -0,0 +1,146 @@
+# 2014 Jan 08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+}
+
+do_execsql_test 1.1 {
+  BEGIN;
+  INSERT INTO ft VALUES('Hello');
+  INSERT INTO ft VALUES('hello');
+  INSERT INTO ft VALUES('HELLO');
+  INSERT INTO ft VALUES('today');
+  INSERT INTO ft VALUES('today');
+  INSERT INTO ft VALUES('today');
+  INSERT INTO ft VALUES('World');
+  INSERT INTO ft VALUES('world');
+  INSERT INTO ft VALUES('WORLD');
+  COMMIT;
+}
+
+do_execsql_test 1.2 { SELECT rowid FROM ft('hello'); } {1 2 3}
+do_execsql_test 1.3 { SELECT rowid FROM ft('today'); } {4 5 6}
+do_execsql_test 1.4 { SELECT rowid FROM ft('world'); } {7 8 9}
+
+do_execsql_test 1.5 {
+  SELECT count(*) FROM ft_data
+} 3
+
+do_execsql_test 1.6 {
+  DELETE FROM ft;
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
+  BEGIN;
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+    )
+    INSERT INTO ft SELECT 'Hello Hello Hello Hello Hello Hello Hello' FROM s;
+    INSERT INTO ft VALUES ('hELLO hELLO hELLO');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('World World World World World World World');
+    INSERT INTO ft VALUES('world world world world world world world');
+    INSERT INTO ft VALUES('WORLD WORLD WORLD WORLD WORLD WORLD WORLD');
+    INSERT INTO ft VALUES('World World World World World World World');
+    INSERT INTO ft VALUES('world world world world world world world');
+    INSERT INTO ft VALUES('WORLD WORLD WORLD WORLD WORLD WORLD WORLD');
+  COMMIT;
+}
+
+do_execsql_test 1.7 {
+  SELECT count(*) FROM ft_data;
+} 23
+
+do_execsql_test 1.8 { SELECT rowid FROM ft('hello') WHERE rowid>100; } {101}
+
+do_execsql_test 1.9 {
+  DELETE FROM ft;
+  INSERT INTO ft(ft) VALUES('optimize');
+  SELECT count(*) FROM ft_data;
+} {2}
+do_execsql_test 1.10 {
+  BEGIN;
+    INSERT INTO ft VALUES('Hello');
+    INSERT INTO ft VALUES('hello');
+    INSERT INTO ft VALUES('HELLO');
+    INSERT INTO ft VALUES('today');
+    INSERT INTO ft VALUES('today');
+    INSERT INTO ft VALUES('today');
+    INSERT INTO ft VALUES('World');
+    INSERT INTO ft VALUES('world');
+    INSERT INTO ft VALUES('WORLD');
+}
+
+do_execsql_test 1.11 { SELECT rowid FROM ft('hello'); } {1 2 3}
+do_execsql_test 1.12 { SELECT rowid FROM ft('today'); } {4 5 6}
+do_execsql_test 1.13 { SELECT rowid FROM ft('world'); } {7 8 9}
+do_execsql_test 1.14 { SELECT rowid FROM ft('hello') ORDER BY rank; } {1 2 3}
+
+#------------------------------------------------------------------------
+reset_db
+sqlite3_fts5_register_origintext db
+proc tokens {cmd} { 
+  set ret [list]
+  for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+    set txt [$cmd xInstToken $iTok 0]
+    set txt [string map [list "\0" "."] $txt]
+    lappend ret $txt
+  }
+  set ret
+}
+sqlite3_fts5_create_function db tokens tokens
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(
+    v, tokenize="origintext unicode61", tokendata=1, detail=none
+  );
+
+  INSERT INTO x1 VALUES('xxx Xxx XXX yyy YYY yyy');
+  INSERT INTO x1 VALUES('xxx yyy xxx yyy yyy yyy');
+}
+
+do_execsql_test 2.1 {
+  SELECT tokens(x1) FROM x1('xxx');
+} {
+  {xxx xxx.Xxx xxx.XXX} {xxx xxx}
+}
+
+do_execsql_test 2.2 {
+  UPDATE x1_content SET c0 = 'xxx xxX xxx yyy yyy yyy' WHERE id=1;
+}
+
+do_execsql_test 2.3 {
+  SELECT tokens(x1) FROM x1('xxx');
+} {
+  {xxx {} xxx} {xxx xxx}
+}
+
+finish_test
+
@@ -0,0 +1,101 @@
+# 2023 November 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext3
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+  reset_db
+
+  sqlite3_fts5_register_origintext db
+  fts5_aux_test_functions db
+  proc insttoken {cmd iIdx iToken} { 
+    set txt [$cmd xInstToken $iIdx $iToken]
+    string map [list "\0" "."] $txt
+  }
+  sqlite3_fts5_create_function db insttoken insttoken
+  
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE ft USING fts5(
+        x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
+    );
+  }
+  
+  do_execsql_test 1.1 {
+    INSERT INTO ft VALUES('Hello world HELLO WORLD hello');
+  }
+  
+  do_execsql_test 1.2 {
+    SELECT fts5_test_poslist(ft) FROM ft('hello');
+  } {{0.0.0 0.0.2 0.0.4}}
+
+  do_execsql_test 1.3 {
+    SELECT 
+      insttoken(ft, 0, 0),
+      insttoken(ft, 1, 0),
+      insttoken(ft, 2, 0)
+    FROM ft('hello');
+  } {hello.Hello hello.HELLO hello}
+
+  do_execsql_test 1.4 {
+    SELECT 
+      insttoken(ft, 0, 0),
+      insttoken(ft, 1, 0),
+      insttoken(ft, 2, 0)
+    FROM ft('hello') ORDER BY rank;
+  } {hello.Hello hello.HELLO hello}
+
+  do_execsql_test 1.5 {
+    CREATE VIRTUAL TABLE ft2 USING fts5(
+        x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
+    );
+    INSERT INTO ft2(rowid, x) VALUES(1, 'ONE one two three ONE');
+    INSERT INTO ft2(rowid, x) VALUES(2, 'TWO one two three TWO');
+    INSERT INTO ft2(rowid, x) VALUES(3, 'THREE one two three THREE');
+  }
+
+  do_execsql_test 1.6 {
+    SELECT insttoken(ft2, 0, 0), rowid FROM ft2('three') ORDER BY rank;
+  } {three.THREE 3 three 1 three 2}
+
+  do_execsql_test 1.7 {
+    INSERT INTO ft2(rowid, x) VALUES(10, 'aaa bbb BBB');
+    INSERT INTO ft2(rowid, x) VALUES(12, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(13, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(14, 'bbb BBB bbb');
+    INSERT INTO ft2(rowid, x) VALUES(15, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(16, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(17, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(18, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(19, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(20, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(21, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(22, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(23, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(24, 'aaa bbb BBB');
+  }
+
+  do_execsql_test 1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24}
+  do_execsql_test 1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24}
+
+}
+
+finish_test
+
@@ -0,0 +1,66 @@
+# 2023 November 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext4
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+}
+
+do_execsql_test 1.1 {
+  BEGIN;
+    INSERT INTO ft SELECT 'the first thing';
+
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<90000
+    )
+    INSERT INTO ft SELECT 'The second thing' FROM s;
+
+    INSERT INTO ft SELECT 'the first thing';
+  COMMIT;
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+
+foreach {tn sql expr} {
+  1 { SELECT rowid FROM ft('the') }       {$mem > 250000}
+  2 { SELECT rowid FROM ft('first') }     {$mem <  50000}
+  3 { SELECT rowid FROM ft('the first') } {$mem <  50000}
+} {
+  db close
+  sqlite3 db test.db
+  sqlite3_fts5_register_origintext db
+
+  execsql $sql
+  do_test 1.2.$tn {
+    set mem [lindex [sqlite3_db_status db CACHE_USED 0] 1]
+    expr $expr
+  } 1
+}
+
+proc b {x} { string map [list "\0" "."] $x }
+db func b b
+# execsql_pp { SELECT segid, b(term), pgno from ft_idx }
+
+finish_test
+
@@ -0,0 +1,273 @@
+# 2023 Dec 04
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests for tables that use both tokendata=1 and contentless_delete=1.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+# Return a random integer between 0 and n-1.
+#
+proc random {n} { expr {abs(int(rand()*$n))} }
+
+# Select an element of the list passed as the only argument at random and
+# return it. 
+#
+proc select_one {list} {
+  set n [llength $list]
+  lindex $list [random $n]
+}
+
+# Given a term that consists entirely of alphabet characters, return all
+# permutations of the term using upper and lower case characters. e.g.
+#
+#    "abc" -> {CBA cBA CbA cbA CBa cBa Cba cba}
+#
+proc casify {term {lRet {{}}}} {
+  if {$term==""} { return $lRet }
+  set t [string range $term 1 end]
+  set f1 [string toupper [string range $term 0 0]]
+  set f2 [string tolower [string range $term 0 0]]
+  set ret [list]
+  foreach x $lRet {
+    lappend ret "$x$f1"
+    lappend ret "$x$f2"
+  }
+  return [casify $t $ret]
+}
+
+proc vocab {} {
+  list abc def ghi jkl mno pqr stu vwx yza
+}
+
+# Return a random 3 letter term.
+#
+proc term {} {
+  if {[info exists ::expanded_vocab]==0} {
+    foreach v [vocab] { lappend ::expanded_vocab {*}[casify $v] }
+  }
+
+  select_one $::expanded_vocab
+}
+
+# Return a document - between 3 and 10 terms.
+#
+proc document {} {
+  set nTerm [expr [random 3] + 7]
+  set doc ""
+  for {set ii 0} {$ii < $nTerm} {incr ii} {
+    lappend doc [term]
+  }
+  set doc
+}
+db func document document
+
+#-------------------------------------------------------------------------
+
+expr srand(6)
+
+set NDOC  200
+set NLOOP 50
+
+sqlite3_fts5_register_origintext db
+
+proc tokens {cmd} { 
+  set ret [list]
+  for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+    set txt [$cmd xInstToken $iTok 0]
+    set txt [string map [list "\0" "."] $txt]
+    lappend ret $txt
+  }
+  set ret
+}
+sqlite3_fts5_create_function db tokens tokens
+
+proc rankfunc {cmd} { 
+  $cmd xRowid
+}
+sqlite3_fts5_create_function db rankfunc rankfunc
+
+proc ctrl_tokens {term args} {
+  set ret [list]
+  set term [string tolower $term]
+  foreach doc $args {
+    foreach a $doc {
+      if {[string tolower $a]==$term} {
+        if {$a==$term} {
+          lappend ret $a
+        } else {
+          lappend ret [string tolower $a].$a
+        }
+      }
+    }
+  }
+  set ret
+}
+db func ctrl_tokens ctrl_tokens
+
+proc do_all_vocab_test {tn} {
+  foreach ::v [concat [vocab] nnn] {
+    set answer [execsql {
+      SELECT id, ctrl_tokens($::v, x) FROM ctrl WHERE x LIKE '%' || $::v || '%'
+    }]
+    do_execsql_test $tn.$::v.1 {
+      SELECT rowid, tokens(ft) FROM ft($::v)
+    } $answer
+    do_execsql_test $tn.$::v.2 {
+      SELECT rowid, tokens(ft) FROM ft($::v) ORDER BY rank
+    } $answer
+  }
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", content=, contentless_delete=1,
+      tokendata=1
+  );
+
+  CREATE TABLE ctrl(id INTEGER PRIMARY KEY, x TEXT);
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
+  INSERT INTO ft(ft, rank) VALUES('rank', 'rankfunc()');
+}
+do_test 1.1 {
+  for {set ii 0} {$ii < $NDOC} {incr ii} {
+    set doc [document]
+    execsql {
+      INSERT INTO ft(rowid, x) VALUES($ii, $doc);
+      INSERT INTO ctrl(id, x) VALUES($ii, $doc);
+    }
+  }
+} {}
+
+#execsql_pp { SELECT * FROM ctrl }
+#execsql_pp { SELECT * FROM ft }
+#fts5_aux_test_functions db
+#execsql_pp { SELECT rowid, tokens(ft), fts5_test_poslist(ft) FROM ft('ghi'); }
+
+do_all_vocab_test 1.2
+
+for {set ii 0} {$ii < $NLOOP} {incr ii} {
+  set lRowid [execsql { SELECT id FROM ctrl WHERE random() % 2 }]
+  foreach r $lRowid {
+    execsql { DELETE FROM ft WHERE rowid = $r }
+    execsql { DELETE FROM ctrl WHERE rowid = $r }
+
+    set doc [document]
+    execsql { INSERT INTO ft(rowid, x) VALUES($r, $doc) }
+    execsql { INSERT INTO ctrl(id, x) VALUES($r, $doc) }
+  }
+  do_all_vocab_test 1.3.$ii
+}
+
+#-------------------------------------------------------------------------
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(
+      x, y, tokenize="origintext unicode61", content=, contentless_delete=1,
+      tokendata=1
+  );
+
+  CREATE TABLE ctrl2(id INTEGER PRIMARY KEY, x TEXT, y TEXT);
+  INSERT INTO ft2(ft2, rank) VALUES('pgsz', 64);
+  INSERT INTO ft2(ft2, rank) VALUES('rank', 'rankfunc()');
+}
+do_test 2.1 {
+  for {set ii 0} {$ii < $NDOC} {incr ii} {
+    set doc1 [document]
+    set doc2 [document]
+    execsql {
+      INSERT INTO ft2(rowid, x, y) VALUES($ii, $doc, $doc2);
+      INSERT INTO ctrl2(id, x, y) VALUES($ii, $doc, $doc2);
+    }
+  }
+} {}
+
+proc do_all_vocab_test2 {tn} {
+  foreach ::v [vocab] {
+    set answer [execsql {
+      SELECT id, ctrl_tokens($::v, x, y) FROM ctrl2
+      WHERE x LIKE '%' || $::v || '%' OR y LIKE '%' || $::v || '%';
+    }]
+    do_execsql_test $tn.$::v.1 {
+      SELECT rowid, tokens(ft2) FROM ft2($::v)
+    } $answer
+    do_execsql_test $tn.$::v.2 {
+      SELECT rowid, tokens(ft2) FROM ft2($::v) ORDER BY rank
+    } $answer
+  }
+}
+
+do_all_vocab_test2 2.2
+
+for {set ii 0} {$ii < $NLOOP} {incr ii} {
+  set lRowid [execsql { SELECT id FROM ctrl2 WHERE random() % 2 }]
+  foreach r $lRowid {
+    execsql { DELETE FROM ft2 WHERE rowid = $r }
+    execsql { DELETE FROM ctrl2 WHERE rowid = $r }
+
+    set doc1 [document]
+    set doc2 [document]
+    execsql { INSERT INTO ft2(rowid, x, y) VALUES($r, $doc, $doc1) }
+    execsql { INSERT INTO ctrl2(id, x, y) VALUES($r, $doc, $doc2) }
+  }
+  do_all_vocab_test 2.3.$ii
+}
+
+#-------------------------------------------------------------------------
+
+unset -nocomplain ::expanded_vocab
+proc vocab {} {
+  list abcde fghij klmno
+}
+
+proc do_all_vocab_test3 {tn} {
+  foreach ::v [concat [vocab] nnn] {
+    set answer [execsql {
+      SELECT rowid, ctrl_tokens($::v, w) FROM ctrl3 WHERE w LIKE '%' || $::v || '%'
+    }]
+    do_execsql_test $tn.$::v.1 {
+      SELECT rowid, tokens(ft3) FROM ft3($::v)
+    } $answer
+    do_execsql_test $tn.$::v.2 {
+      SELECT rowid, tokens(ft3) FROM ft3($::v) ORDER BY rank
+    } $answer
+  }
+}
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft3 USING fts5(
+      w, tokenize="origintext unicode61", content=, contentless_delete=1,
+      tokendata=1
+  );
+  INSERT INTO ft3(ft3, rank) VALUES('rank', 'rankfunc()');
+  CREATE TABLE ctrl3(w);
+}
+
+do_execsql_test 3.1 {
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<2
+  )
+  INSERT INTO ctrl3 SELECT document() FROM s;
+  INSERT INTO ft3(rowid, w) SELECT rowid, w FROM ctrl3;
+}
+
+do_all_vocab_test3 3.2
+
+
+finish_test
+
@@ -0,0 +1,51 @@
+# 2023 Nov 23
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5secure8
+
+proc sql_repeat {txt n} {
+  string repeat $txt $n
+}
+db func repeat sql_repeat
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x);
+
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
+
+  INSERT INTO ft(rowid, x) VALUES(100, 'hello world');
+  INSERT INTO ft(rowid, x) VALUES(200, 'one day');
+
+  BEGIN;
+    INSERT INTO ft(rowid, x) VALUES(45, 'one two three');
+    UPDATE ft SET x = repeat('hello world ', 500) WHERE rowid=100;
+  COMMIT
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO ft(ft, rank) VALUES('secure-delete', 1);
+  DELETE FROM ft WHERE rowid=100;
+}
+
+do_execsql_test 1.2 {
+  PRAGMA integrity_check;
+} {ok}
+
+
+
+
+
+finish_test
+
+
@@ -343,7 +343,9 @@ do_execsql_test 17.0 {
    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
  COMMIT;
 }
-do_execsql_test 17.1 { SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 }
+do_execsql_test 17.1 { 
+  SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 
+}
 do_execsql_test 17.2 {
  BEGIN;
    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
@@ -280,6 +280,30 @@ do_catchsql_test 5.2 {
  INSERT INTO t1 SELECT randomblob(3000) FROM v1
 } {1 {query aborted}}

+#-------------------------------------------------------------------------
+reset_db
+sqlite3_fts5_may_be_corrupt 1
+
+do_execsql_test 6.0 {
+  BEGIN TRANSACTION;
+    CREATE VIRTUAL TABLE t1 USING fts5(a,b unindexed,c,tokenize="porter ascii",tokendata=1);
+    REPLACE INTO t1_data VALUES(1,X'03090009');
+    REPLACE INTO t1_data VALUES(10,X'000000000103030003010101020101030101');
+    REPLACE INTO t1_data VALUES(137438953473,X'0000002e023061010202010162010203010163010204010167010601020201016801060102030101690106010204040606060808');
+    REPLACE INTO t1_data VALUES(274877906945,X'0000001f013067020802010202010168020803010203010169020804010204040909');
+    REPLACE INTO t1_data VALUES(412316860417,X'0000002e023061030202010162030203010163030204010167030601020201016803060102030101690306010204040606060808');
+  COMMIT;
+}
+
+do_execsql_test 6.1 {
+  CREATE VIRTUAL TABLE t3 USING fts5vocab('t1', 'row');
+}
+
+do_catchsql_test 6.2 {
+  SELECT * FROM t3;
+} {1 {database disk image is malformed}}
+
+sqlite3_fts5_may_be_corrupt 0

 finish_test

@@ -582,6 +582,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
            bNextPage = 1;
          }else{
            iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload);
+            if( nPayload>0x7fffff00 ) nPayload &= 0x3fff;
          }
    
          /* If this is a leaf intkey cell, load the rowid */
@@ -854,7 +854,7 @@ dir.sql := sql
 speedtest1 := ../../speedtest1
 speedtest1.c := ../../test/speedtest1.c
 speedtest1.sql := $(dir.sql)/speedtest1.sql
-speedtest1.cliflags := --size 25 --big-transactions
+speedtest1.cliflags := --size 10 --big-transactions
 $(speedtest1):
 	$(MAKE) -C ../.. speedtest1
 $(speedtest1.sql): $(speedtest1) $(MAKEFILE)
@@ -247,9 +247,8 @@ globalThis.sqlite3Worker1Promiser = function callee(config = callee.defaultConfi
 globalThis.sqlite3Worker1Promiser.defaultConfig = {
  worker: function(){
 //#if target=es6-bundler-friendly
-    return new Worker("sqlite3-worker1-bundler-friendly.mjs",{
-      type: 'module' /* Noting that neither Firefox nor Safari suppor this,
-                        as of this writing. */
+    return new Worker(new URL("sqlite3-worker1-bundler-friendly.mjs", import.meta.url),{
+      type: 'module'
    });
 //#else
    let theJs = "sqlite3-worker1.js";
@@ -267,8 +266,12 @@ globalThis.sqlite3Worker1Promiser.defaultConfig = {
    }
    return new Worker(theJs + globalThis.location.search);
 //#endif
-  }.bind({
+  }
+//#ifnot target=es6-bundler-friendly
+  .bind({
    currentScript: globalThis?.document?.currentScript
-  }),
+  })
+//#endif
+  ,
  onerror: (...args)=>console.error('worker1 promiser error',...args)
 };
@@ -0,0 +1,86 @@
+<!doctype html>
+<html lang="en-us">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon">
+    <link rel="stylesheet" href="common/testing.css"/>
+    <title>sqlite3-api batch SQL runner for the SAHPool VFS</title>
+  </head>
+  <body>
+    <header id='titlebar'><span>sqlite3-api batch SQL runner for the SAHPool VFS</span></header>
+    <div>
+      <span class='input-wrapper'>
+        <input type='checkbox' class='disable-during-eval' id='cb-reverse-log-order' checked></input>
+        <label for='cb-reverse-log-order' id='lbl-reverse-log-order'>Reverse log order</label>
+      </span>
+    </div>
+    <div id='test-output' class='reverse'></div>
+    <script>
+      (function(){
+        const E = (sel)=>document.querySelector(sel);
+
+        const eOut = E('#test-output');
+        const log2 = function(cssClass,...args){
+          let ln;
+          if(1 || cssClass){
+            ln = document.createElement('div');
+            if(cssClass) ln.classList.add(cssClass);
+            ln.append(document.createTextNode(args.join(' ')));
+          }else{
+            // This doesn't work with the "reverse order" option!
+            ln = document.createTextNode(args.join(' ')+'\n');
+          }
+          eOut.append(ln);
+        };
+        const log = (...args)=>{
+          //console.log(...args);
+          log2('', ...args);
+        };
+        const logErr = function(...args){
+          console.error(...args);
+          log2('error', ...args);
+        };
+        const logWarn = function(...args){
+          console.warn(...args);
+          log2('warning', ...args);
+        };
+
+        const cbReverseLog = E('#cb-reverse-log-order');
+        const lblReverseLog = E('#lbl-reverse-log-order');
+        if(cbReverseLog.checked){
+            lblReverseLog.classList.add('warning');
+            eOut.classList.add('reverse');
+        }
+        cbReverseLog.addEventListener('change', function(){
+            if(this.checked){
+                eOut.classList.add('reverse');
+                lblReverseLog.classList.add('warning');
+            }else{
+                eOut.classList.remove('reverse');
+                lblReverseLog.classList.remove('warning');
+            }
+        }, false);
+
+        const w = new Worker('batch-runner-sahpool.js?sqlite3.dir=jswasm');
+        w.onmessage = function(msg){
+          msg = msg.data;
+          switch(msg.type){
+            case 'stdout': log(...msg.data); break;
+            case 'warn': logWarn(...msg.data); break;
+            case 'error': logErr(...msg.data); break;
+            default:
+              logErr("Unhandled worker message type:",msg);
+              break;
+          }
+        };
+      })();
+    </script>
+    <style>
+      #test-output {
+          white-space: break-spaces;
+          overflow: auto;
+      }
+    </style>
+  </body>
+</html>
@@ -0,0 +1,341 @@
+/*
+  2023-11-30
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  A basic batch SQL runner for the SAHPool VFS. This file must be run in
+  a worker thread. This is not a full-featured app, just a way to get some
+  measurements for batch execution of SQL for the OPFS SAH Pool VFS.
+*/
+'use strict';
+
+const wMsg = function(msgType,...args){
+  postMessage({
+    type: msgType,
+    data: args
+  });
+};
+const toss = function(...args){throw new Error(args.join(' '))};
+const warn = (...args)=>{ wMsg('warn',...args); };
+const error = (...args)=>{ wMsg('error',...args); };
+const log = (...args)=>{ wMsg('stdout',...args); }
+let sqlite3;
+const urlParams = new URL(globalThis.location.href).searchParams;
+const cacheSize = (()=>{
+  if(urlParams.has('cachesize')) return +urlParams.get('cachesize');
+  return 200;
+})();
+
+
+/** Throws if the given sqlite3 result code is not 0. */
+const checkSqliteRc = (dbh,rc)=>{
+  if(rc) toss("Prepare failed:",sqlite3.capi.sqlite3_errmsg(dbh));
+};
+
+const sqlToDrop = [
+  "SELECT type,name FROM sqlite_schema ",
+  "WHERE name NOT LIKE 'sqlite\\_%' escape '\\' ",
+  "AND name NOT LIKE '\\_%' escape '\\'"
+].join('');
+
+const clearDbSqlite = function(db){
+  // This would be SO much easier with the oo1 API, but we specifically want to
+  // inject metrics we can't get via that API, and we cannot reliably (OPFS)
+  // open the same DB twice to clear it using that API, so...
+  const rc = sqlite3.wasm.exports.sqlite3_wasm_db_reset(db.handle);
+  log("reset db rc =",rc,db.id, db.filename);
+};
+
+const App = {
+  db: undefined,
+  cache:Object.create(null),
+  log: log,
+  warn: warn,
+  error: error,
+  metrics: {
+    fileCount: 0,
+    runTimeMs: 0,
+    prepareTimeMs: 0,
+    stepTimeMs: 0,
+    stmtCount: 0,
+    strcpyMs: 0,
+    sqlBytes: 0
+  },
+  fileList: undefined,
+  execSql: async function(name,sql){
+    const db = this.db;
+    const banner = "========================================";
+    this.log(banner,
+             "Running",name,'('+sql.length,'bytes)');
+    const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm;
+    let pStmt = 0, pSqlBegin;
+    const metrics = db.metrics = Object.create(null);
+    metrics.prepTotal = metrics.stepTotal = 0;
+    metrics.stmtCount = 0;
+    metrics.malloc = 0;
+    metrics.strcpy = 0;
+    if(this.gotErr){
+      this.error("Cannot run SQL: error cleanup is pending.");
+      return;
+    }
+    // Run this async so that the UI can be updated for the above header...
+    const endRun = ()=>{
+      metrics.evalSqlEnd = performance.now();
+      metrics.evalTimeTotal = (metrics.evalSqlEnd - metrics.evalSqlStart);
+      this.log("metrics:",JSON.stringify(metrics, undefined, ' '));
+      this.log("prepare() count:",metrics.stmtCount);
+      this.log("Time in prepare_v2():",metrics.prepTotal,"ms",
+               "("+(metrics.prepTotal / metrics.stmtCount),"ms per prepare())");
+      this.log("Time in step():",metrics.stepTotal,"ms",
+               "("+(metrics.stepTotal / metrics.stmtCount),"ms per step())");
+      this.log("Total runtime:",metrics.evalTimeTotal,"ms");
+      this.log("Overhead (time - prep - step):",
+               (metrics.evalTimeTotal - metrics.prepTotal - metrics.stepTotal)+"ms");
+      this.log(banner,"End of",name);
+      this.metrics.prepareTimeMs += metrics.prepTotal;
+      this.metrics.stepTimeMs += metrics.stepTotal;
+      this.metrics.stmtCount += metrics.stmtCount;
+      this.metrics.strcpyMs += metrics.strcpy;
+      this.metrics.sqlBytes += sql.length;
+    };
+
+    const runner = function(resolve, reject){
+      ++this.metrics.fileCount;
+      metrics.evalSqlStart = performance.now();
+      const stack = wasm.scopedAllocPush();
+      try {
+        let t, rc;
+        let sqlByteLen = sql.byteLength;
+        const [ppStmt, pzTail] = wasm.scopedAllocPtr(2);
+        t = performance.now();
+        pSqlBegin = wasm.scopedAlloc( sqlByteLen + 1/*SQL + NUL*/) || toss("alloc(",sqlByteLen,") failed");
+        metrics.malloc = performance.now() - t;
+        metrics.byteLength = sqlByteLen;
+        let pSql = pSqlBegin;
+        const pSqlEnd = pSqlBegin + sqlByteLen;
+        t = performance.now();
+        wasm.heap8().set(sql, pSql);
+        wasm.poke(pSql + sqlByteLen, 0);
+        //log("SQL:",wasm.cstrToJs(pSql));
+        metrics.strcpy = performance.now() - t;
+        let breaker = 0;
+        while(pSql && wasm.peek8(pSql)){
+          wasm.pokePtr(ppStmt, 0);
+          wasm.pokePtr(pzTail, 0);
+          t = performance.now();
+          rc = capi.sqlite3_prepare_v2(
+            db.handle, pSql, sqlByteLen, ppStmt, pzTail
+          );
+          metrics.prepTotal += performance.now() - t;
+          checkSqliteRc(db.handle, rc);
+          pStmt = wasm.peekPtr(ppStmt);
+          pSql = wasm.peekPtr(pzTail);
+          sqlByteLen = pSqlEnd - pSql;
+          if(!pStmt) continue/*empty statement*/;
+          ++metrics.stmtCount;
+          t = performance.now();
+          rc = capi.sqlite3_step(pStmt);
+          capi.sqlite3_finalize(pStmt);
+          pStmt = 0;
+          metrics.stepTotal += performance.now() - t;
+          switch(rc){
+            case capi.SQLITE_ROW:
+            case capi.SQLITE_DONE: break;
+            default: checkSqliteRc(db.handle, rc); toss("Not reached.");
+          }
+        }
+        resolve(this);
+      }catch(e){
+        if(pStmt) capi.sqlite3_finalize(pStmt);
+        this.gotErr = e;
+        reject(e);
+      }finally{
+        capi.sqlite3_exec(db.handle,"rollback;",0,0,0);
+        wasm.scopedAllocPop(stack);
+      }
+    }.bind(this);
+    const p = new Promise(runner);
+    return p.catch(
+      (e)=>this.error("Error via execSql("+name+",...):",e.message)
+    ).finally(()=>{
+      endRun();
+    });
+  },
+
+  /**
+     Loads batch-runner.list and populates the selection list from
+     it. Returns a promise which resolves to nothing in particular
+     when it completes. Only intended to be run once at the start
+     of the app.
+  */
+  loadSqlList: async function(){
+    const infile = 'batch-runner.list';
+    this.log("Loading list of SQL files:", infile);
+    let txt;
+    try{
+      const r = await fetch(infile);
+      if(404 === r.status){
+        toss("Missing file '"+infile+"'.");
+      }
+      if(!r.ok) toss("Loading",infile,"failed:",r.statusText);
+      txt = await r.text();
+    }catch(e){
+      this.error(e.message);
+      throw e;
+    }
+    App.fileList = txt.split(/\n+/).filter(x=>!!x);
+    this.log("Loaded",infile);
+  },
+
+  /** Fetch ./fn and return its contents as a Uint8Array. */
+  fetchFile: async function(fn, cacheIt=false){
+    if(cacheIt && this.cache[fn]) return this.cache[fn];
+    this.log("Fetching",fn,"...");
+    let sql;
+    try {
+      const r = await fetch(fn);
+      if(!r.ok) toss("Fetch failed:",r.statusText);
+      sql = new Uint8Array(await r.arrayBuffer());
+    }catch(e){
+      this.error(e.message);
+      throw e;
+    }
+    this.log("Fetched",sql.length,"bytes from",fn);
+    if(cacheIt) this.cache[fn] = sql;
+    return sql;
+  }/*fetchFile()*/,
+
+  /**
+     Converts this.metrics() to a form which is suitable for easy conversion to
+     CSV. It returns an array of arrays. The first sub-array is the column names.
+     The 2nd and subsequent are the values, one per test file (only the most recent
+     metrics are kept for any given file).
+  */
+  metricsToArrays: function(){
+    const rc = [];
+    Object.keys(this.dbs).sort().forEach((k)=>{
+      const d = this.dbs[k];
+      const m = d.metrics;
+      delete m.evalSqlStart;
+      delete m.evalSqlEnd;
+      const mk = Object.keys(m).sort();
+      if(!rc.length){
+        rc.push(['db', ...mk]);
+      }
+      const row = [k.split('/').pop()/*remove dir prefix from filename*/];
+      rc.push(row);
+      row.push(...mk.map((kk)=>m[kk]));
+    });
+    return rc;
+  },
+
+  metricsToBlob: function(colSeparator='\t'){
+    const ar = [], ma = this.metricsToArrays();
+    if(!ma.length){
+      this.error("Metrics are empty. Run something.");
+      return;
+    }
+    ma.forEach(function(row){
+      ar.push(row.join(colSeparator),'\n');
+    });
+    return new Blob(ar);
+  },
+
+  /**
+     Fetch file fn and eval it as an SQL blob. This is an async
+     operation and returns a Promise which resolves to this
+     object on success.
+  */
+  evalFile: async function(fn){
+    const sql = await this.fetchFile(fn);
+    return this.execSql(fn,sql);
+  }/*evalFile()*/,
+
+  /**
+     Fetches the handle of the db associated with
+     this.e.selImpl.value, opening it if needed.
+  */
+  initDb: function(){
+    const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm;
+    const stack = wasm.scopedAllocPush();
+    let pDb = 0;
+    const d = Object.create(null);
+    d.filename = "/batch.db";
+    try{
+      const oFlags = capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE;
+      const ppDb = wasm.scopedAllocPtr();
+      const rc = capi.sqlite3_open_v2(d.filename, ppDb, oFlags, this.PoolUtil.vfsName);
+      pDb = wasm.peekPtr(ppDb)
+      if(rc) toss("sqlite3_open_v2() failed with code",rc);
+      capi.sqlite3_exec(pDb, "PRAGMA cache_size="+cacheSize, 0, 0, 0);
+      this.log("cache_size =",cacheSize);
+    }catch(e){
+      if(pDb) capi.sqlite3_close_v2(pDb);
+      throw e;
+    }finally{
+      wasm.scopedAllocPop(stack);
+    }
+    d.handle = pDb;
+    this.log("Opened db:",d.filename,'@',d.handle);
+    return d;
+  },
+
+  closeDb: function(){
+    if(this.db.handle){
+      this.sqlite3.capi.sqlite3_close_v2(this.db.handle);
+      this.db.handle = undefined;
+    }
+  },
+
+  run: async function(sqlite3){
+    delete this.run;
+    this.sqlite3 = sqlite3;
+    const capi = sqlite3.capi, wasm = sqlite3.wasm;
+    this.log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid());
+    this.log("WASM heap size =",wasm.heap8().length);
+    let timeStart;
+    sqlite3.installOpfsSAHPoolVfs({
+      clearOnInit: true, initialCapacity: 4,
+      name: 'batch-sahpool',
+      verbosity: 2
+    }).then(PoolUtil=>{
+      App.PoolUtil = PoolUtil;
+      App.db = App.initDb();
+    })
+      .then(async ()=>this.loadSqlList())
+      .then(async ()=>{
+        timeStart = performance.now();
+        for(let i = 0; i < App.fileList.length; ++i){
+          const fn = App.fileList[i];
+          await App.evalFile(fn);
+          if(App.gotErr) throw App.gotErr;
+        }
+      })
+      .then(()=>{
+        App.metrics.runTimeMs = performance.now() - timeStart;
+        App.log("total metrics:",JSON.stringify(App.metrics, undefined, ' '));
+        App.log("Reload the page to run this again.");
+        App.closeDb();
+        App.PoolUtil.removeVfs();
+      })
+      .catch(e=>this.error("ERROR:",e));
+  }/*run()*/
+}/*App*/;
+
+let sqlite3Js = 'sqlite3.js';
+if(urlParams.has('sqlite3.dir')){
+  sqlite3Js = urlParams.get('sqlite3.dir') + '/' + sqlite3Js;
+}
+importScripts(sqlite3Js);
+globalThis.sqlite3InitModule().then(async function(sqlite3_){
+  log("Done initializing. Running batch runner...");
+  sqlite3 = sqlite3_;
+  App.run(sqlite3_);
+});
@@ -72,7 +72,6 @@
    App.logHtml("reset db rc =",rc,db.id, db.filename);
  };

-  
  const E = (s)=>document.querySelector(s);
  const App = {
    e: {
@@ -91,6 +90,15 @@
    db: Object.create(null),
    dbs: Object.create(null),
    cache:{},
+    metrics: {
+      fileCount: 0,
+      runTimeMs: 0,
+      prepareTimeMs: 0,
+      stepTimeMs: 0,
+      stmtCount: 0,
+      strcpyMs: 0,
+      sqlBytes: 0
+    },
    log: console.log.bind(console),
    warn: console.warn.bind(console),
    cls: function(){this.e.output.innerHTML = ''},
@@ -117,7 +125,6 @@
                   "Running",name,'('+sql.length,'bytes) using',db.id);
      const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm;
      let pStmt = 0, pSqlBegin;
-      const stack = wasm.scopedAllocPush();
      const metrics = db.metrics = Object.create(null);
      metrics.prepTotal = metrics.stepTotal = 0;
      metrics.stmtCount = 0;
@@ -142,6 +149,11 @@
        this.logHtml("Overhead (time - prep - step):",
                     (metrics.evalTimeTotal - metrics.prepTotal - metrics.stepTotal)+"ms");
        this.logHtml(banner,"End of",name);
+        this.metrics.prepareTimeMs += metrics.prepTotal;
+        this.metrics.stepTimeMs += metrics.stepTotal;
+        this.metrics.stmtCount += metrics.stmtCount;
+        this.metrics.strcpyMs += metrics.strcpy;
+        this.metrics.sqlBytes += sql.length;
      };

      let runner;
@@ -214,7 +226,9 @@
        }.bind(this);
      }else{/*sqlite3 db...*/
        runner = function(resolve, reject){
+          ++this.metrics.fileCount;
          metrics.evalSqlStart = performance.now();
+          const stack = wasm.scopedAllocPush();
          try {
            let t;
            let sqlByteLen = sql.byteLength;
@@ -269,7 +283,7 @@
      let p;
      if(1){
        p = new Promise(function(res,rej){
-          setTimeout(()=>runner(res, rej), 50)/*give UI a chance to output the "running" banner*/;
+          setTimeout(()=>runner(res, rej), 0)/*give UI a chance to output the "running" banner*/;
        });
      }else{
        p = new Promise(runner);
@@ -401,7 +415,7 @@
      });
      return new Blob(ar);
    },
-    
+
    downloadMetrics: function(){
      const b = this.metricsToBlob();
      if(!b) return;
@@ -576,6 +590,8 @@
        const timeTotal = performance.now() - timeStart;
        who.logHtml("Run-remaining time:",timeTotal,"ms ("+(timeTotal/1000/60)+" minute(s))");
        who.clearStorage();
+        App.metrics.runTimeMs = timeTotal;
+        who.logHtml("Total metrics:",JSON.stringify(App.metrics,undefined,'  '));
      }, false);
    }/*run()*/
  }/*App*/;
@@ -20,7 +20,7 @@
     the main (UI) thread.
  */
  let logHtml;
-  if(self.window === self /* UI thread */){
+  if(globalThis.window === globalThis /* UI thread */){
    console.log("Running demo from main UI thread.");
    logHtml = function(cssClass,...args){
      const ln = document.createElement('div');
@@ -250,7 +250,7 @@
  }/*demo1()*/;

  log("Loading and initializing sqlite3 module...");
-  if(self.window!==self) /*worker thread*/{
+  if(globalThis.window!==globalThis) /*worker thread*/{
    /*
      If sqlite3.js is in a directory other than this script, in order
      to get sqlite3.js to resolve sqlite3.wasm properly, we have to
@@ -262,19 +262,20 @@
      that's not needed.

      URL arguments passed as part of the filename via importScripts()
-      are simply lost, and such scripts see the self.location of
+      are simply lost, and such scripts see the globalThis.location of
      _this_ script.
    */
    let sqlite3Js = 'sqlite3.js';
-    const urlParams = new URL(self.location.href).searchParams;
+    const urlParams = new URL(globalThis.location.href).searchParams;
    if(urlParams.has('sqlite3.dir')){
      sqlite3Js = urlParams.get('sqlite3.dir') + '/' + sqlite3Js;
    }
    importScripts(sqlite3Js);
  }
-  self.sqlite3InitModule({
-    // We can redirect any stdout/stderr from the module
-    // like so...
+  globalThis.sqlite3InitModule({
+    /* We can redirect any stdout/stderr from the module like so, but
+       note that doing so makes use of Emscripten-isms, not
+       well-defined sqlite APIs. */
    print: log,
    printErr: error
  }).then(function(sqlite3){
@@ -46,6 +46,9 @@
          file:// URLs.</li>
        <li>Any OPFS-related pages or tests require:
          <ul>
+            <li>An OPFS-capable browser released after February
+              2023. Some tests will work with Chromium-based browsers
+              going back to around v102.</li>
            <li>That the web server emit the so-called
              <a href='https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cross-Origin-Opener-Policy'>COOP</a>
              and
@@ -53,12 +56,6 @@
              headers. <a href='https://sqlite.org/althttpd'>althttpd</a> requires the
              <code>-enable-sab</code> flag for that.
            </li>
-            <li>A very recent version of a Chromium-based browser
-              (v102 at least, possibly newer). OPFS support in the
-              other major browsers is pending. Development and testing
-              is currently done against a dev-channel release of
-              Chrome (v111 as of 2023-02-10).
-            </li>
          </ul>
        </li>
      </ul>
@@ -31,6 +31,9 @@
          file:// URLs.</li>
        <li>Any OPFS-related pages or tests require:
          <ul>
+            <li>An OPFS-capable browser released after February
+              2023. Some tests will work with Chromium-based browsers
+              going back to around v102.</li>
            <li>That the web server emit the so-called
              <a href='https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cross-Origin-Opener-Policy'>COOP</a>
              and
@@ -38,12 +41,6 @@
              headers. <a href='https://sqlite.org/althttpd'>althttpd</a> requires the
              <code>-enable-sab</code> flag for that.
            </li>
-            <li>A very recent version of a
-              Chromium-based browser (v102 at least, possibly newer). OPFS
-              support in the other major browsers is pending. Development
-              and testing is currently done against a dev-channel release
-              of Chrome (v111 as of 2023-02-10).
-            </li>
          </ul>
        </li>
      </ul>
@@ -350,7 +350,7 @@
                eControls.classList.remove('hidden');
                break;
            case 'stdout': log(msg.data); break;
-            case 'stdout': logErr(msg.data); break;
+            case 'stderr': logErr(msg.data); break;
            case 'run-start':
                eControls.disabled = true;
                log("Running speedtest1 with argv =",msg.data.join(' '));
@@ -7,9 +7,9 @@
  }
  importScripts(speedtestJs);
  /**
-     If this environment contains OPFS, this function initializes it and
-     returns the name of the dir on which OPFS is mounted, else it returns
-     an empty string.
+     If this build includes WASMFS, this function initializes it and
+     returns the name of the dir on which OPFS is mounted, else it
+     returns an empty string.
  */
  const wasmfsDir = function f(wasmUtil){
    if(undefined !== f._) return f._;
@@ -47,6 +47,7 @@
  };
  const log = (...args)=>logMsg('stdout',args);
  const logErr = (...args)=>logMsg('stderr',args);
+  const realSahName = 'opfs-sahpool-speedtest1';

  const runSpeedtest = async function(cliFlagsArray){
    const scope = App.wasm.scopedAllocPush();
@@ -57,7 +58,6 @@
      ];
      App.logBuffer.length = 0;
      const ndxSahPool = argv.indexOf('opfs-sahpool');
-      const realSahName = 'opfs-sahpool-speedtest1';
      if(ndxSahPool>0){
        argv[ndxSahPool] = realSahName;
        log("Updated argv for opfs-sahpool: --vfs",realSahName);
@@ -73,7 +73,7 @@
          clearOnInit: true,
          verbosity: 2
        }).then(PoolUtil=>{
-          log("opfs-sahpool successfully installed as",realSahName);
+          log("opfs-sahpool successfully installed as",PoolUtil.vfsName);
          App.sqlite3.$SAHPoolUtil = PoolUtil;
          //console.log("sqlite3.oo1.OpfsSAHPoolDb =", App.sqlite3.oo1.OpfsSAHPoolDb);
        });
@@ -102,19 +102,6 @@
    }
  };

-  const sahpSanityChecks = function(sqlite3){
-    log("Attempting OpfsSAHPoolDb sanity checks...");
-    const db = new sqlite3.oo1.OpfsSAHPoolDb('opfs-sahpoool.db');
-    const fn = db.filename;
-    db.exec([
-      'create table t(a);',
-      'insert into t(a) values(1),(2),(3);'
-    ]);
-    db.close();
-    sqlite3.wasm.sqlite3_wasm_vfs_unlink(sqlite3_vfs_find("opfs-sahpool"), fn);
-    log("SAH sanity checks done.");
-  };
-
  const EmscriptenModule = {
    print: log,
    printErr: logErr,
@@ -1,12 +1,12 @@
-C Pickup\sstray\soput?()\scalls\sthat\sshould\shave\sgone\sto\sstdout.\sSimplify\sconsole\sline\sreading,\ssacrificing\sspeed\s(which\sdoes\snot\smatter\sthen)\sfor\scode\ssize.
-D 2023-11-27T15:08:50.454
+C Add\sNEVER\sto\stwo\sunreachable\sbranches\sin\sJSON.
+D 2023-12-12T17:13:10.122
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
 F Makefile.in 890caf094636c308bc981032baf8f9208bf307755f9197ae4218a9fbcec2e449
 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6
 F Makefile.msc 59bb36dba001f0b38212be0794fb838f25371008b180929bcf08aa799694c168
-F README.md 963d30019abf0cc06b263cd2824bce022893f3f93a531758f6f04ff2194a16a8
+F README.md 6358805260a03ebead84e168bbf3740ddf3f683b477e478567186aa7afb490d3
 F VERSION 73573d4545343f001bf5dc5461173a7c78c203dd046cabcf99153878cf25d3a6
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
@@ -41,7 +41,7 @@ F doc/compile-for-windows.md 50b27d77be96195c66031a3181cb8684ed822327ea834e07f9c
 F doc/json-enhancements.md e356fc834781f1f1aa22ee300027a270b2c960122468499bf347bb123ce1ea4f
 F doc/lemon.html 44a53a1d2b42d7751f7b2f478efb23c978e258d794bfd172442307a755b9fa44
 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710
-F doc/testrunner.md 2434864be2219d4f0b6ffc99d0a2172d531c4ca4345340776f67ad4edd90dc90
+F doc/testrunner.md 8d36ec692cf4994bb66d84a4645b9afa1ce9d47dc12cbf8d437c5a5fb6ddeedb
 F doc/trusted-schema.md 33625008620e879c7bcfbbfa079587612c434fa094d338b08242288d358c3e8a
 F doc/vdbesort-memory.md 4da2639c14cd24a31e0af694b1a8dd37eaf277aff3867e9a8cc14046bc49df56
 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a
@@ -50,8 +50,8 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 6f247666b495c477628dd19364d279c78ea48cd90c72d9f9b98ad1aff3294f94
 F ext/async/sqlite3async.h 46b47c79357b97ad85d20d2795942c0020dc20c532114a49808287f04aa5309a
-F ext/consio/console_io.c 274bff35fe8dd94c7428393c92e336bf84e693e1dacfd26f4edd55e766c6c11c x
-F ext/consio/console_io.h 030661ee4ed0dae5d3a8309b15a8e81a0f83b2d3d5fdb02b20748a5f1bf50e16
+F ext/consio/console_io.c 097323f60037d70523421f0248958d2280851f8ff2f9a443d4ee1474c4769118 x
+F ext/consio/console_io.h 3228dff1717481202a24f6dcf45ce0b75e7a778bf6877089518a44e1473b76a3
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4
 F ext/expert/expert1.test 0dd5cb096d66bed593e33053a3b364f6ef52ed72064bf5cf298364636dbf3cd6
@@ -88,28 +88,28 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
-F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
-F ext/fts5/fts5.h 05501612cc655504c5dce8ba765ab621d50fc478490089beaa0d75e00b23e520
-F ext/fts5/fts5Int.h 78a63cc0795186cde5384816a9403a68c65774b35d952e05b81a1b4b158e07c8
+F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e
+F ext/fts5/fts5.h 9d7867cc63631296462c90bafe57c4881b56e480fa510ffaee6a77049258c714
+F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880e9834
 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
-F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081
-F ext/fts5/fts5_expr.c bd3b81ce669c4104e34ffe66570af1999a317b142c15fccb112de9fb0caa57a6
-F ext/fts5/fts5_hash.c 076058f93327051952a752dc765df1acfe783eb11b419b30652aa1fc1f987902
-F ext/fts5/fts5_index.c 809407c520c6afc3c1a0b8080cbd9080e4d84442f52a31654cc0e3d3acccea17
-F ext/fts5/fts5_main.c a07ed863b8bd9e6fefb62db2fd40a3518eb30a5f7dcfda5be915dd2db45efa2f
+F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf
+F ext/fts5/fts5_expr.c b1ec526371b9ffde82341423a5b9753c42cbea629a41b69f26fa377d13b95a8e
+F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
+F ext/fts5/fts5_index.c ed206045ff0f2226d870fa41fba45f738c0cc953ab74ba68477091b9a574ccd3
+F ext/fts5/fts5_main.c b908696c52410e8383019ac0657c8a5cd0c8f60e78edc169e9b3c4b93f24c933
 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
-F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
+F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b
 F ext/fts5/fts5_tokenize.c 83cfcede3898001cab84432a36ce1503e3080cf9b1c682b022ec82e267ea4c13
 F ext/fts5/fts5_unicode2.c eca63dbc797f8ff0572e97caf4631389c0ab900d6364861b915bdd4735973f00
 F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80
-F ext/fts5/fts5_vocab.c aed56169ae5c1aa9b8189c779ffeef04ed516d3c712c06914e6d91a6759f4e4a
+F ext/fts5/fts5_vocab.c 209e0c151e108d5f3621fa24b91e9b02f3750ee6c3f9ccec312df39481b68a09
 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
-F ext/fts5/test/fts5_common.tcl a9de9c2209cc4e7ae3c753e783504e67206c6c1467d08f209cd0c5923d3e8d8b
-F ext/fts5/test/fts5aa.test ba5158eba7d61359becdfca895ef471072c7bf7b20e5e60dcb4d024c8419c926
+F ext/fts5/test/fts5_common.tcl 8b1848ac2baad10e444e4183034a52050b52d20b3796d9d30e78f01ab0d05583
+F ext/fts5/test/fts5aa.test 4db81519863244a3cab35795fe65ab6b592e7970c7409eba098b23ebbfc08d95
 F ext/fts5/test/fts5ab.test bd932720c748383277456b81f91bc00453de2174f9762cd05f95d0495dc50390
 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d
 F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de
@@ -145,7 +145,7 @@ F ext/fts5/test/fts5corrupt.test b6d4034b682bb3387bc44c510c71b3c67d4349e4df13949
 F ext/fts5/test/fts5corrupt2.test 99e7e23a58b4d89eb7167c6de1669cbc595cd3c79ab333e0eb56405473319e77
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
 F ext/fts5/test/fts5corrupt4.test f4c08e2182a48d8b70975fd869ee5391855c06d8a0ff87b6a2529e7c5a88a1d3
-F ext/fts5/test/fts5corrupt5.test eb6ba5ca28ef7c4c6b01e850d388cdb3dacc8c4c2f383f79d0a98128257742b4
+F ext/fts5/test/fts5corrupt5.test f9dbed6785c38d891c0b947fd6e973e9c4eb44991e1322f8d7e5281a019276f2
 F ext/fts5/test/fts5corrupt6.test bf8eeae07825b088b9665d9d8e4accbd8dc9bf3cb85b6c64cf6c9e18ccc420a4
 F ext/fts5/test/fts5corrupt7.test 80ad7f683a8bda2404731bb77e8c3dbbb620c1f6cc583cca8239f6accd6338c0
 F ext/fts5/test/fts5delete.test 619295b20dbc1d840b403ee07c878f52378849c3c02e44f2ee143b3e978a0aa7
@@ -170,6 +170,7 @@ F ext/fts5/test/fts5faultD.test e7ed7895abfe6bc98a5e853826f6b74956e7ba7f594f1860
 F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e4710c77eb8ce7075
 F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1
 F ext/fts5/test/fts5faultG.test d2e5a4d9a34e08dcaadcaeafef74d10cbc2abdd11aa2659a18af0294bf2812d3
+F ext/fts5/test/fts5faultH.test 57f53c87ffd59be0265840f2b54a16811f9cb9012db86aad9b41d0d14d85dfe3
 F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b996159f6909dc8079
 F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717
 F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e
@@ -182,7 +183,7 @@ F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fc
 F ext/fts5/test/fts5matchinfo.test 10c9a6f7fe61fb132299c4183c012770b10c4d5c2f2edb6df0b6607f683d737a
 F ext/fts5/test/fts5merge.test e92a8db28b45931e7a9c7b1bbd36101692759d00274df74d83fd29d25d53b3a6
 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2
-F ext/fts5/test/fts5misc.test 5ca82f2a5ee016b0842043155d1382f98a34d0d86b2791165a44d7807f6e0f54
+F ext/fts5/test/fts5misc.test dd97c86c9cbc3e587067e640f6ce88842cfbf5d23bb0e0fbb7f6707623b2d505
 F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581
 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45
 F ext/fts5/test/fts5near.test 211477940142d733ac04fad97cb24095513ab2507073a99c2765c3ddd2ef58bd
@@ -190,6 +191,11 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618
 F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1
 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785
 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca
+F ext/fts5/test/fts5origintext.test d2796fa08ee7aecfabdc0c45bb8a2fb16a00ea8757e63fbc153b718dbe430a39
+F ext/fts5/test/fts5origintext2.test f3b9436de540828d01f0672df855b09ebc0863e126d5b56234701d71dfa73634
+F ext/fts5/test/fts5origintext3.test 0d25933506600452a5ab3873cbb418ed5f2de2446c3672b9997b1ea104b0e7f0
+F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871
+F ext/fts5/test/fts5origintext5.test a037bdf7235a22033c4663837bdb12d9738245464a3ac2f60c71fc40d07ede7d
 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b
 F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a
 F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15
@@ -209,9 +215,10 @@ F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b98
 F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a
 F ext/fts5/test/fts5secure6.test 74bf04733cc523bccca519bb03d3b4e2ed6f6e3db7c59bf6be82c88a0ac857fd
 F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a
+F ext/fts5/test/fts5secure8.test eb3579e9d58b0acad97e8082dee1f99b2d393198f03500b453c2b25761c0c298
 F ext/fts5/test/fts5securefault.test dbca2b6a1c16700017f5051138991b705410889933f2a37c57ae8a23b296b10b
 F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b
-F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0
+F ext/fts5/test/fts5simple2.test 8dd2389ee75e21a1429fe87e5f8c7d9a97ad1470304a8a2d3ba4b8c3c345fecd
 F ext/fts5/test/fts5simple3.test d5c74a9d3ca71bd5dd5cacb7c55b86ea12cdddfc8b1910e3de2995206898380f
 F ext/fts5/test/fts5synonym.test 1651815b8008de170e8e600dcacc17521d765482ea8f074ae82cfa870d8bb7fb
 F ext/fts5/test/fts5synonym2.test 8f891fc49cc1e8daed727051e77e1f42849c784a6a54bef82564761b2cb3e016
@@ -231,7 +238,7 @@ F ext/fts5/test/fts5unindexed.test 9021af86a0fb9fc616f7a69a996db0116e7936d0db638
 F ext/fts5/test/fts5update.test b8affd796e45c94a4d19ad5c26606ea06065a0f162a9562d9f005b5a80ccf0bc
 F ext/fts5/test/fts5version.test d6e5a5897550afeccc2f8531d87404dc1c289ee89385dd4318dbdd75e71d7a67
 F ext/fts5/test/fts5vocab.test 7ed80d9af1ddaaa1637da05e406327b5aac250848bc604c1c1cc667908b87760
-F ext/fts5/test/fts5vocab2.test 681980e92e031c9f3fe8d9c149189e876c108da2fb0fb3a25bd8a9b94bff8f68
+F ext/fts5/test/fts5vocab2.test 1b1f0059f762ffb404213d35dac013e010621f08128589b6ec7bec59d9a710f3
 F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85
 F ext/fts5/tool/fts5txt2db.tcl c0d43c8590656f8240e622b00957b3a0facc49482411a9fdc2870b45c0c82f9f
 F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093
@@ -459,7 +466,7 @@ F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69
 F ext/rbu/sqlite3rbu.c d4ddf8f0e93772556e452a6c2814063cf47efb760a0834391a9d0cd9859fa4b9
 F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304
 F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055
-F ext/recover/dbdata.c fc7147a68422cbbbaa481ee92ae1752cc25f5a24302bece1c70dcb76345bd736
+F ext/recover/dbdata.c b7746c2ec801b453840831311be4b31f8c8f9dd97791060a69bbf12392c78949
 F ext/recover/recover1.test c484d01502239f11b61f23c1cee9f5dd19fa17617f8974e42e74d64639c524cf
 F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a
 F ext/recover/recoverbuild.test c74170e0f7b02456af41838afeb5353fdb985a48cc2331d661bbabbca7c6b8e3
@@ -567,7 +574,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb
 F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c
-F ext/wasm/GNUmakefile 0e362f3fc04eab6628cbe4f1e35f4ab4a200881f6b5f753b27fb45eabeddd9d2
+F ext/wasm/GNUmakefile 57439eec2b8b4d4074e5d861e93aab3f32bb0f44339a2b472c23f4e638b7e8a3
 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576
 F ext/wasm/README.md a8a2962c3aebdf8d2104a9102e336c5554e78fc6072746e5daf9c61514e7d193
 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff
@@ -595,10 +602,12 @@ F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f
 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 595953994aa3ae2287c889c4da39ab3d6f17b6461ecf4bec334b7a3faafddb02
 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 46c4afa6c50d7369252c104f274ad977a97e91ccfafc38b400fe36e90bdda88e
 F ext/wasm/api/sqlite3-wasm.c d0e09eb5ed3743c00294e30019e591c3aa150572ae7ffe8a8994568a7377589f
-F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f
+F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 569d4e859968e65f55dec5fac0b879828a23c8b179162cc7812fec19f844dd21
 F ext/wasm/api/sqlite3-worker1.c-pp.js a541112aa51e16705f13a99bb943c64efe178aa28c86704a955f8fd9afe4ba37
+F ext/wasm/batch-runner-sahpool.html e9a38fdeb36a13eac7b50241dfe7ae066fe3f51f5c0b0151e7baee5fce0d07a7
+F ext/wasm/batch-runner-sahpool.js 54a3ac228e6c4703fe72fb65c897e19156263a51fe9b7e21d2834a45e876aabd
 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8
-F ext/wasm/batch-runner.js 0dad6a02ad796f1003d3b7048947d275c4d6277f63767b8e685c27df8fdac93e
+F ext/wasm/batch-runner.js 05ec254f5dbfe605146d9640b3db17d6ef8c3fbef6aa8396051ca72bb5884e3f
 F ext/wasm/c-pp.c 6d80d8569d85713effe8b0818a3cf51dc779e3f0bf8dc88771b8998552ee25b4
 F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51
 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15
@@ -606,7 +615,7 @@ F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a7
 F ext/wasm/common/whwasmutil.js 4c64594eecc7af4ae64259e95a71ba2a7edf118881aaff0bba86d0c7164e78e4
 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed
 F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508
-F ext/wasm/demo-123.js 38aa8faec4d0ace1c973bc8a7a1533584463ebeecd4c420daa7d9687beeb9cb5
+F ext/wasm/demo-123.js c7b3cca50c55841c381a9ca4f9396e5bbdc6114273d0b10a43e378e32e7be5bf
 F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e
 F ext/wasm/demo-jsstorage.js 44e3ae7ec2483b6c511384c3c290beb6f305c721186bcf5398ca4e00004a06b8
 F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98ab22f5786620b3354ed15f
@@ -620,8 +629,8 @@ F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d695
 F ext/wasm/fiddle/fiddle-worker.js e0153f9af6500805c6f09c0b3cfdb7d857e9d6863dbee9d50d1628fccf5f4b4d
 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715
 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2
-F ext/wasm/index-dist.html 22379774f0ad4edcaaa8cf9c674c82e794cc557719a8addabed74eb8069d412e
-F ext/wasm/index.html 4e7847b909f4ae0da8c829b150b79454050e53b3658431f138636257729cd42b
+F ext/wasm/index-dist.html e91d76e4581185238fd3d42ed86ec600f7023ed3e3a944c5c356f25304bf1263
+F ext/wasm/index.html b31ce41c0da476d5ffcef23069b9d3415b419d65af5779096ebcfbcbade453a9
 F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54
 F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb
 F ext/wasm/module-symbols.html dc476b403369b26a1a23773e13b80f41b9a49f0825e81435fe3600a7cfbbe337
@@ -629,8 +638,8 @@ F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d
 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63
 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d
 F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe
-F ext/wasm/speedtest1-worker.html e33e2064bda572c0c3ebaec7306c35aa758d9d27e245d67e807f8cc4a9351cc5
-F ext/wasm/speedtest1-worker.js 315d26198c46be7c85e26fda15d80ef882424276abde25ffd8b026fb02a35d8c
+F ext/wasm/speedtest1-worker.html 864b65ed78ce24847a348c180e7f267621a02ca027068a1863ec1c90187c1852
+F ext/wasm/speedtest1-worker.js 4d2ea70a3c24e05bdca78025202841f33d298c4fa9541a0070c3228661f89ecd
 F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da
 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
 F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
@@ -668,8 +677,8 @@ F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645
 F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522
 F src/btree.c f3b09c5414de3a11db73e11e1d66f4c5e53c9e89876ff3b531a887ab656ca303 x
 F src/btree.h 03e3356f5208bcab8eed4e094240fdac4a7f9f5ddf5e91045ce589f67d47c240
-F src/btreeInt.h ef12a72b708677e48d6bc8dcd66fed25434740568b89e2cfa368093cfc5b9d15
-F src/build.c 189e4517d67f09f0a3e0d8e1faa6e2ef0c2e95f6ac82e33c912cb7efa2a359cc
+F src/btreeInt.h 3e2589726c4f105e653461814f65857465da68be1fac688de340c43b873f4062
+F src/build.c 386eadecabe2e99a3783eb802ca01e665f8e0c2af0e0aab28161fd7def219a9d
 F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 23331529e654be40ca97d171cbbffe9b3d4c71cc53b78fe5501230675952da8b
@@ -677,20 +686,20 @@ F src/date.c 3b8d02977d160e128469de38493b4085f7c5cf4073193459909a6af3cf6d7c91
 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782
 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43
 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500
-F src/expr.c e9a491c7f156e5b25641c28af11b735a424e108a21b9f83b6f3e51c99a8141d9
+F src/expr.c 05278def9c186b5875d6903ea26148c7461b9ce0344f0fd7be9a0dfea0a4538a
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00
 F src/func.c 472f6dcfa39cf54f89a6aec76c79c225fb880a6c14469c15d361331662b9bf43
-F src/global.c 29f56a330ed9d1b5cd9b79ac0ca36f97ac3afc730ff8bfa987b0db9e559d684d
+F src/global.c 765a0656d6cbf043cb272ff0ae38f39cc46713539ffe6793258ed3eb4b188b52
 F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220
 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c f93bf3df3651b1e01e2b57f7dc56f727e7b0e212d934eacf21c6fc8b31bf685e
+F src/json.c cd270629b4b99d065522b45c3d2b5185491b1ab3544843c6f780aa50755d54dc
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36
-F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9
+F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4
 F src/malloc.c f016922435dc7d1f1f5083a03338a3e91f8c67ce2c5bdcfa4cdef62e612f5fcc
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2
@@ -711,7 +720,7 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63
 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06
 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a
 F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107
-F src/os_unix.c dc5404b56da7fb13cf272ddb94c3753cf9e82d32a65cba35dbb6aadcb849419c
+F src/os_unix.c 97bdcd43315da7aaec9fea2da1ff7c9de458f93dd363e073f2742403a7f2e011
 F src/os_win.c 4a50a154aeebc66a1f8fb79c1ff6dd5fe3d005556533361e0d460d41cb6a45a8
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 987ab3a2cd9065d62e9955474470ff733445e2357432a67e3d0f5a8f9313e334
@@ -720,19 +729,19 @@ F src/parse.y 020d80386eb216ec9520549106353c517d2bbc89be28752ffdca649a9eaf56ec
 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75
 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5
 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00
-F src/pragma.c b3b4ad9c0298d63098a067acca613c21a5f56b4d176d5842922bcd0b07b7164e
+F src/pragma.c b5b4cff830575e6188cd56a295a57448d2b9dbc53f0dae58e22b97354cda3781
 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c
-F src/printf.c 9da63b9ae1c14789bcae12840f5d800fd9302500cd2d62733fac77f0041b4750
+F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c d017bad7ba8e778617701a0e986fdeb393d67d6afa84fb28ef4e8b8ad2acf916
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
-F src/select.c 85857bedd2913d888aa571755b48c54cd2e6e7fcb0087e19b226ee0368cfda1e
-F src/shell.c.in a731aa9fe56329a093dae62296e98f0bb2ea8bdb771032238992dd137a865950
-F src/sqlite.h.in d93a4821d2f792467a60f7dc81268d1bb8634f40c31694ef254cab4f9921f96a
+F src/select.c 9f55c9f3307b9888f62abe709eec245e98ff217bd14c044f93d72810bb7dc445
+F src/shell.c.in 0cd2ef4b3c814dded5436625ab664d9a973cbc4266a1768e3aa4cbf11bb41eca
+F src/sqlite.h.in adcc7dbfeea1e69d6d487139a7e90db8a48fe998f3f5bb0f85c683e6a6fa68ca
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
-F src/sqliteInt.h cd171cba32c7a553e7623fbd82b68b36a1b6c81079ab963260777ea9b3abe4d9
+F src/sqliteInt.h 615524fff083abf9353aa4a3a8adca2d32b3e61e014e62b1a5335c454a386783
 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6
 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -797,11 +806,11 @@ F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242
 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
 F src/util.c b22cc9f203a8c0b9ee5338a67f8860347d14845864c10248bebe84518a781677
 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104
-F src/vdbe.c 319af2cf092d20e233e8ad4267ae49bfe33c50ac4db4ee7e47af898f824c2368
-F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0
+F src/vdbe.c f73bead140670fac1aa4227188827ada52387a5fe0ccff0dd5af2a906754d904
+F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb
 F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c
-F src/vdbeapi.c b07df805110dc6e81f2a3f9cd4e83f56ea523277a59bcec489a12b740c1079e7
-F src/vdbeaux.c b34dfbc09403ccb676608da16ff0780d23d466470563d24fdf6350b8d2271d5e
+F src/vdbeapi.c 8f57d60c89da0b60e6d4e272358c511f6bae4e24330bdb11f8b42f986d1bf21b
+F src/vdbeaux.c c5a471b34e9c4cfc0295a3e10734fd197670ffaebcb742f284c8e17e8026ceea
 F src/vdbeblob.c 13f9287b55b6356b4b1845410382d6bede203ceb29ef69388a4a3d007ffacbe5
 F src/vdbemem.c 0012d5f01cc866833847c2f3ae4c318ac53a1cb3d28acad9c35e688039464cf0
 F src/vdbesort.c 237840ca1947511fa59bd4e18b9eeae93f2af2468c34d2427b059f896230a547
@@ -809,10 +818,10 @@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf8
 F src/vdbevtab.c 2143db7db0ceed69b21422581f434baffc507a08d831565193a7a02882a1b6a7
 F src/vtab.c 154725ebecd3bc02f7fbd7ad3974334f73fff76e02a964e828e48a7c5fb7efff
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 69aa3ce60b2862a24cd86bb528e653e2137388ead258ef64db49ec9038807f5f
+F src/wal.c e5247a3406531b705b44630e9ccf9ca0e5c74955ef19c06fbb146d765c500c20
 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452
 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2
-F src/where.c 1fdc69ce1333e9bd6d7d3df9fa5af1373a3f5bfdd52108d1dbc0ca85a55f777e
+F src/where.c dad4b472b8dc886c0962409e70978568c06b5108bcf3dcd4938a9305a3540440
 F src/whereInt.h 4b38c5889514e3aead3f27d0ee9a26e47c3f150efc59e2a8b4e3bc8835e4d7a1
 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1
 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00
@@ -1220,7 +1229,7 @@ F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c
 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634
 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830
 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2
-F test/fuzzcheck.c 69b8549e112fb815931a8c14c7955a0c407ae91a79356eecb82458384f2cb989
+F test/fuzzcheck.c 442b0b02146bb905c47c4ee1388949e7fd9d27acf22d9535e9abf216e582b956
 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517
 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f
 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba
@@ -1321,15 +1330,17 @@ F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd7
 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa
 F test/json/README.md 63e3e589e1df8fd3cc1588ba1faaff659214003f8b77a15af5c6452b35e30ee2
 F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f
+F test/json/json-q1-b.txt 1e180fe6491efab307e318b22879e3a736ac9a96539bbde7911a13ee5b33abc7
 F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307
-F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x
-F test/json101.test bc05d2476fd6f7ead31ec05b43d1b24b2b193ae112fd8f0d2ed56d9a904f9fa5
-F test/json102.test 4c69694773a470f1fda34e5f4ba24920b35184fb66050b450fc2ef9ab5ad310b
+F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x
+F test/json101.test 70587d7d35ef9e2126364ba70f0c951f70827cfbd28649d779ff3df7e8f87547
+F test/json102.test 557a46e16df1aa9bdbc4076a71a45814ea0e7503d6621d87d42a8c04cbc2b0ef
 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe
 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1
-F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d
-F test/json501.test c419deb835b70c1a2c8532936927bcc1146730328edd2052276715bfd209724d
-F test/json502.test 98c38e3c4573841028a1381dfb81d4c3f9b105d39668167da10d055e503f6d0b
+F test/json105.test e64a8d73677fbae67886642cd5076e2ef3efe89f8483b87595cf9c030216c9bd
+F test/json501.test ab168a12eb6eb14d479f8c1cdae3ac062fd5a4679f17f976e96f1af518408330
+F test/json502.test 3c697e506fc38ccb455b49660b21b6e62e08ede0f2d0c869a7d171e17809093c
+F test/jsonb01.test cace70765b36a36aec9a85a41ea65667d3bbf647d4400ddc3ac76f8fe7d94f90
 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff
 F test/kvtest.c 6e0228409ea7ca0497dad503fbd109badb5e59545d131014b6aaac68b56f484a
 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63
@@ -1652,7 +1663,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163
 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc
 F test/tester.tcl 68454ef88508c196d19e8694daa27bff7107a91857799eaa12f417188ae53ede
-F test/testrunner.tcl 8a6721213bce1cfd3b33e1588cc6431143d96b98819206bf91f5a205fbb150d4
+F test/testrunner.tcl e18d71f2e797da808ba6d31335e504ed6b2791581b89287a72b697a2f31b1ea1
 F test/testrunner_data.tcl e4d5017290a6d5c11785e36cc94c67d8bb950c8cdc2dbe4c1db2a3a583812560
 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899
 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502
@@ -2142,9 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d65d9579ee6e1f2270917213d2b6bc25b66182a33176b2df50c691e02e612b0c
-Q +e9951ede184ce07cf725152723d795f299922460715ab76225cd3071bf0f18ee
-R 9e56423e4aab5da81ec6b75e9ca1e2d0
-U larrybr
-Z 8a82498f1d45fd5c83b1705256102297
+P be1864eac4eb75cc30bf98f73092c8608467f4bd956240df6a0cbea9f1e09e85
+R 6e221a94cd5525fe9f8a7e94ea467b60
+U drh
+Z 2ea9e875faf360824d045f495f8867aa
 # Remove this line to create a well-formed Fossil manifest.
@@ -1 +1 @@
-8e20645cc29335408c86a98b2141abc7d276abbffee728ffaf2960338d65bc60
+c96ebb086feb89341565cc52b970ae7799ce1327fe1ad4fc790f1b0dcaa6e229
@@ -64,7 +64,7 @@
 **     22       1     Min embedded payload fraction (must be 32)
 **     23       1     Min leaf payload fraction (must be 32)
 **     24       4     File change counter
-**     28       4     Reserved for future use
+**     28       4     The size of the database in pages
 **     32       4     First freelist page
 **     36       4     Number of freelist pages in the file
 **     40      60     15 4-byte meta values passed to higher layers
@@ -720,7 +720,7 @@ void sqlite3ColumnSetExpr(
 */
 Expr *sqlite3ColumnExpr(Table *pTab, Column *pCol){
  if( pCol->iDflt==0 ) return 0;
-  if( NEVER(!IsOrdinaryTable(pTab)) ) return 0;
+  if( !IsOrdinaryTable(pTab) ) return 0;
  if( NEVER(pTab->u.tab.pDfltList==0) ) return 0;
  if( NEVER(pTab->u.tab.pDfltList->nExpr<pCol->iDflt) ) return 0;
  return pTab->u.tab.pDfltList->a[pCol->iDflt-1].pExpr;
@@ -873,6 +873,9 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){
  if( db->pnBytesFreed==0 && (--pTable->nTabRef)>0 ) return;
  deleteTable(db, pTable);
 }
+void sqlite3DeleteTableGeneric(sqlite3 *db, void *pTable){
+  sqlite3DeleteTable(db, (Table*)pTable);
+}


 /*
@@ -1410,7 +1413,8 @@ void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){
 /*
 ** Clean up the data structures associated with the RETURNING clause.
 */
-static void sqlite3DeleteReturning(sqlite3 *db, Returning *pRet){
+static void sqlite3DeleteReturning(sqlite3 *db, void *pArg){
+  Returning *pRet = (Returning*)pArg;
  Hash *pHash;
  pHash = &(db->aDb[1].pSchema->trigHash);
  sqlite3HashInsert(pHash, pRet->zName, 0);
@@ -1452,8 +1456,7 @@ void sqlite3AddReturning(Parse *pParse, ExprList *pList){
  pParse->u1.pReturning = pRet;
  pRet->pParse = pParse;
  pRet->pReturnEL = pList;
-  sqlite3ParserAddCleanup(pParse,
-     (void(*)(sqlite3*,void*))sqlite3DeleteReturning, pRet);
+  sqlite3ParserAddCleanup(pParse, sqlite3DeleteReturning, pRet);
  testcase( pParse->earlyCleanup );
  if( db->mallocFailed ) return;
  sqlite3_snprintf(sizeof(pRet->zName), pRet->zName,
@@ -5692,4 +5695,7 @@ void sqlite3WithDelete(sqlite3 *db, With *pWith){
    sqlite3DbFree(db, pWith);
  }
 }
+void sqlite3WithDeleteGeneric(sqlite3 *db, void *pWith){
+  sqlite3WithDelete(db, (With*)pWith);
+}
 #endif /* !defined(SQLITE_OMIT_CTE) */
@@ -1223,9 +1223,7 @@ void sqlite3ExprAddFunctionOrderBy(
  assert( ExprUseXList(pExpr) );
  if( pExpr->x.pList==0 || NEVER(pExpr->x.pList->nExpr==0) ){
    /* Ignore ORDER BY on zero-argument aggregates */
-    sqlite3ParserAddCleanup(pParse,
-        (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
-        pOrderBy);
+    sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, pOrderBy);
    return;
  }
  if( IsWindowFunc(pExpr) ){
@@ -1406,6 +1404,9 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){
 void sqlite3ExprDelete(sqlite3 *db, Expr *p){
  if( p ) sqlite3ExprDeleteNN(db, p);
 }
+void sqlite3ExprDeleteGeneric(sqlite3 *db, void *p){
+  if( ALWAYS(p) ) sqlite3ExprDeleteNN(db, (Expr*)p);
+}

 /*
 ** Clear both elements of an OnOrUsing object
@@ -1431,9 +1432,7 @@ void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){
 ** pExpr to the pParse->pConstExpr list with a register number of 0.
 */
 void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){
-  sqlite3ParserAddCleanup(pParse,
-    (void(*)(sqlite3*,void*))sqlite3ExprDelete,
-    pExpr);
+  sqlite3ParserAddCleanup(pParse, sqlite3ExprDeleteGeneric, pExpr);
 }

 /* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the
@@ -2239,6 +2238,9 @@ static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){
 void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){
  if( pList ) exprListDeleteNN(db, pList);
 }
+void sqlite3ExprListDeleteGeneric(sqlite3 *db, void *pList){
+  if( ALWAYS(pList) ) exprListDeleteNN(db, (ExprList*)pList);
+}

 /*
 ** Return the bitwise-OR of all Expr.flags fields in the given
@@ -244,6 +244,9 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = {
   0,                         /* bSmallMalloc */
   1,                         /* bExtraSchemaChecks */
   sizeof(LONGDOUBLE_TYPE)>8, /* bUseLongDouble */
+#ifdef SQLITE_DEBUG
+   0,                         /* bJsonSelfcheck */
+#endif
   0x7ffffffe,                /* mxStrlen */
   0,                         /* neverCorrupt */
   SQLITE_DEFAULT_LOOKASIDE,  /* szLookaside, nLookaside */
@@ -4659,6 +4659,28 @@ int sqlite3_test_control(int op, ...){
      break;
    }
 #endif
+
+    /* sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &onOff);
+    **
+    ** Activate or deactivate validation of JSONB that is generated from
+    ** text.  Off by default, as the validation is slow.  Validation is
+    ** only available if compiled using SQLITE_DEBUG.
+    **
+    ** If onOff is initially 1, then turn it on.  If onOff is initially
+    ** off, turn it off.  If onOff is initially -1, then change onOff
+    ** to be the current setting.
+    */
+    case SQLITE_TESTCTRL_JSON_SELFCHECK: {
+#if defined(SQLITE_DEBUG)
+      int *pOnOff = va_arg(ap, int*);
+      if( *pOnOff<0 ){
+        *pOnOff = sqlite3Config.bJsonSelfcheck;
+      }else{
+        sqlite3Config.bJsonSelfcheck = (u8)((*pOnOff)&0xff);
+      }
+#endif
+      break;
+    }
  }
  va_end(ap);
 #endif /* SQLITE_UNTESTABLE */
@@ -4595,7 +4595,20 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){
      pShmNode->isUnlocked = 1;
      rc = SQLITE_READONLY_CANTINIT;
    }else{
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+      /* Do not use a blocking lock here. If the lock cannot be obtained
+      ** immediately, it means some other connection is truncating the
+      ** *-shm file. And after it has done so, it will not release its
+      ** lock, but only downgrade it to a shared lock. So no point in
+      ** blocking here. The call below to obtain the shared DMS lock may 
+      ** use a blocking lock. */
+      int iSaveTimeout = pDbFd->iBusyTimeout;
+      pDbFd->iBusyTimeout = 0;
+#endif
      rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1);
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+      pDbFd->iBusyTimeout = iSaveTimeout;
+#endif
      /* The first connection to attach must truncate the -shm file.  We
      ** truncate to 3 bytes (an arbitrary small number, less than the
      ** -shm header size) rather than 0 as a system debugging aid, to
@@ -1779,7 +1779,8 @@ void sqlite3Pragma(
          if( pVTab->pModule->iVersion<4 ) continue;
          if( pVTab->pModule->xIntegrity==0 ) continue;
          sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick);
-          sqlite3VdbeAppendP4(v, pTab, P4_TABLE);
+          pTab->nTabRef++;
+          sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF);
          a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v);
          integrityCheckResultRow(v);
          sqlite3VdbeJumpHere(v, a1);
@@ -1369,7 +1369,7 @@ void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){


 /*****************************************************************************
-** Reference counted string storage
+** Reference counted string/blob storage
 *****************************************************************************/

 /*
@@ -184,6 +184,9 @@ Select *sqlite3SelectNew(
 void sqlite3SelectDelete(sqlite3 *db, Select *p){
  if( OK_IF_ALWAYS_TRUE(p) ) clearSelect(db, p, 1);
 }
+void sqlite3SelectDeleteGeneric(sqlite3 *db, void *p){
+  if( ALWAYS(p) ) clearSelect(db, (Select*)p, 1);
+}

 /*
 ** Return a pointer to the right-most SELECT statement in a compound.
@@ -3204,9 +3207,7 @@ multi_select_end:
  pDest->iSdst = dest.iSdst;
  pDest->nSdst = dest.nSdst;
  if( pDelete ){
-    sqlite3ParserAddCleanup(pParse,
-        (void(*)(sqlite3*,void*))sqlite3SelectDelete,
-        pDelete);
+    sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pDelete);
  }
  return rc;
 }
@@ -3757,8 +3758,7 @@ static int multiSelectOrderBy(
  /* Make arrangements to free the 2nd and subsequent arms of the compound
  ** after the parse has finished */
  if( pSplit->pPrior ){
-    sqlite3ParserAddCleanup(pParse,
-       (void(*)(sqlite3*,void*))sqlite3SelectDelete, pSplit->pPrior);
+    sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pSplit->pPrior);
  }
  pSplit->pPrior = pPrior;
  pPrior->pNext = pSplit;
@@ -4579,9 +4579,7 @@ static int flattenSubquery(
    Table *pTabToDel = pSubitem->pTab;
    if( pTabToDel->nTabRef==1 ){
      Parse *pToplevel = sqlite3ParseToplevel(pParse);
-      sqlite3ParserAddCleanup(pToplevel,
-         (void(*)(sqlite3*,void*))sqlite3DeleteTable,
-         pTabToDel);
+      sqlite3ParserAddCleanup(pToplevel, sqlite3DeleteTableGeneric, pTabToDel);
      testcase( pToplevel->earlyCleanup );
    }else{
      pTabToDel->nTabRef--;
@@ -5628,8 +5626,7 @@ static struct Cte *searchWith(
 With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){
  if( pWith ){
    if( bFree ){
-      pWith = (With*)sqlite3ParserAddCleanup(pParse,
-                      (void(*)(sqlite3*,void*))sqlite3WithDelete,
+      pWith = (With*)sqlite3ParserAddCleanup(pParse, sqlite3WithDeleteGeneric,
                      pWith);
      if( pWith==0 ) return 0;
    }
@@ -7037,7 +7034,8 @@ static SrcItem *isSelfJoinView(
 /*
 ** Deallocate a single AggInfo object
 */
-static void agginfoFree(sqlite3 *db, AggInfo *p){
+static void agginfoFree(sqlite3 *db, void *pArg){
+  AggInfo *p = (AggInfo*)pArg;
  sqlite3DbFree(db, p->aCol);
  sqlite3DbFree(db, p->aFunc);
  sqlite3DbFreeNN(db, p);
@@ -7291,9 +7289,8 @@ int sqlite3Select(
        sqlite3TreeViewExprList(0, p->pOrderBy, 0, "ORDERBY");
      }
 #endif   
-      sqlite3ParserAddCleanup(pParse,
-        (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
-        p->pOrderBy);
+      sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric,
+                              p->pOrderBy);
      testcase( pParse->earlyCleanup );
      p->pOrderBy = 0;
    }
@@ -7485,9 +7482,8 @@ int sqlite3Select(
    ){
      TREETRACE(0x800,pParse,p,
                ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1));
-      sqlite3ParserAddCleanup(pParse,
-         (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
-         pSub->pOrderBy);
+      sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric,
+                              pSub->pOrderBy);
      pSub->pOrderBy = 0;
    }

@@ -8016,8 +8012,7 @@ int sqlite3Select(
    */
    pAggInfo = sqlite3DbMallocZero(db, sizeof(*pAggInfo) );
    if( pAggInfo ){
-      sqlite3ParserAddCleanup(pParse,
-          (void(*)(sqlite3*,void*))agginfoFree, pAggInfo);
+      sqlite3ParserAddCleanup(pParse, agginfoFree, pAggInfo);
      testcase( pParse->earlyCleanup );
    }
    if( db->mallocFailed ){
@@ -817,11 +817,12 @@ static char *local_getline(char *zLine, FILE *in){
 */
 #ifndef SQLITE_SHELL_FIDDLE
 static char *one_input_line(FILE *in, char *zPrior, int isContinuation){
+  char *zPrompt;
  char *zResult;
  if( in!=0 ){
    zResult = local_getline(zPrior, in);
  }else{
-    char *zPrompt = isContinuation ? CONTINUATION_PROMPT : mainPrompt;
+    zPrompt = isContinuation ? CONTINUATION_PROMPT : mainPrompt;
 #if SHELL_USE_LOCAL_GETLINE
    sputz(stdout, zPrompt);
    fflush(stdout);
@@ -10755,6 +10756,7 @@ static int do_meta_command(char *zLine, ShellState *p){
    {"fk_no_action",       SQLITE_TESTCTRL_FK_NO_ACTION, 0, "BOOLEAN"       },
    {"imposter",         SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"},
    {"internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,""          },
+    {"json_selfcheck",     SQLITE_TESTCTRL_JSON_SELFCHECK ,0,"BOOLEAN"      },
    {"localtime_fault",    SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN"      },
    {"never_corrupt",      SQLITE_TESTCTRL_NEVER_CORRUPT,1, "BOOLEAN"       },
    {"optimizations",      SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK"   },
@@ -10973,6 +10975,16 @@ static int do_meta_command(char *zLine, ShellState *p){
            isOk = 3;
          }
          break;
+        case SQLITE_TESTCTRL_JSON_SELFCHECK:
+          if( nArg==2 ){
+            rc2 = -1;
+            isOk = 1;
+          }else{
+            rc2 = booleanValue(azArg[2]);
+            isOk = 3;
+          }
+          sqlite3_test_control(testctrl, &rc2);
+          break;
      }
    }
    if( isOk==0 && iCtrl>=0 ){
@@ -12060,10 +12072,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
    }else if( cli_strcmp(z,"-init")==0 ){
      zInitFile = cmdline_option_value(argc, argv, ++i);
    }else if( cli_strcmp(z,"-interactive")==0 ){
-      /* Need to check for interactive override here to so that it can
-      ** affect console setup (for Windows only) and testing thereof.
-      */
-      stdin_is_interactive = 1;
    }else if( cli_strcmp(z,"-batch")==0 ){
      /* Need to check for batch mode here to so we can avoid printing
      ** informational messages (like from process_sqliterc) before
@@ -12337,7 +12345,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
            sqlite3_libversion(), sqlite3_sourceid(), 8*(int)sizeof(char*));
      return 0;
    }else if( cli_strcmp(z,"-interactive")==0 ){
-      /* already handled */
+      /* Need to check for interactive override here to so that it can
+      ** affect console setup (for Windows only) and testing thereof.
+      */
+      stdin_is_interactive = 1;
    }else if( cli_strcmp(z,"-batch")==0 ){
      /* already handled */
    }else if( cli_strcmp(z,"-utf8")==0 ){
@@ -8300,6 +8300,7 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_ASSERT                  12
 #define SQLITE_TESTCTRL_ALWAYS                  13
 #define SQLITE_TESTCTRL_RESERVE                 14  /* NOT USED */
+#define SQLITE_TESTCTRL_JSON_SELFCHECK          14
 #define SQLITE_TESTCTRL_OPTIMIZATIONS           15
 #define SQLITE_TESTCTRL_ISKEYWORD               16  /* NOT USED */
 #define SQLITE_TESTCTRL_SCRATCHMALLOC           17  /* NOT USED */
@@ -2114,11 +2114,11 @@ struct FuncDestructor {
 #define MFUNCTION(zName, nArg, xPtr, xFunc) \
  {nArg, SQLITE_FUNC_BUILTIN|SQLITE_FUNC_CONSTANT|SQLITE_UTF8, \
   xPtr, 0, xFunc, 0, 0, 0, #zName, {0} }
-#define JFUNCTION(zName, nArg, bUseCache, bWS, bRS, iArg, xFunc) \
+#define JFUNCTION(zName, nArg, bUseCache, bWS, bRS, bJsonB, iArg, xFunc) \
  {nArg, SQLITE_FUNC_BUILTIN|SQLITE_DETERMINISTIC|SQLITE_FUNC_CONSTANT|\
   SQLITE_UTF8|((bUseCache)*SQLITE_FUNC_RUNONLY)|\
   ((bRS)*SQLITE_SUBTYPE)|((bWS)*SQLITE_RESULT_SUBTYPE), \
-   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} }
+   SQLITE_INT_TO_PTR(iArg|((bJsonB)*JSON_BLOB)),0,xFunc,0, 0, 0, #zName, {0} }
 #define INLINE_FUNC(zName, nArg, iArg, mFlags) \
  {nArg, SQLITE_FUNC_BUILTIN|\
   SQLITE_UTF8|SQLITE_FUNC_INLINE|SQLITE_FUNC_CONSTANT|(mFlags), \
@@ -4116,6 +4116,9 @@ struct sqlite3_str {
 **
 **   3.  Make a (read-only) copy of a read-only RCStr string using
 **       sqlite3RCStrRef().
+**
+** "String" is in the name, but an RCStr object can also be used to hold
+** binary data.
 */
 struct RCStr {
  u64 nRCRef;            /* Number of references */
@@ -4174,6 +4177,9 @@ struct Sqlite3Config {
  u8 bSmallMalloc;                  /* Avoid large memory allocations if true */
  u8 bExtraSchemaChecks;            /* Verify type,name,tbl_name in schema */
  u8 bUseLongDouble;                /* Make use of long double */
+#ifdef SQLITE_DEBUG
+  u8 bJsonSelfcheck;                /* Double-check JSON parsing */
+#endif
  int mxStrlen;                     /* Maximum string length */
  int neverCorrupt;                 /* Database is always well-formed */
  int szLookaside;                  /* Default lookaside buffer size */
@@ -4800,6 +4806,7 @@ void sqlite3ExprOrderByAggregateError(Parse*,Expr*);
 void sqlite3ExprFunctionUsable(Parse*,const Expr*,const FuncDef*);
 void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32);
 void sqlite3ExprDelete(sqlite3*, Expr*);
+void sqlite3ExprDeleteGeneric(sqlite3*,void*);
 void sqlite3ExprDeferredDelete(Parse*, Expr*);
 void sqlite3ExprUnmapAndDelete(Parse*, Expr*);
 ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*);
@@ -4809,6 +4816,7 @@ void sqlite3ExprListSetSortOrder(ExprList*,int,int);
 void sqlite3ExprListSetName(Parse*,ExprList*,const Token*,int);
 void sqlite3ExprListSetSpan(Parse*,ExprList*,const char*,const char*);
 void sqlite3ExprListDelete(sqlite3*, ExprList*);
+void sqlite3ExprListDeleteGeneric(sqlite3*,void*);
 u32 sqlite3ExprListFlags(const ExprList*);
 int sqlite3IndexHasDuplicateRootPage(Index*);
 int sqlite3Init(sqlite3*, char**);
@@ -4899,6 +4907,7 @@ void sqlite3CreateView(Parse*,Token*,Token*,Token*,ExprList*,Select*,int,int);
 void sqlite3DropTable(Parse*, SrcList*, int, int);
 void sqlite3CodeDropTable(Parse*, Table*, int, int);
 void sqlite3DeleteTable(sqlite3*, Table*);
+void sqlite3DeleteTableGeneric(sqlite3*, void*);
 void sqlite3FreeIndex(sqlite3*, Index*);
 #ifndef SQLITE_OMIT_AUTOINCREMENT
  void sqlite3AutoincrementBegin(Parse *pParse);
@@ -4935,6 +4944,7 @@ int sqlite3Select(Parse*, Select*, SelectDest*);
 Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*,
                         Expr*,ExprList*,u32,Expr*);
 void sqlite3SelectDelete(sqlite3*, Select*);
+void sqlite3SelectDeleteGeneric(sqlite3*,void*);
 Table *sqlite3SrcListLookup(Parse*, SrcList*);
 int sqlite3IsReadOnly(Parse*, Table*, Trigger*);
 void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
@@ -5507,6 +5517,7 @@ const char *sqlite3JournalModename(int);
  void sqlite3CteDelete(sqlite3*,Cte*);
  With *sqlite3WithAdd(Parse*,With*,Cte*);
  void sqlite3WithDelete(sqlite3*,With*);
+  void sqlite3WithDeleteGeneric(sqlite3*,void*);
  With *sqlite3WithPush(Parse*, With*, u8);
 #else
 # define sqlite3CteNew(P,T,E,S)   ((void*)0)
@@ -8180,9 +8180,10 @@ case OP_VCheck: {             /* out2 */

  pOut = &aMem[pOp->p2];
  sqlite3VdbeMemSetNull(pOut);  /* Innocent until proven guilty */
-  assert( pOp->p4type==P4_TABLE );
+  assert( pOp->p4type==P4_TABLEREF );
  pTab = pOp->p4.pTab;
  assert( pTab!=0 );
+  assert( pTab->nTabRef>0 );
  assert( IsVirtual(pTab) );
  if( pTab->u.vtab.p==0 ) break;
  pVtab = pTab->u.vtab.p->pVtab;
@@ -8191,13 +8192,11 @@ case OP_VCheck: {             /* out2 */
  assert( pModule!=0 );
  assert( pModule->iVersion>=4 );
  assert( pModule->xIntegrity!=0 );
-  pTab->nTabRef++;
  sqlite3VtabLock(pTab->u.vtab.p);
  assert( pOp->p1>=0 && pOp->p1<db->nDb );
  rc = pModule->xIntegrity(pVtab, db->aDb[pOp->p1].zDbSName, pTab->zName,
                           pOp->p3, &zErr);
  sqlite3VtabUnlock(pTab->u.vtab.p);
-  sqlite3DeleteTable(db, pTab);
  if( rc ){
    sqlite3_free(zErr);
    goto abort_due_to_error;
@@ -8322,6 +8321,7 @@ case OP_VColumn: {           /* ncycle */
  const sqlite3_module *pModule;
  Mem *pDest;
  sqlite3_context sContext;
+  FuncDef nullFunc;

  VdbeCursor *pCur = p->apCsr[pOp->p1];
  assert( pCur!=0 );
@@ -8339,6 +8339,9 @@ case OP_VColumn: {           /* ncycle */
  memset(&sContext, 0, sizeof(sContext));
  sContext.pOut = pDest;
  sContext.enc = encoding;
+  nullFunc.pUserData = 0;
+  nullFunc.funcFlags = SQLITE_RESULT_SUBTYPE;
+  sContext.pFunc = &nullFunc;
  assert( pOp->p5==OPFLAG_NOCHNG || pOp->p5==0 );
  if( pOp->p5 & OPFLAG_NOCHNG ){
    sqlite3VdbeMemSetNull(pDest);
@@ -126,6 +126,7 @@ typedef struct VdbeOpList VdbeOpList;
 #define P4_INT64      (-13) /* P4 is a 64-bit signed integer */
 #define P4_INTARRAY   (-14) /* P4 is a vector of 32-bit integers */
 #define P4_FUNCCTX    (-15) /* P4 is a pointer to an sqlite3_context object */
+#define P4_TABLEREF   (-16) /* Like P4_TABLE, but reference counted */

 /* Error message codes for OP_Halt */
 #define P5_ConstraintNotNull 1
@@ -950,9 +950,8 @@ int sqlite3_step(sqlite3_stmt *pStmt){
 void *sqlite3_user_data(sqlite3_context *p){
 #ifdef SQLITE_ENABLE_API_ARMOR
  if( p==0 ) return 0;
-#else
-  assert( p && p->pFunc );
 #endif
+  assert( p && p->pFunc );
  return p->pFunc->pUserData;
 }

@@ -1400,6 +1400,10 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){
      if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4);
      break;
    }
+    case P4_TABLEREF: {
+      if( db->pnBytesFreed==0 ) sqlite3DeleteTable(db, (Table*)p4);
+      break;
+    }
  }
 }

@@ -2004,6 +2004,19 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){
 }

 #ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+
+
+/*
+** Attempt to enable blocking locks that block for nMs ms. Return 1 if 
+** blocking locks are successfully enabled, or 0 otherwise.
+*/
+static int walEnableBlockingMs(Wal *pWal, int nMs){
+  int rc = sqlite3OsFileControl(
+      pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&nMs
+  );
+  return (rc==SQLITE_OK);
+}
+
 /*
 ** Attempt to enable blocking locks. Blocking locks are enabled only if (a)
 ** they are supported by the VFS, and (b) the database handle is configured
@@ -2015,11 +2028,7 @@ static int walEnableBlocking(Wal *pWal){
  if( pWal->db ){
    int tmout = pWal->db->busyTimeout;
    if( tmout ){
-      int rc;
-      rc = sqlite3OsFileControl(
-          pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&tmout
-      );
-      res = (rc==SQLITE_OK);
+      res = walEnableBlockingMs(pWal, tmout);
    }
  }
  return res;
@@ -2068,20 +2077,10 @@ void sqlite3WalDb(Wal *pWal, sqlite3 *db){
  pWal->db = db;
 }

-/*
-** Take an exclusive WRITE lock. Blocking if so configured.
-*/
-static int walLockWriter(Wal *pWal){
-  int rc;
-  walEnableBlocking(pWal);
-  rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1);
-  walDisableBlocking(pWal);
-  return rc;
-}
 #else
 # define walEnableBlocking(x) 0
 # define walDisableBlocking(x)
-# define walLockWriter(pWal) walLockExclusive((pWal), WAL_WRITE_LOCK, 1)
+# define walEnableBlockingMs(pWal, ms) 0
 # define sqlite3WalDb(pWal, db)
 #endif   /* ifdef SQLITE_ENABLE_SETLK_TIMEOUT */

@@ -2682,7 +2681,9 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
      }
    }else{
      int bWriteLock = pWal->writeLock;
-      if( bWriteLock || SQLITE_OK==(rc = walLockWriter(pWal)) ){
+      if( bWriteLock 
+       || SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) 
+      ){
        pWal->writeLock = 1;
        if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){
          badHdr = walIndexTryHdr(pWal, pChanged);
@@ -2690,7 +2691,8 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
            /* If the wal-index header is still malformed even while holding
            ** a WRITE lock, it can only mean that the header is corrupted and
            ** needs to be reconstructed.  So run recovery to do exactly that.
-            */
+            ** Disable blocking locks first.  */
+            walDisableBlocking(pWal);
            rc = walIndexRecover(pWal);
            *pChanged = 1;
          }
@@ -2957,6 +2959,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
  int i;                          /* Loop counter */
  int rc = SQLITE_OK;             /* Return code  */
  u32 mxFrame;                    /* Wal frame to lock to */
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+  int nBlockTmout = 0;
+#endif

  assert( pWal->readLock<0 );     /* Not currently locked */

@@ -2987,6 +2992,19 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
      return SQLITE_PROTOCOL;
    }
    if( cnt>=10 ) nDelay = (cnt-9)*(cnt-9)*39;
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+    /* In SQLITE_ENABLE_SETLK_TIMEOUT builds, configure the file-descriptor
+    ** to block for locks for approximately nDelay us. This affects three
+    ** locks: (a) the shared lock taken on the DMS slot in os_unix.c (if
+    ** using os_unix.c), (b) the WRITER lock taken in walIndexReadHdr() if the
+    ** first attempted read fails, and (c) the shared lock taken on the DMS
+    ** slot in os_unix.c. All three of these locks are attempted from within 
+    ** the call to walIndexReadHdr() below.  */
+    nBlockTmout = (nDelay+998) / 1000;
+    if( !useWal && walEnableBlockingMs(pWal, nBlockTmout) ){
+      nDelay = 1;
+    }
+#endif
    sqlite3OsSleep(pWal->pVfs, nDelay);
  }

@@ -2995,6 +3013,10 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
    if( pWal->bShmUnreliable==0 ){
      rc = walIndexReadHdr(pWal, pChanged);
    }
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+    walDisableBlocking(pWal);
+    if( rc==SQLITE_BUSY_TIMEOUT ) rc = SQLITE_BUSY;
+#endif
    if( rc==SQLITE_BUSY ){
      /* If there is not a recovery running in another thread or process
      ** then convert BUSY errors to WAL_RETRY.  If recovery is known to
@@ -3109,9 +3131,12 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
    return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT;
  }

+  (void)walEnableBlockingMs(pWal, nBlockTmout);
  rc = walLockShared(pWal, WAL_READ_LOCK(mxI));
+  walDisableBlocking(pWal);
  if( rc ){
-    return rc==SQLITE_BUSY ? WAL_RETRY : rc;
+    assert( (rc&0xFF)!=SQLITE_BUSY||rc==SQLITE_BUSY||rc==SQLITE_BUSY_TIMEOUT );
+    return (rc&0xFF)==SQLITE_BUSY ? WAL_RETRY : rc;
  }
  /* Now that the read-lock has been obtained, check that neither the
  ** value in the aReadMark[] array or the contents of the wal-index
@@ -2020,7 +2020,8 @@ static int whereRangeScanEst(
          ** sample, then assume they are 4x more selective.  This brings
          ** the estimated selectivity more in line with what it would be
          ** if estimated without the use of STAT4 tables. */
-          if( iLwrIdx==iUprIdx ) nNew -= 20;  assert( 20==sqlite3LogEst(4) );
+          if( iLwrIdx==iUprIdx ){ nNew -= 20; }
+          assert( 20==sqlite3LogEst(4) );
        }else{
          nNew = 10;        assert( 10==sqlite3LogEst(2) );
        }
@@ -1267,6 +1267,8 @@ int runCombinedDbSqlInput(
  }
  sqlite3_limit(cx.db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH, 100);
  sqlite3_hard_heap_limit64(heapLimit);
+  rc = 1;
+  sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &rc);

  if( nDb>=20 && aDb[18]==2 && aDb[19]==2 ){
    aDb[18] = aDb[19] = 1;
@@ -0,0 +1,24 @@
+.mode qbox
+.timer on
+.param set $label 'q87'
+SELECT rowid, x->>$label FROM data1 WHERE x->>$label IS NOT NULL;
+
+CREATE TEMP TABLE t2(x JSON TEXT);
+WITH RECURSIVE
+  c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<25000),
+  array1(y) AS (
+    SELECT json_group_array(
+             json_object('x',x,'y',random(),'z',hex(randomblob(50)))
+           )
+      FROM c
+  ),
+  c2(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c2 WHERE n<5)
+INSERT INTO t2(x)
+  SELECT jsonb_object('a',n,'b',n*2,'c',y,'d',3,'e',5,'f',6) FROM array1, c2;
+CREATE INDEX t2x1 ON t2(x->>'a');
+CREATE INDEX t2x2 ON t2(x->>'b');
+CREATE INDEX t2x3 ON t2(x->>'e');
+CREATE INDEX t2x4 ON t2(x->>'f');
+UPDATE t2 SET x=jsonb_replace(x,'$.f',(x->>'f')+1);
+UPDATE t2 SET x=jsonb_set(x,'$.e',(x->>'f')-1);
+UPDATE t2 SET x=jsonb_remove(x,'$.d');
@@ -40,6 +40,7 @@ doCachegrind=1
 doVdbeProfile=0
 doWal=1
 doDiff=1
+doJsonB=0
 while test "$1" != ""; do
  case $1 in
    --nodiff)
@@ -54,6 +55,9 @@ while test "$1" != ""; do
    --gcc7)
        CC=gcc-7
        ;;
+    --jsonb)
+        doJsonB=1
+        ;;
    -*)
        CC_OPTS="$CC_OPTS $1"
        ;;
@@ -69,12 +73,18 @@ rm -f cachegrind.out.* jsonshell
 $CC -g -Os -Wall -I. $CC_OPTS ./shell.c ./sqlite3.c -o jsonshell -ldl -lpthread
 ls -l jsonshell | tee -a summary-$NAME.txt
 home=`echo $0 | sed -e 's,/[^/]*$,,'`
-echo ./jsonshell json100mb.db "<$home/json-q1.txt"
-valgrind --tool=cachegrind ./jsonshell json100mb.db <$home/json-q1.txt \
-      2>&1 | tee -a summary-$NAME.txt
+if test $doJsonB -eq 1; then
+  echo ./jsonshell json100mb_b.db "<$home/json-q1-b.txt"
+  valgrind --tool=cachegrind ./jsonshell json100mb_b.db <$home/json-q1-b.txt \
+        2>&1 | tee -a summary-$NAME.txt
+else
+  echo ./jsonshell json100mb.db "<$home/json-q1.txt"
+  valgrind --tool=cachegrind ./jsonshell json100mb.db <$home/json-q1.txt \
+        2>&1 | tee -a summary-$NAME.txt
+fi
 cg_anno.tcl cachegrind.out.* >jout-$NAME.txt
 echo '*****************************************************' >>jout-$NAME.txt
 sed 's/^[0-9=-]\{9\}/==00000==/' summary-$NAME.txt >>jout-$NAME.txt
-if test "$NAME" != "$BASELINE"; then
+if test "$NAME" != "$BASELINE" -a $doDiff -ne 0; then
  fossil xdiff --tk -c 20 jout-$BASELINE.txt jout-$NAME.txt
 fi
@@ -36,6 +36,9 @@ do_execsql_test json101-1.2 {
 do_catchsql_test json101-1.3 {
  SELECT json_array(1,printf('%.1000c','x'),x'abcd',3);
 } {1 {JSON cannot hold BLOB values}}
+do_catchsql_test json101-1.3b {
+  SELECT jsonb_array(1,printf('%.1000c','x'),x'abcd',3);
+} {1 {JSON cannot hold BLOB values}}
 do_execsql_test json101-1.4 {
  SELECT json_array(-9223372036854775808,9223372036854775807,0,1,-1,
                    0.0, 1.0, -1.0, -1e99, +2e100,
@@ -47,36 +50,83 @@ do_execsql_test json101-1.4 {
                    'abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ',
                    99);
 } {[-9223372036854775808,9223372036854775807,0,1,-1,0.0,1.0,-1.0,-1.0e+99,2.0e+100,"one","two","three",4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,null,21,22,23,24,25,26,27,28,29,30,31,"abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ",99]}
+do_execsql_test json101-1.4b {
+  SELECT json(jsonb_array(-9223372036854775808,9223372036854775807,0,1,-1,
+                    0.0, 1.0, -1.0, -1e99, +2e100,
+                    'one','two','three',
+                    4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
+                    19, NULL, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
+                    'abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ',
+                    'abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ',
+                    'abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ',
+                    99));
+} {[-9223372036854775808,9223372036854775807,0,1,-1,0.0,1.0,-1.0,-1.0e+99,2.0e+100,"one","two","three",4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,null,21,22,23,24,25,26,27,28,29,30,31,"abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ",99]}

 do_execsql_test json101-2.1 {
  SELECT json_object('a',1,'b',2.5,'c',null,'d','String Test');
 } {{{"a":1,"b":2.5,"c":null,"d":"String Test"}}}
+do_execsql_test json101-2.1b {
+  SELECT json(jsonb_object('a',1,'b',2.5,'c',null,'d','String Test'));
+} {{{"a":1,"b":2.5,"c":null,"d":"String Test"}}}
 do_catchsql_test json101-2.2 {
  SELECT json_object('a',printf('%.1000c','x'),2,2.5);
 } {1 {json_object() labels must be TEXT}}
+do_catchsql_test json101-2.2b {
+  SELECT jsonb_object('a',printf('%.1000c','x'),2,2.5);
+} {1 {json_object() labels must be TEXT}}
+do_execsql_test json101-2.2.2 {
+  SELECT json_object('a',json_array('xyx',77,4.5),'x',2.5);
+} {{{"a":["xyx",77,4.5],"x":2.5}}}
+do_execsql_test json101-2.2.2b {
+  SELECT json(jsonb_object('a',json_array('xyx',77,4.5),'x',2.5));
+} {{{"a":["xyx",77,4.5],"x":2.5}}}
+do_execsql_test json101-2.2.3 {
+  SELECT json_object('a',jsonb_array('xyx',77,4.5),'x',2.5);
+} {{{"a":["xyx",77,4.5],"x":2.5}}}
+do_execsql_test json101-2.2.3b {
+  SELECT json(jsonb_object('a',jsonb_array('xyx',77,4.5),'x',2.5));
+} {{{"a":["xyx",77,4.5],"x":2.5}}}
 do_catchsql_test json101-2.3 {
  SELECT json_object('a',1,'b');
 } {1 {json_object() requires an even number of arguments}}
 do_catchsql_test json101-2.4 {
  SELECT json_object('a',printf('%.1000c','x'),'b',x'abcd');
 } {1 {JSON cannot hold BLOB values}}
+do_execsql_test json101-2.5 {
+  SELECT json_object('a',printf('%.10c','x'),'b',jsonb_array(1,2,3));
+} {{{"a":"xxxxxxxxxx","b":[1,2,3]}}}

 do_execsql_test json101-3.1 {
  SELECT json_replace('{"a":1,"b":2}','$.a','[3,4,5]');
 } {{{"a":"[3,4,5]","b":2}}}
+do_execsql_test json101-3.1b {
+  SELECT json(jsonb_replace('{"a":1,"b":2}','$.a','[3,4,5]'));
+} {{{"a":"[3,4,5]","b":2}}}
 do_execsql_test json101-3.2 {
  SELECT json_replace('{"a":1,"b":2}','$.a',json('[3,4,5]'));
 } {{{"a":[3,4,5],"b":2}}}
+do_execsql_test json101-3.2b {
+  SELECT json_replace('{"a":1,"b":2}','$.a',jsonb('[3,4,5]'));
+} {{{"a":[3,4,5],"b":2}}}
 do_execsql_test json101-3.3 {
  SELECT json_type(json_set('{"a":1,"b":2}','$.b','{"x":3,"y":4}'),'$.b');
 } {text}
+do_execsql_test json101-3.3b {
+  SELECT json_type(jsonb_set('{"a":1,"b":2}','$.b','{"x":3,"y":4}'),'$.b');
+} {text}
 do_execsql_test json101-3.4 {
  SELECT json_type(json_set('{"a":1,"b":2}','$.b',json('{"x":3,"y":4}')),'$.b');
 } {object}
+do_execsql_test json101-3.4b {
+  SELECT json_type(jsonb_set('{"a":1,"b":2}','$.b',jsonb('{"x":3,"y":4}')),'$.b');
+} {object}
 ifcapable vtab {
-do_execsql_test json101-3.5 {
-  SELECT fullkey, atom, '|' FROM json_tree(json_set('{}','$.x',123,'$.x',456));
-} {{$} {} | {$.x} 456 |}
+  do_execsql_test json101-3.5 {
+    SELECT fullkey, atom, '|' FROM json_tree(json_set('{}','$.x',123,'$.x',456));
+  } {{$} {} | {$.x} 456 |}
+  do_execsql_test json101-3.5b {
+    SELECT fullkey, atom, '|' FROM json_tree(jsonb_set('{}','$.x',123,'$.x',456));
+  } {{$} {} | {$.x} 456 |}
 }

 # Per rfc7159, any JSON value is allowed at the top level, and whitespace
@@ -131,6 +181,13 @@ do_execsql_test json101-4.10 {
   WHERE json_extract(x,'$')<>x
     AND json_type(x) IN ('object','array');
 } {4}
+do_execsql_test json101-4.10b {
+  CREATE TABLE j1b AS SELECT jsonb(x) AS "x" FROM j1;
+  SELECT count(*) FROM j1b WHERE json_type(x) IN ('object','array');
+  SELECT json(x) FROM j1b
+   WHERE json_extract(x,'$')<>json(x)
+     AND json_type(x) IN ('object','array');
+} {4}

 do_execsql_test json101-5.1 {
  CREATE TABLE j2(id INTEGER PRIMARY KEY, json, src);
@@ -258,11 +315,17 @@ do_execsql_test json101-5.1 {
 	}
   ]','https://adobe.github.io/Spry/samples/data_region/JSONDataSetSample.html');
   SELECT count(*) FROM j2;
-} {3}
+   CREATE TABLE j2b(id INTEGER PRIMARY KEY, json, src);
+   INSERT INTO J2b(id,json,src) SELECT id, jsonb(json), src FROM j2;
+   SELECT count(*) FROM j2b;
+} {3 3}

 do_execsql_test json101-5.2 {
  SELECT id, json_valid(json), json_type(json), '|' FROM j2 ORDER BY id;
 } {1 1 object | 2 1 object | 3 1 array |}
+do_execsql_test json101-5.2b {
+  SELECT id, json_valid(json,5), json_type(json), '|' FROM j2b ORDER BY id;
+} {1 1 object | 2 1 object | 3 1 array |}

 ifcapable !vtab {
  finish_test
@@ -277,6 +340,12 @@ do_execsql_test json101-5.3 {
   WHERE fullkey!=(path || CASE WHEN typeof(key)=='integer' THEN '['||key||']'
                                ELSE '.'||key END);
 } {}
+do_execsql_test json101-5.3b {
+  SELECT j2b.rowid, jx.rowid, fullkey, path, key
+    FROM j2b, json_tree(j2b.json) AS jx
+   WHERE fullkey!=(path || CASE WHEN typeof(key)=='integer' THEN '['||key||']'
+                                ELSE '.'||key END);
+} {}
 do_execsql_test json101-5.4 {
  SELECT j2.rowid, jx.rowid, fullkey, path, key
    FROM j2, json_each(j2.json) AS jx
@@ -371,6 +440,13 @@ do_execsql_test json101-8.1 {
  UPDATE t8 SET b=json_array(a);
  SELECT b FROM t8;
 } {{["abc\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f !\"#xyz"]}}
+do_execsql_test json101-8.1b {
+  DROP TABLE IF EXISTS t8;
+  CREATE TABLE t8(a,b);
+  INSERT INTO t8(a) VALUES('abc' || char(1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35) || 'xyz');
+  UPDATE t8 SET b=jsonb_array(a);
+  SELECT json(b) FROM t8;
+} {{["abc\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f !\"#xyz"]}}
 do_execsql_test json101-8.2 {
  SELECT a=json_extract(b,'$[0]') FROM t8;
 } {1}
@@ -401,7 +477,7 @@ do_execsql_test json101-9.4 {
  SELECT json_quote(null);
 } {"null"}
 do_catchsql_test json101-9.5 {
-  SELECT json_quote(x'30313233');
+  SELECT json_quote(x'3031323334');
 } {1 {JSON cannot hold BLOB values}}
 do_catchsql_test json101-9.6 {
  SELECT json_quote(123,456)
@@ -769,14 +845,23 @@ do_execsql_test json101-12.100 {
        }
     }');
 } {}
+
 do_execsql_test json101-12.110 {
  SELECT json_remove(x, '$.settings.layer2."dis.legomenon".forceDisplay')
    FROM t12;
 } {{{"settings":{"layer2":{"hapax.legomenon":{"forceDisplay":true,"transliterate":true,"add.footnote":true,"summary.report":true},"dis.legomenon":{"transliterate":false,"add.footnote":false,"summary.report":true},"tris.legomenon":{"forceDisplay":true,"transliterate":false,"add.footnote":false,"summary.report":false}}}}}}
+do_execsql_test json101-12.110b {
+  SELECT json_remove(jsonb(x), '$.settings.layer2."dis.legomenon".forceDisplay')
+    FROM t12;
+} {{{"settings":{"layer2":{"hapax.legomenon":{"forceDisplay":true,"transliterate":true,"add.footnote":true,"summary.report":true},"dis.legomenon":{"transliterate":false,"add.footnote":false,"summary.report":true},"tris.legomenon":{"forceDisplay":true,"transliterate":false,"add.footnote":false,"summary.report":false}}}}}}
 do_execsql_test json101-12.120 {
  SELECT json_extract(x, '$.settings.layer2."tris.legomenon"."summary.report"')
    FROM t12;
 } {0}
+do_execsql_test json101-12.120b {
+  SELECT json_extract(jsonb(x), '$.settings.layer2."tris.legomenon"."summary.report"')
+    FROM t12;
+} {0}

 # 2018-01-26
 # ticket https://www.sqlite.org/src/tktview/80177f0c226ff54f6ddd41
@@ -840,16 +925,16 @@ do_execsql_test json101-14.170 {
 #
 do_execsql_test json101-15.100 {
  SELECT * FROM JSON_EACH('{"a":1, "b":2}');
-} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+} {a 1 integer 1 1 {} {$.a} {$} b 2 integer 2 5 {} {$.b} {$}}
 do_execsql_test json101-15.110 {
  SELECT xyz.* FROM JSON_EACH('{"a":1, "b":2}') AS xyz;
-} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+} {a 1 integer 1 1 {} {$.a} {$} b 2 integer 2 5 {} {$.b} {$}}
 do_execsql_test json101-15.120 {
  SELECT * FROM (JSON_EACH('{"a":1, "b":2}'));
-} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+} {a 1 integer 1 1 {} {$.a} {$} b 2 integer 2 5 {} {$.b} {$}}
 do_execsql_test json101-15.130 {
  SELECT xyz.* FROM (JSON_EACH('{"a":1, "b":2}')) AS xyz;
-} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+} {a 1 integer 1 1 {} {$.a} {$} b 2 integer 2 5 {} {$.b} {$}}

 # 2019-11-10
 # Mailing list bug report on the handling of surrogate pairs
@@ -889,7 +974,7 @@ do_execsql_test json101-18.4 {
 } {6}
 do_catchsql_test json101-18.5 {
  SELECT json_extract('{"":8}', '$.');
-} {1 {JSON path error near ''}}
+} {1 {bad JSON path: '$.'}}

 # 2022-08-29 https://sqlite.org/forum/forumpost/9b9e4716c0d7bbd1
 # This is not a problem specifically with JSON functions.  It is
@@ -1054,6 +1139,29 @@ do_execsql_test json101-23.2 {
    FROM (SELECT json_set('[]','$[#]',0,'$[#]',1) AS j);
 } {{[0,1]} 0 1}

-
+# Insert/Set/Replace where the path specifies substructure that
+# does not yet exist
+#
+proc tx x {return [string map [list ( \173 ) \175 ' \042 < \133 > \135] $x]}
+foreach {id start path ins set repl} {
+  1 {{}}       {$.a.b.c}     ('a':('b':('c':9)))      ('a':('b':('c':9)))      ()
+  2 {{a:4}}    {$.a.b.c}     ('a':4)                  ('a':4)                  ('a':4)
+  3 {{a:{}}}   {$.a.b.c}     ('a':('b':('c':9)))      ('a':('b':('c':9)))      ('a':())
+  4 {[0,1,2]}  {$[3].a[0].b} <0,1,2,('a':<('b':9)>)>  <0,1,2,('a':<('b':9)>)>  <0,1,2>
+  5 {[0,1,2]}  {$[1].a[0].b} <0,1,2>                  <0,1,2>                  <0,1,2>
+  6 {[0,{},2]} {$[1].a[0].b} <0,('a':<('b':9)>),2>    <0,('a':<('b':9)>),2>    <0,(),2>
+  7 {[0,1,2]}  {$[3][0].b}   <0,1,2,<('b':9)>>        <0,1,2,<('b':9)>>        <0,1,2>
+  8 {[0,1,2]}  {$[1][0].b}   <0,1,2>                  <0,1,2>                  <0,1,2>
+} {
+  do_execsql_test json101-24.$id.insert {
+    SELECT json_insert($start,$path,9);
+  } [list [tx $ins]]
+  do_execsql_test json101-24.$id.set {
+    SELECT json_set($start,$path,9);
+  } [list [tx $set]]
+  do_execsql_test json101-24.$id.replace {
+    SELECT json_replace($start,$path,9);
+  } [list [tx $repl]]
+}

 finish_test
@@ -21,159 +21,492 @@ source $testdir/tester.tcl
 do_execsql_test json102-100 {
  SELECT json_object('ex','[52,3.14159]');
 } {{{"ex":"[52,3.14159]"}}}
+do_execsql_test json102-100b {
+  SELECT json(jsonb_object('ex','[52,3.14159]'));
+} {{{"ex":"[52,3.14159]"}}}
 do_execsql_test json102-110 {
  SELECT json_object('ex',json('[52,3.14159]'));
 } {{{"ex":[52,3.14159]}}}
+do_execsql_test json102-110-2 {
+  SELECT json(jsonb_object('ex',json('[52,3.14159]')));
+} {{{"ex":[52,3.14159]}}}
+do_execsql_test json102-110-3 {
+  SELECT json_object('ex',jsonb('[52,3.14159]'));
+} {{{"ex":[52,3.14159]}}}
+do_execsql_test json102-110-3 {
+  SELECT json(jsonb_object('ex',jsonb('[52,3.14159]')));
+} {{{"ex":[52,3.14159]}}}
 do_execsql_test json102-120 {
  SELECT json_object('ex',json_array(52,3.14159));
 } {{{"ex":[52,3.14159]}}}
+do_execsql_test json102-120-2 {
+  SELECT json(jsonb_object('ex',json_array(52,3.14159)));
+} {{{"ex":[52,3.14159]}}}
+do_execsql_test json102-120-3 {
+  SELECT json_object('ex',jsonb_array(52,3.14159));
+} {{{"ex":[52,3.14159]}}}
+do_execsql_test json102-120-4 {
+  SELECT json(jsonb_object('ex',jsonb_array(52,3.14159)));
+} {{{"ex":[52,3.14159]}}}
 do_execsql_test json102-130 {
  SELECT json(' { "this" : "is", "a": [ "test" ] } ');
 } {{{"this":"is","a":["test"]}}}
+do_execsql_test json102-130b {
+  SELECT json(jsonb(' { "this" : "is", "a": [ "test" ] } '));
+} {{{"this":"is","a":["test"]}}}
 do_execsql_test json102-140 {
  SELECT json_array(1,2,'3',4);
 } {{[1,2,"3",4]}}
+do_execsql_test json102-140b {
+  SELECT json(jsonb_array(1,2,'3',4));
+} {{[1,2,"3",4]}}
 do_execsql_test json102-150 {
  SELECT json_array('[1,2]');
 } {{["[1,2]"]}}
+do_execsql_test json102-150b {
+  SELECT json(jsonb_array('[1,2]'));
+} {{["[1,2]"]}}
 do_execsql_test json102-160 {
  SELECT json_array(json_array(1,2));
 } {{[[1,2]]}}
+do_execsql_test json102-160-2 {
+  SELECT json_array(jsonb_array(1,2));
+} {{[[1,2]]}}
+do_execsql_test json102-160-3 {
+  SELECT json(jsonb_array(json_array(1,2)));
+} {{[[1,2]]}}
+do_execsql_test json102-160-4 {
+  SELECT json(jsonb_array(jsonb_array(1,2)));
+} {{[[1,2]]}}
 do_execsql_test json102-170 {
  SELECT json_array(1,null,'3','[4,5]','{"six":7.7}');
 } {{[1,null,"3","[4,5]","{\"six\":7.7}"]}}
+do_execsql_test json102-170b {
+  SELECT json(jsonb_array(1,null,'3','[4,5]','{"six":7.7}'));
+} {{[1,null,"3","[4,5]","{\"six\":7.7}"]}}
 do_execsql_test json102-180 {
  SELECT json_array(1,null,'3',json('[4,5]'),json('{"six":7.7}'));
 } {{[1,null,"3",[4,5],{"six":7.7}]}}
+do_execsql_test json102-180-2 {
+  SELECT json_array(1,null,'3',jsonb('[4,5]'),json('{"six":7.7}'));
+} {{[1,null,"3",[4,5],{"six":7.7}]}}
+do_execsql_test json102-180-3 {
+  SELECT json(jsonb_array(1,null,'3',json('[4,5]'),json('{"six":7.7}')));
+} {{[1,null,"3",[4,5],{"six":7.7}]}}
+do_execsql_test json102-180-4 {
+  SELECT json(jsonb_array(1,null,'3',jsonb('[4,5]'),jsonb('{"six":7.7}')));
+} {{[1,null,"3",[4,5],{"six":7.7}]}}
 do_execsql_test json102-190 {
  SELECT json_array_length('[1,2,3,4]');
 } {{4}}
+do_execsql_test json102-190b {
+  SELECT json_array_length(jsonb('[1,2,3,4]'));
+} {{4}}
 do_execsql_test json102-191 {
  SELECT json_array_length( json_remove('[1,2,3,4]','$[2]') );
 } {{3}}
+do_execsql_test json102-191b {
+  SELECT json_array_length( jsonb_remove('[1,2,3,4]','$[2]') );
+} {{3}}
 do_execsql_test json102-200 {
  SELECT json_array_length('[1,2,3,4]', '$');
 } {{4}}
+do_execsql_test json102-200b {
+  SELECT json_array_length(jsonb('[1,2,3,4]'), '$');
+} {{4}}
 do_execsql_test json102-210 {
  SELECT json_array_length('[1,2,3,4]', '$[2]');
 } {{0}}
+do_execsql_test json102-210b {
+  SELECT json_array_length(jsonb('[1,2,3,4]'), '$[2]');
+} {{0}}
 do_execsql_test json102-220 {
  SELECT json_array_length('{"one":[1,2,3]}');
 } {{0}}
-do_execsql_test json102-230 {
-  SELECT json_array_length('{"one":[1,2,3]}', '$.one');
+do_execsql_test json102-220 {
+  SELECT json_array_length('{"one":[1,2,3]}');
+} {{0}}
+do_execsql_test json102-230b {
+  SELECT json_array_length(jsonb('{"one":[1,2,3]}'), '$.one');
 } {{3}}
 do_execsql_test json102-240 {
  SELECT json_array_length('{"one":[1,2,3]}', '$.two');
 } {{}}
+do_execsql_test json102-240b {
+  SELECT json_array_length(jsonb('{"one":[1,2,3]}'), '$.two');
+} {{}}
 do_execsql_test json102-250 {
  SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$');
 } {{{"a":2,"c":[4,5,{"f":7}]}}}
+do_execsql_test json102-250-2 {
+  SELECT json_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$');
+} {{{"a":2,"c":[4,5,{"f":7}]}}}
+do_execsql_test json102-250-3 {
+  SELECT json(jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$'));
+} {{{"a":2,"c":[4,5,{"f":7}]}}}
+do_execsql_test json102-250-4 {
+  SELECT json(jsonb_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$'));
+} {{{"a":2,"c":[4,5,{"f":7}]}}}
 do_execsql_test json102-260 {
  SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c');
 } {{[4,5,{"f":7}]}}
+do_execsql_test json102-260-2 {
+  SELECT json_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.c');
+} {{[4,5,{"f":7}]}}
+do_execsql_test json102-260-3 {
+  SELECT json(jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c'));
+} {{[4,5,{"f":7}]}}
+do_execsql_test json102-260-4 {
+  SELECT json(jsonb_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.c'));
+} {{[4,5,{"f":7}]}}
 do_execsql_test json102-270 {
  SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2]');
 } {{{"f":7}}}
+do_execsql_test json102-270-2 {
+  SELECT json_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.c[2]');
+} {{{"f":7}}}
+do_execsql_test json102-270-3 {
+  SELECT json(jsonb_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.c[2]'));
+} {{{"f":7}}}
+do_execsql_test json102-270-4 {
+  SELECT json(jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2]'));
+} {{{"f":7}}}
 do_execsql_test json102-280 {
  SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2].f');
 } {{7}}
+do_execsql_test json102-280b {
+  SELECT jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2].f');
+} {{7}}
 do_execsql_test json102-290 {
  SELECT json_extract('{"a":2,"c":[4,5],"f":7}','$.c','$.a');
 } {{[[4,5],2]}}
+do_execsql_test json102-290-2 {
+  SELECT json_extract(jsonb('{"a":2,"c":[4,5],"f":7}'),'$.c','$.a');
+} {{[[4,5],2]}}
+do_execsql_test json102-290-3 {
+  SELECT json(jsonb_extract('{"a":2,"c":[4,5],"f":7}','$.c','$.a'));
+} {{[[4,5],2]}}
+do_execsql_test json102-290-4 {
+  SELECT json(jsonb_extract(jsonb('{"a":2,"c":[4,5],"f":7}'),'$.c','$.a'));
+} {{[[4,5],2]}}
 do_execsql_test json102-300 {
  SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x');
 } {{}}
+do_execsql_test json102-300b {
+  SELECT jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x');
+} {{}}
 do_execsql_test json102-310 {
  SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x', '$.a');
 } {{[null,2]}}
+do_execsql_test json102-310-2 {
+  SELECT json_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.x', '$.a');
+} {{[null,2]}}
+do_execsql_test json102-310-3 {
+  SELECT json(jsonb_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.x', '$.a'));
+} {{[null,2]}}
+do_execsql_test json102-310-43 {
+  SELECT json(jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x', '$.a'));
+} {{[null,2]}}
 do_execsql_test json102-320 {
  SELECT json_insert('{"a":2,"c":4}', '$.a', 99);
 } {{{"a":2,"c":4}}}
+do_execsql_test json102-320-2 {
+  SELECT json_insert(jsonb('{"a":2,"c":4}'), '$.a', 99);
+} {{{"a":2,"c":4}}}
+do_execsql_test json102-320-3 {
+  SELECT json(jsonb_insert('{"a":2,"c":4}', '$.a', 99));
+} {{{"a":2,"c":4}}}
+do_execsql_test json102-320-4 {
+  SELECT json(jsonb_insert(jsonb('{"a":2,"c":4}'), '$.a', 99));
+} {{{"a":2,"c":4}}}
 do_execsql_test json102-330 {
  SELECT json_insert('{"a":2,"c":4}', '$.e', 99);
 } {{{"a":2,"c":4,"e":99}}}
+do_execsql_test json102-330-2 {
+  SELECT json_insert(jsonb('{"a":2,"c":4}'), '$.e', 99);
+} {{{"a":2,"c":4,"e":99}}}
+do_execsql_test json102-330-3 {
+  SELECT json(jsonb_insert('{"a":2,"c":4}', '$.e', 99));
+} {{{"a":2,"c":4,"e":99}}}
+do_execsql_test json102-330-4 {
+  SELECT json(jsonb_insert(jsonb('{"a":2,"c":4}'), '$.e', 99));
+} {{{"a":2,"c":4,"e":99}}}
 do_execsql_test json102-340 {
  SELECT json_replace('{"a":2,"c":4}', '$.a', 99);
 } {{{"a":99,"c":4}}}
+do_execsql_test json102-340-2 {
+  SELECT json_replace(jsonb('{"a":2,"c":4}'), '$.a', 99);
+} {{{"a":99,"c":4}}}
+do_execsql_test json102-340-3 {
+  SELECT json(jsonb_replace('{"a":2,"c":4}', '$.a', 99));
+} {{{"a":99,"c":4}}}
+do_execsql_test json102-340-4 {
+  SELECT json(jsonb_replace(jsonb('{"a":2,"c":4}'), '$.a', 99));
+} {{{"a":99,"c":4}}}
 do_execsql_test json102-350 {
  SELECT json_replace('{"a":2,"c":4}', '$.e', 99);
 } {{{"a":2,"c":4}}}
+do_execsql_test json102-350-2 {
+  SELECT json_replace(jsonb('{"a":2,"c":4}'), '$.e', 99);
+} {{{"a":2,"c":4}}}
+do_execsql_test json102-350-3 {
+  SELECT json(jsonb_replace('{"a":2,"c":4}', '$.e', 99));
+} {{{"a":2,"c":4}}}
+do_execsql_test json102-350-4 {
+  SELECT json(jsonb_replace(jsonb('{"a":2,"c":4}'), '$.e', 99));
+} {{{"a":2,"c":4}}}
 do_execsql_test json102-360 {
  SELECT json_set('{"a":2,"c":4}', '$.a', 99);
 } {{{"a":99,"c":4}}}
+do_execsql_test json102-360-2 {
+  SELECT json_set(jsonb('{"a":2,"c":4}'), '$.a', 99);
+} {{{"a":99,"c":4}}}
+do_execsql_test json102-360-3 {
+  SELECT json(jsonb_set('{"a":2,"c":4}', '$.a', 99));
+} {{{"a":99,"c":4}}}
+do_execsql_test json102-360-4 {
+  SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.a', 99));
+} {{{"a":99,"c":4}}}
 do_execsql_test json102-370 {
  SELECT json_set('{"a":2,"c":4}', '$.e', 99);
 } {{{"a":2,"c":4,"e":99}}}
+do_execsql_test json102-370-2 {
+  SELECT json_set(jsonb('{"a":2,"c":4}'), '$.e', 99);
+} {{{"a":2,"c":4,"e":99}}}
+do_execsql_test json102-370-3 {
+  SELECT json(jsonb_set('{"a":2,"c":4}', '$.e', 99));
+} {{{"a":2,"c":4,"e":99}}}
+do_execsql_test json102-370-4 {
+  SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.e', 99));
+} {{{"a":2,"c":4,"e":99}}}
 do_execsql_test json102-380 {
  SELECT json_set('{"a":2,"c":4}', '$.c', '[97,96]');
 } {{{"a":2,"c":"[97,96]"}}}
+do_execsql_test json102-380-2 {
+  SELECT json_set(jsonb('{"a":2,"c":4}'), '$.c', '[97,96]');
+} {{{"a":2,"c":"[97,96]"}}}
+do_execsql_test json102-380-3 {
+  SELECT json(jsonb_set('{"a":2,"c":4}', '$.c', '[97,96]'));
+} {{{"a":2,"c":"[97,96]"}}}
+do_execsql_test json102-380-4 {
+  SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.c', '[97,96]'));
+} {{{"a":2,"c":"[97,96]"}}}
 do_execsql_test json102-390 {
  SELECT json_set('{"a":2,"c":4}', '$.c', json('[97,96]'));
 } {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-390-2 {
+  SELECT json_set(jsonb('{"a":2,"c":4}'), '$.c', json('[97,96]'));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-390-3 {
+  SELECT json(jsonb_set('{"a":2,"c":4}', '$.c', json('[97,96]')));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-390-4 {
+  SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.c', json('[97,96]')));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-390-5 {
+  SELECT json_set('{"a":2,"c":4}', '$.c', jsonb('[97,96]'));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-390-6 {
+  SELECT json_set(jsonb('{"a":2,"c":4}'), '$.c', jsonb('[97,96]'));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-390-7 {
+  SELECT json(jsonb_set('{"a":2,"c":4}', '$.c', jsonb('[97,96]')));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-390-8 {
+  SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.c', jsonb('[97,96]')));
+} {{{"a":2,"c":[97,96]}}}
 do_execsql_test json102-400 {
  SELECT json_set('{"a":2,"c":4}', '$.c', json_array(97,96));
 } {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-400-2 {
+  SELECT json_set(jsonb('{"a":2,"c":4}'), '$.c', json_array(97,96));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-400-3 {
+  SELECT json(jsonb_set('{"a":2,"c":4}', '$.c', json_array(97,96)));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-400-4 {
+  SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.c', json_array(97,96)));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-400-5 {
+  SELECT json_set('{"a":2,"c":4}', '$.c', jsonb_array(97,96));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-400-6 {
+  SELECT json_set(jsonb('{"a":2,"c":4}'), '$.c', jsonb_array(97,96));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-400-7 {
+  SELECT json(jsonb_set('{"a":2,"c":4}', '$.c', jsonb_array(97,96)));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-400-8 {
+  SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.c', jsonb_array(97,96)));
+} {{{"a":2,"c":[97,96]}}}
 do_execsql_test json102-410 {
  SELECT json_object('a',2,'c',4);
 } {{{"a":2,"c":4}}}
+do_execsql_test json102-410b {
+  SELECT json(jsonb_object('a',2,'c',4));
+} {{{"a":2,"c":4}}}
 do_execsql_test json102-420 {
  SELECT json_object('a',2,'c','{e:5}');
 } {{{"a":2,"c":"{e:5}"}}}
+do_execsql_test json102-420b {
+  SELECT json(jsonb_object('a',2,'c','{e:5}'));
+} {{{"a":2,"c":"{e:5}"}}}
 do_execsql_test json102-430 {
  SELECT json_object('a',2,'c',json_object('e',5));
 } {{{"a":2,"c":{"e":5}}}}
+do_execsql_test json102-430-2 {
+  SELECT json(jsonb_object('a',2,'c',json_object('e',5)));
+} {{{"a":2,"c":{"e":5}}}}
+do_execsql_test json102-430-3 {
+  SELECT json_object('a',2,'c',jsonb_object('e',5));
+} {{{"a":2,"c":{"e":5}}}}
+do_execsql_test json102-430-4 {
+  SELECT json(jsonb_object('a',2,'c',jsonb_object('e',5)));
+} {{{"a":2,"c":{"e":5}}}}
 do_execsql_test json102-440 {
  SELECT json_remove('[0,1,2,3,4]','$[2]');
 } {{[0,1,3,4]}}
+do_execsql_test json102-440-2 {
+  SELECT json_remove(jsonb('[0,1,2,3,4]'),'$[2]');
+} {{[0,1,3,4]}}
+do_execsql_test json102-440-3 {
+  SELECT json(jsonb_remove('[0,1,2,3,4]','$[2]'));
+} {{[0,1,3,4]}}
+do_execsql_test json102-440-4 {
+  SELECT json(jsonb_remove(jsonb('[0,1,2,3,4]'),'$[2]'));
+} {{[0,1,3,4]}}
 do_execsql_test json102-450 {
  SELECT json_remove('[0,1,2,3,4]','$[2]','$[0]');
 } {{[1,3,4]}}
+do_execsql_test json102-450-2 {
+  SELECT json_remove(jsonb('[0,1,2,3,4]'),'$[2]','$[0]');
+} {{[1,3,4]}}
+do_execsql_test json102-450-3 {
+  SELECT json(jsonb_remove('[0,1,2,3,4]','$[2]','$[0]'));
+} {{[1,3,4]}}
+do_execsql_test json102-450-4 {
+  SELECT json(jsonb_remove(jsonb('[0,1,2,3,4]'),'$[2]','$[0]'));
+} {{[1,3,4]}}
 do_execsql_test json102-460 {
  SELECT json_remove('[0,1,2,3,4]','$[0]','$[2]');
 } {{[1,2,4]}}
+do_execsql_test json102-460-2 {
+  SELECT json_remove(jsonb('[0,1,2,3,4]'),'$[0]','$[2]');
+} {{[1,2,4]}}
+do_execsql_test json102-460-3 {
+  SELECT json(jsonb_remove('[0,1,2,3,4]','$[0]','$[2]'));
+} {{[1,2,4]}}
+do_execsql_test json102-460-4 {
+  SELECT json(jsonb_remove(jsonb('[0,1,2,3,4]'),'$[0]','$[2]'));
+} {{[1,2,4]}}
 do_execsql_test json102-470 {
  SELECT json_remove('{"x":25,"y":42}');
 } {{{"x":25,"y":42}}}
+do_execsql_test json102-470-2 {
+  SELECT json_remove(jsonb('{"x":25,"y":42}'));
+} {{{"x":25,"y":42}}}
+do_execsql_test json102-470-3 {
+  SELECT json(jsonb_remove('{"x":25,"y":42}'));
+} {{{"x":25,"y":42}}}
+do_execsql_test json102-470-4 {
+  SELECT json(jsonb_remove(jsonb('{"x":25,"y":42}')));
+} {{{"x":25,"y":42}}}
 do_execsql_test json102-480 {
  SELECT json_remove('{"x":25,"y":42}','$.z');
 } {{{"x":25,"y":42}}}
+do_execsql_test json102-480-2 {
+  SELECT json_remove(jsonb('{"x":25,"y":42}'),'$.z');
+} {{{"x":25,"y":42}}}
+do_execsql_test json102-480-3 {
+  SELECT json(jsonb_remove('{"x":25,"y":42}','$.z'));
+} {{{"x":25,"y":42}}}
+do_execsql_test json102-480-4 {
+  SELECT json(jsonb_remove(jsonb('{"x":25,"y":42}'),'$.z'));
+} {{{"x":25,"y":42}}}
 do_execsql_test json102-490 {
  SELECT json_remove('{"x":25,"y":42}','$.y');
 } {{{"x":25}}}
+do_execsql_test json102-490-2 {
+  SELECT json_remove(jsonb('{"x":25,"y":42}'),'$.y');
+} {{{"x":25}}}
+do_execsql_test json102-490-3 {
+  SELECT json(jsonb_remove('{"x":25,"y":42}','$.y'));
+} {{{"x":25}}}
+do_execsql_test json102-490-4 {
+  SELECT json(jsonb_remove(jsonb('{"x":25,"y":42}'),'$.y'));
+} {{{"x":25}}}
 do_execsql_test json102-500 {
  SELECT json_remove('{"x":25,"y":42}','$');
 } {{}}
+do_execsql_test json102-500-2 {
+  SELECT json_remove(jsonb('{"x":25,"y":42}'),'$');
+} {{}}
+do_execsql_test json102-500-3 {
+  SELECT json(jsonb_remove('{"x":25,"y":42}','$'));
+} {{}}
+do_execsql_test json102-500-4 {
+  SELECT json(jsonb_remove(jsonb('{"x":25,"y":42}'),'$'));
+} {{}}
 do_execsql_test json102-510 {
  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}');
 } {{object}}
+do_execsql_test json102-510b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778');
+} {{object}}
 do_execsql_test json102-520 {
  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$');
 } {{object}}
+do_execsql_test json102-520b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$');
+} {{object}}
 do_execsql_test json102-530 {
  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a');
 } {{array}}
+do_execsql_test json102-530b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a');
+} {{array}}
 do_execsql_test json102-540 {
  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[0]');
 } {{integer}}
+do_execsql_test json102-540b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[0]');
+} {{integer}}
 do_execsql_test json102-550 {
  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[1]');
 } {{real}}
+do_execsql_test json102-550b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[1]');
+} {{real}}
 do_execsql_test json102-560 {
  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[2]');
 } {{true}}
+do_execsql_test json102-560b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[2]');
+} {{true}}
 do_execsql_test json102-570 {
  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[3]');
 } {{false}}
+do_execsql_test json102-570b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[3]');
+} {{false}}
 do_execsql_test json102-580 {
  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[4]');
 } {{null}}
+do_execsql_test json102-580b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[4]');
+} {{null}}
 do_execsql_test json102-590 {
  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[5]');
 } {{text}}
+do_execsql_test json102-590b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[5]');
+} {{text}}
 do_execsql_test json102-600 {
  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[6]');
 } {{}}
+do_execsql_test json102-600b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[6]');
+} {{}}
 do_execsql_test json102-610 {
  SELECT json_valid(char(123)||'"x":35'||char(125));
 } {{1}}
@@ -183,17 +516,24 @@ do_execsql_test json102-620 {

 ifcapable vtab {
 do_execsql_test json102-1000 {
-  CREATE TABLE user(name,phone);
+  CREATE TABLE user(name,phone,phoneb);
  INSERT INTO user(name,phone) VALUES
     ('Alice','["919-555-2345","804-555-3621"]'),
     ('Bob','["201-555-8872"]'),
     ('Cindy','["704-555-9983"]'),
     ('Dave','["336-555-8421","704-555-4321","803-911-4421"]');
+  UPDATE user SET phoneb=jsonb(phone);
  SELECT DISTINCT user.name
    FROM user, json_each(user.phone)
   WHERE json_each.value LIKE '704-%'
   ORDER BY 1;
 } {Cindy Dave}
+do_execsql_test json102-1000b {
+  SELECT DISTINCT user.name
+    FROM user, json_each(user.phoneb)
+   WHERE json_each.value LIKE '704-%'
+   ORDER BY 1;
+} {Cindy Dave}

 do_execsql_test json102-1010 {
  UPDATE user
@@ -253,6 +593,12 @@ do_execsql_test json102-1110 {
   WHERE json_tree.type NOT IN ('object','array')
   ORDER BY +big.rowid, +json_tree.id
 } $correct_answer
+do_execsql_test json102-1110b {
+  SELECT big.rowid, fullkey, value
+    FROM big, json_tree(jsonb(big.json))
+   WHERE json_tree.type NOT IN ('object','array')
+   ORDER BY +big.rowid, +json_tree.id
+} $correct_answer
 do_execsql_test json102-1120 {
  SELECT big.rowid, fullkey, atom
    FROM big, json_tree(big.json)
@@ -96,18 +96,18 @@ json_replace_test 80 {'$.b[#-1]','AAA','$.b[#-1]','BBB'} \

 do_catchsql_test json105-6.10 {
  SELECT json_extract(j, '$.b[#-]') FROM t1;
-} {1 {JSON path error near '[#-]'}}
+} {1 {bad JSON path: '$.b[#-]'}}
 do_catchsql_test json105-6.20 {
  SELECT json_extract(j, '$.b[#9]') FROM t1;
-} {1 {JSON path error near '[#9]'}}
+} {1 {bad JSON path: '$.b[#9]'}}
 do_catchsql_test json105-6.30 {
  SELECT json_extract(j, '$.b[#+2]') FROM t1;
-} {1 {JSON path error near '[#+2]'}}
+} {1 {bad JSON path: '$.b[#+2]'}}
 do_catchsql_test json105-6.40 {
  SELECT json_extract(j, '$.b[#-1') FROM t1;
-} {1 {JSON path error near '[#-1'}}
+} {1 {bad JSON path: '$.b[#-1'}}
 do_catchsql_test json105-6.50 {
  SELECT json_extract(j, '$.b[#-1x]') FROM t1;
-} {1 {JSON path error near '[#-1x]'}}
+} {1 {bad JSON path: '$.b[#-1x]'}}

 finish_test
@@ -252,13 +252,13 @@ do_execsql_test 8.11 {

 do_execsql_test 9.1 {
  WITH c(x) AS (VALUES('{x: +Infinity}')) SELECT x->>'x', json(x) FROM c;
-} {Inf {{"x":9.0e999}}}
+} {Inf {{"x":9e999}}}
 do_execsql_test 9.2 {
  WITH c(x) AS (VALUES('{x: -Infinity}')) SELECT x->>'x', json(x) FROM c;
-} {-Inf {{"x":-9.0e999}}}
+} {-Inf {{"x":-9e999}}}
 do_execsql_test 9.3 {
  WITH c(x) AS (VALUES('{x: Infinity}')) SELECT x->>'x', json(x) FROM c;
-} {Inf {{"x":9.0e999}}}
+} {Inf {{"x":9e999}}}
 do_execsql_test 9.4 {
  WITH c(x) AS (VALUES('{x: NaN}')) SELECT x->>'x', json(x) FROM c;
 } {{} {{"x":null}}}
@@ -36,5 +36,30 @@ do_catchsql_test 2.3 {
  SELECT '{a:null,{"h":[1,[1,2,3]],"j":"abc"}:true}'->'$h[#-1]';
 } {1 {malformed JSON}}

+# Verify that escaped label names are compared correctly.
+# 
+do_execsql_test 3.1 {
+  SELECT '{"a\x62c":123}' ->> 'abc';
+} 123
+do_execsql_test 3.2 {
+  SELECT '{"abc":123}' ->> 'a\x62c';
+} 123
+
+db null null
+do_execsql_test 3.3 {
+  DROP TABLE t1;
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES(json_insert('{}','$.a\',111,'$."b\\"',222));
+  INSERT INTO t1 VALUES(jsonb_insert('{}','$.a\',111,'$."b\\"',222));
+  SELECT x->'$.a\', x->'$.a\\', x->'$."a\\"', x->'$."b\\"' FROM t1;
+} {111 null 111 222 111 null 111 222}
+
+do_execsql_test 3.4 {
+  SELECT json_patch('{"a\x62c":123}','{"ab\x63":456}') ->> 'abc';
+} 456
+
+do_execsql_test 4.1 {
+  SELECT * FROM json_tree('{"\u0017":1}','$."\x17"');
+} {{\x17} 1 integer 1 1 null {$."\x17"} {$}}

 finish_test
@@ -0,0 +1,49 @@
+# 2023-11-15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# Test cases for JSONB
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_execsql_test jsonb01-1.1 {
+  CREATE TABLE t1(x JSON BLOB);
+  INSERT INTO t1 VALUES(jsonb('{a:5,b:{x:10,y:11},c:[1,2,3,4]}'));
+}
+foreach {id path res} {
+  1 {$.a}    {{{"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+  2 {$.b}    {{{"a":5,"c":[1,2,3,4]}}}
+  3 {$.c}    {{{"a":5,"b":{"x":10,"y":11}}}}
+  4 {$.d}    {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+  5 {$.b.x}  {{{"a":5,"b":{"y":11},"c":[1,2,3,4]}}}
+  6 {$.b.y}  {{{"a":5,"b":{"x":10},"c":[1,2,3,4]}}}
+  7 {$.c[0]}  {{{"a":5,"b":{"x":10,"y":11},"c":[2,3,4]}}}
+  8 {$.c[1]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,3,4]}}}
+  9 {$.c[2]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,4]}}}
+ 10 {$.c[3]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3]}}}
+ 11 {$.c[4]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+ 12 {$.c[#]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+ 13 {$.c[#-1]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3]}}}
+ 14 {$.c[#-2]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,4]}}}
+ 15 {$.c[#-3]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,3,4]}}}
+ 16 {$.c[#-4]}  {{{"a":5,"b":{"x":10,"y":11},"c":[2,3,4]}}}
+ 17 {$.c[#-5]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+ 18 {$.c[#-6]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+} {
+  do_execsql_test jsonb01-1.2.$id.1 {
+    SELECT json(jsonb_remove(x,$path)) FROM t1;
+  } $res
+  do_execsql_test jsonb01-1.2.$id.2 {
+    SELECT json_remove(x,$path) FROM t1;
+  } $res
+}
+
+finish_test
@@ -148,6 +148,8 @@ set TRG(cmdline) $argv
 set TRG(reporttime) 2000
 set TRG(fuzztest) 0                 ;# is the fuzztest option present.
 set TRG(zipvfs) ""                  ;# -zipvfs option, if any
+set TRG(buildonly) 0                ;# True if --buildonly option 
+set TRG(dryrun) 0                   ;# True if --dryrun option 

 switch -nocase -glob -- $tcl_platform(os) {
  *darwin* {
@@ -427,6 +429,10 @@ for {set ii 0} {$ii < [llength $argv]} {incr ii} {
      incr ii
      set TRG(zipvfs) [file normalize [lindex $argv $ii]]
      if {$isLast} { usage }
+    } elseif {($n>2 && [string match "$a*" --buildonly]) || $a=="-b"} {
+      set TRG(buildonly) 1
+    } elseif {($n>2 && [string match "$a*" --dryrun]) || $a=="-d"} {
+      set TRG(dryrun) 1
    } else {
      usage
    }
@@ -752,6 +758,20 @@ proc add_zipvfs_jobs {} {
  set ::env(SQLITE_TEST_DIR) $::testdir
 }

+# Used to add jobs for "mdevtest" and "sdevtest".
+#
+proc add_devtest_jobs {lBld patternlist} {
+  global TRG
+
+  foreach b $lBld {
+    set bld [add_build_job $b $TRG(testfixture)]
+    add_tcl_jobs $bld veryquick $patternlist
+    if {$patternlist==""} {
+      add_fuzztest_jobs $b
+    }
+  }
+}
+
 proc add_jobs_from_cmdline {patternlist} {
  global TRG

@@ -775,33 +795,28 @@ proc add_jobs_from_cmdline {patternlist} {
    }

    mdevtest {
-      foreach b [list All-O0 All-Debug] {
-        set bld [add_build_job $b $TRG(testfixture)]
-        add_tcl_jobs $bld veryquick ""
-        add_fuzztest_jobs $b
-      }
+      add_devtest_jobs {All-O0 All-Debug} [lrange $patternlist 1 end]
    }

    sdevtest {
-      foreach b [list All-Sanitize All-Debug] {
-        set bld [add_build_job $b $TRG(testfixture)]
-        add_tcl_jobs $bld veryquick ""
-        add_fuzztest_jobs $b
-      }
+      add_devtest_jobs {All-Sanitize All-Debug} [lrange $patternlist 1 end]
    }

    release {
+      set patternlist [lrange $patternlist 1 end]
      foreach b [trd_builds $TRG(platform)] {
        set bld [add_build_job $b $TRG(testfixture)]
        foreach c [trd_configs $TRG(platform) $b] {
-          add_tcl_jobs $bld $c ""
+          add_tcl_jobs $bld $c $patternlist
        }

-        foreach e [trd_extras $TRG(platform) $b] {
-          if {$e=="fuzztest"} {
-            add_fuzztest_jobs $b
-          } else {
-            add_make_job $bld $e
+        if {$patternlist==""} {
+          foreach e [trd_extras $TRG(platform) $b] {
+            if {$e=="fuzztest"} {
+              add_fuzztest_jobs $b
+            } else {
+              add_make_job $bld $e
+            }
          }
        }
      }
@@ -834,6 +849,17 @@ proc make_new_testset {} {

 }

+proc mark_job_as_finished {jobid output state endtm} {
+  r_write_db {
+    trdb eval {
+      UPDATE jobs 
+        SET output=$output, state=$state, endtime=$endtm
+        WHERE jobid=$jobid;
+      UPDATE jobs SET state='ready' WHERE depid=$jobid;
+    }
+  }
+}
+
 proc script_input_ready {fd iJob jobid} {
  global TRG
  global O
@@ -868,15 +894,7 @@ proc script_input_ready {fd iJob jobid} {
    puts $TRG(log) "### $job(displayname) ${jobtm}ms ($state)"
    puts $TRG(log) [string trim $O($iJob)]

-    r_write_db {
-      set output $O($iJob)
-      trdb eval {
-        UPDATE jobs 
-          SET output=$output, state=$state, endtime=$tm
-          WHERE jobid=$jobid;
-        UPDATE jobs SET state='ready' WHERE depid=$jobid;
-      }
-    }
+    mark_job_as_finished $jobid $O($iJob) $state $tm

    dirs_freeDir $iJob
    launch_some_jobs
@@ -928,16 +946,28 @@ proc launch_another_job {iJob} {
    close $fd
  }

-  set pwd [pwd]
-  cd $dir
-  set fd [open $TRG(run) w]
-  puts $fd $job(cmd) 
-  close $fd
-  set fd [open "|$TRG(runcmd) 2>@1" r]
-  cd $pwd
+  if { $TRG(dryrun) } {

-  fconfigure $fd -blocking false
-  fileevent $fd readable [list script_input_ready $fd $iJob $job(jobid)]
+    mark_job_as_finished $job(jobid) "" done 0
+    dirs_freeDir $iJob
+    if {$job(build)!=""} {
+      puts $TRG(log) "(cd $dir ; $job(cmd) )"
+    } else {
+      puts $TRG(log) "$job(cmd)"
+    }
+
+  } else {
+    set pwd [pwd]
+    cd $dir
+    set fd [open $TRG(run) w]
+    puts $fd $job(cmd) 
+    close $fd
+    set fd [open "|$TRG(runcmd) 2>@1" r]
+    cd $pwd
+
+    fconfigure $fd -blocking false
+    fileevent $fd readable [list script_input_ready $fd $iJob $job(jobid)]
+  }

  return 1
 }
@@ -1035,6 +1065,16 @@ proc run_testset {} {
  puts "Test log is $TRG(logname)"
 }

+# Handle the --buildonly option, if it was specified.
+#
+proc handle_buildonly {} {
+  global TRG
+  if {$TRG(buildonly)} {
+    r_write_db {
+      trdb eval { DELETE FROM jobs WHERE displaytype!='bld' }
+    }
+  }
+}

 sqlite3 trdb $TRG(dbname)
 trdb timeout $TRG(timeout)
@@ -1043,6 +1083,8 @@ if {$TRG(nJob)>1} {
  puts "splitting work across $TRG(nJob) jobs"
 }
 puts "built testset in [expr $tm/1000]ms.."
+
+handle_buildonly
 run_testset
 trdb close
 #puts [pwd]