Add NEVER to two unreachable branches in JSON.

FossilOrigin-Name: c96ebb086feb89341565cc52b970ae7799ce1327fe1ad4fc790f1b0dcaa6e229
Improvements to JSONB validation - catch more cases where the input does
2023-12-12 17:13:10 +00:00 · 2023-12-12 14:33:52 +00:00 · 2023-12-12 02:31:12 +00:00 · 2023-12-11 21:00:55 +00:00 · 2023-12-11 20:44:21 +00:00 · 2023-12-11 20:19:10 +00:00
49 changed files with 4577 additions and 2177 deletions
@@ -159,7 +159,7 @@ extension and only later escaped to the wild as an independent library.)

 Test scripts and programs are found in the **test/** subdirectory.
 Additional test code is found in other source repositories.
-See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for
+See [How SQLite Is Tested](https://www.sqlite.org/testing.html) for
 additional information.

 The **ext/** subdirectory contains code for extensions.  The
@@ -183,7 +183,7 @@ manually-edited files and automatically-generated files.

 The SQLite interface is defined by the **sqlite3.h** header file, which is
 generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION.  The
-[Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
+[Tcl script](https://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
 The manifest.uuid file contains the SHA3 hash of the particular check-in
 and is used to generate the SQLITE\_SOURCE\_ID macro.  The VERSION file
 contains the current SQLite version number.  The sqlite3.h header is really
@@ -250,14 +250,14 @@ individual source file exceeds 32K lines in length.
 ## How It All Fits Together

 SQLite is modular in design.
-See the [architectural description](http://www.sqlite.org/arch.html)
+See the [architectural description](https://www.sqlite.org/arch.html)
 for details. Other documents that are useful in
 (helping to understand how SQLite works include the
-[file format](http://www.sqlite.org/fileformat2.html) description,
-the [virtual machine](http://www.sqlite.org/opcode.html) that runs
+[file format](https://www.sqlite.org/fileformat2.html) description,
+the [virtual machine](https://www.sqlite.org/opcode.html) that runs
 prepared statements, the description of
-[how transactions work](http://www.sqlite.org/atomiccommit.html), and
-the [overview of the query planner](http://www.sqlite.org/optoverview.html).
+[how transactions work](https://www.sqlite.org/atomiccommit.html), and
+the [overview of the query planner](https://www.sqlite.org/optoverview.html).

 Years of effort have gone into optimizing SQLite, both
 for small size and high performance.  And optimizations tend to result in
@@ -353,7 +353,7 @@ hidden by also modifying the makefiles.

 ## Contacts

-The main SQLite website is [http:/sqlite.org/](http://sqlite.org/)
+The main SQLite website is [https://sqlite.org/](https://sqlite.org/)
 with geographically distributed backups at
-[http://www2.sqlite.org/](http://www2.sqlite.org) and
-[http://www3.sqlite.org/](http://www3.sqlite.org).
+[https://www2.sqlite.org/](https://www2.sqlite.org) and
+[https://www3.sqlite.org/](https://www3.sqlite.org).
@@ -223,10 +223,12 @@ proc main {data} {
    Fts5ExtensionApi {
      set struct [get_fts5_struct $data "^struct Fts5ExtensionApi" "^.;"]
      set map [list]
+      set lKey [list]
      foreach {k v} [get_struct_members $data] {
        if {[string match x* $k]==0} continue
-        lappend map $k "<a href=#$k>$k</a>"
+        lappend lKey $k
      }
+      foreach k [lsort -decr $lKey] { lappend map $k "<a href=#$k>$k</a>" }
      output [string map $map $struct]
    }

@@ -261,9 +261,34 @@ struct Fts5PhraseIter {
 **
 ** xPhraseNextColumn()
 **   See xPhraseFirstColumn above.
+**
+** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)
+**   This is used to access token iToken of phrase iPhrase of the current
+**   query. Before returning, output parameter *ppToken is set to point
+**   to a buffer containing the requested token, and *pnToken to the
+**   size of this buffer in bytes.
+**
+**   The output text is not a copy of the query text that specified the
+**   token. It is the output of the tokenizer module. For tokendata=1
+**   tables, this includes any embedded 0x00 and trailing data.
+**
+** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)
+**   This is used to access token iToken of phrase hit iIdx within the
+**   current row. Output variable (*ppToken) is set to point to a buffer
+**   containing the matching document token, and (*pnToken) to the size
+**   of that buffer in bytes. This API is not available if the specified 
+**   token matches a prefix query term. In that case both output variables 
+**   are always set to 0.
+**
+**   The output text is not a copy of the document text that was tokenized.
+**   It is the output of the tokenizer module. For tokendata=1 tables, this 
+**   includes any embedded 0x00 and trailing data.
+**
+**   This API can be quite slow if used with an FTS5 table created with the
+**   "detail=none" or "detail=column" option.
 */
 struct Fts5ExtensionApi {
-  int iVersion;                   /* Currently always set to 2 */
+  int iVersion;                   /* Currently always set to 3 */

  void *(*xUserData)(Fts5Context*);

@@ -298,6 +323,13 @@ struct Fts5ExtensionApi {

  int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
  void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
+
+  /* Below this point are iVersion>=3 only */
+  int (*xQueryToken)(Fts5Context*, 
+      int iPhrase, int iToken, 
+      const char **ppToken, int *pnToken
+  );
+  int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
 };

 /* 
@@ -196,6 +196,7 @@ struct Fts5Config {
  char *zContent;                 /* content table */ 
  char *zContentRowid;            /* "content_rowid=" option value */ 
  int bColumnsize;                /* "columnsize=" option value (dflt==1) */
+  int bTokendata;                 /* "tokendata=" option value (dflt==0) */
  int eDetail;                    /* FTS5_DETAIL_XXX value */
  char *zContentExprlist;
  Fts5Tokenizer *pTok;
@@ -384,17 +385,19 @@ struct Fts5IndexIter {
 /*
 ** Values used as part of the flags argument passed to IndexQuery().
 */
-#define FTS5INDEX_QUERY_PREFIX     0x0001   /* Prefix query */
-#define FTS5INDEX_QUERY_DESC       0x0002   /* Docs in descending rowid order */
-#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004   /* Do not use prefix index */
-#define FTS5INDEX_QUERY_SCAN       0x0008   /* Scan query (fts5vocab) */
+#define FTS5INDEX_QUERY_PREFIX      0x0001  /* Prefix query */
+#define FTS5INDEX_QUERY_DESC        0x0002  /* Docs in descending rowid order */
+#define FTS5INDEX_QUERY_TEST_NOIDX  0x0004  /* Do not use prefix index */
+#define FTS5INDEX_QUERY_SCAN        0x0008  /* Scan query (fts5vocab) */

 /* The following are used internally by the fts5_index.c module. They are
 ** defined here only to make it easier to avoid clashes with the flags
 ** above. */
-#define FTS5INDEX_QUERY_SKIPEMPTY  0x0010
-#define FTS5INDEX_QUERY_NOOUTPUT   0x0020
-#define FTS5INDEX_QUERY_SKIPHASH   0x0040
+#define FTS5INDEX_QUERY_SKIPEMPTY   0x0010
+#define FTS5INDEX_QUERY_NOOUTPUT    0x0020
+#define FTS5INDEX_QUERY_SKIPHASH    0x0040
+#define FTS5INDEX_QUERY_NOTOKENDATA 0x0080
+#define FTS5INDEX_QUERY_SCANONETERM 0x0100

 /*
 ** Create/destroy an Fts5Index object.
@@ -463,6 +466,10 @@ void *sqlite3Fts5StructureRef(Fts5Index*);
 void sqlite3Fts5StructureRelease(void*);
 int sqlite3Fts5StructureTest(Fts5Index*, void*);

+/*
+** Used by xInstToken():
+*/
+int sqlite3Fts5IterToken(Fts5IndexIter*, i64, int, int, const char**, int*);

 /*
 ** Insert or remove data to or from the index. Each time a document is 
@@ -540,6 +547,13 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin);
 int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid);

+void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*);
+
+/* Used to populate hash tables for xInstToken in detail=none/column mode. */
+int sqlite3Fts5IndexIterWriteTokendata(
+    Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff
+);
+
 /*
 ** End of interface to code in fts5_index.c.
 **************************************************************************/
@@ -645,6 +659,7 @@ void sqlite3Fts5HashScanNext(Fts5Hash*);
 int sqlite3Fts5HashScanEof(Fts5Hash*);
 void sqlite3Fts5HashScanEntry(Fts5Hash *,
  const char **pzTerm,            /* OUT: term (nul-terminated) */
+  int *pnTerm,                    /* OUT: Size of term in bytes */
  const u8 **ppDoclist,           /* OUT: pointer to doclist */
  int *pnDoclist                  /* OUT: size of doclist in bytes */
 );
@@ -771,6 +786,10 @@ int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);

 int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);

+int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*);
+int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*);
+void sqlite3Fts5ExprClearTokens(Fts5Expr*);
+
 /*******************************************
 ** The fts5_expr.c API above this point is used by the other hand-written
 ** C code in this module. The interfaces below this point are called by
@@ -398,6 +398,16 @@ static int fts5ConfigParseSpecial(
    return rc;
  }

+  if( sqlite3_strnicmp("tokendata", zCmd, nCmd)==0 ){
+    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
+      *pzErr = sqlite3_mprintf("malformed tokendata=... directive");
+      rc = SQLITE_ERROR;
+    }else{
+      pConfig->bTokendata = (zArg[0]=='1');
+    }
+    return rc;
+  }
+
  *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
  return SQLITE_ERROR;
 }
@@ -100,7 +100,9 @@ struct Fts5ExprNode {
 struct Fts5ExprTerm {
  u8 bPrefix;                     /* True for a prefix term */
  u8 bFirst;                      /* True if token must be first in column */
-  char *zTerm;                    /* nul-terminated term */
+  char *pTerm;                    /* Term data */
+  int nQueryTerm;                 /* Effective size of term in bytes */
+  int nFullTerm;                  /* Size of term in bytes incl. tokendata */
  Fts5IndexIter *pIter;           /* Iterator for this term */
  Fts5ExprTerm *pSynonym;         /* Pointer to first in list of synonyms */
 };
@@ -967,7 +969,7 @@ static int fts5ExprNearInitAll(
            p->pIter = 0;
          }
          rc = sqlite3Fts5IndexQuery(
-              pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
+              pExpr->pIndex, p->pTerm, p->nQueryTerm,
              (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
              (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
              pNear->pColset,
@@ -1604,7 +1606,7 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
      Fts5ExprTerm *pSyn;
      Fts5ExprTerm *pNext;
      Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
-      sqlite3_free(pTerm->zTerm);
+      sqlite3_free(pTerm->pTerm);
      sqlite3Fts5IterClose(pTerm->pIter);
      for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){
        pNext = pSyn->pSynonym;
@@ -1702,6 +1704,7 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
 typedef struct TokenCtx TokenCtx;
 struct TokenCtx {
  Fts5ExprPhrase *pPhrase;
+  Fts5Config *pConfig;
  int rc;
 };

@@ -1735,8 +1738,10 @@ static int fts5ParseTokenize(
      rc = SQLITE_NOMEM;
    }else{
      memset(pSyn, 0, (size_t)nByte);
-      pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
-      memcpy(pSyn->zTerm, pToken, nToken);
+      pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
+      pSyn->nFullTerm = pSyn->nQueryTerm = nToken;
+      if( pCtx->pConfig->bTokendata ) pSyn->nQueryTerm = strlen(pSyn->pTerm);
+      memcpy(pSyn->pTerm, pToken, nToken);
      pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
      pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn;
    }
@@ -1761,7 +1766,11 @@ static int fts5ParseTokenize(
    if( rc==SQLITE_OK ){
      pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
      memset(pTerm, 0, sizeof(Fts5ExprTerm));
-      pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
+      pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
+      pTerm->nFullTerm = pTerm->nQueryTerm = nToken;
+      if( pCtx->pConfig->bTokendata && rc==SQLITE_OK ){ 
+        pTerm->nQueryTerm = strlen(pTerm->pTerm);
+      }
    }
  }

@@ -1828,6 +1837,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(

  memset(&sCtx, 0, sizeof(TokenCtx));
  sCtx.pPhrase = pAppend;
+  sCtx.pConfig = pConfig;

  rc = fts5ParseStringFromToken(pToken, &z);
  if( rc==SQLITE_OK ){
@@ -1877,8 +1887,7 @@ int sqlite3Fts5ExprClonePhrase(
  int rc = SQLITE_OK;             /* Return code */
  Fts5ExprPhrase *pOrig;          /* The phrase extracted from pExpr */
  Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
-  TokenCtx sCtx = {0,0};          /* Context object for fts5ParseTokenize */
-
+  TokenCtx sCtx = {0,0,0};        /* Context object for fts5ParseTokenize */
  pOrig = pExpr->apExprPhrase[iPhrase];
  pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
  if( rc==SQLITE_OK ){
@@ -1909,13 +1918,12 @@ int sqlite3Fts5ExprClonePhrase(

  if( pOrig->nTerm ){
    int i;                          /* Used to iterate through phrase terms */
+    sCtx.pConfig = pExpr->pConfig;
    for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
      int tflags = 0;
      Fts5ExprTerm *p;
      for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
-        const char *zTerm = p->zTerm;
-        rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
-            0, 0);
+        rc = fts5ParseTokenize((void*)&sCtx, tflags, p->pTerm,p->nFullTerm,0,0);
        tflags = FTS5_TOKEN_COLOCATED;
      }
      if( rc==SQLITE_OK ){
@@ -2296,11 +2304,13 @@ static Fts5ExprNode *fts5ParsePhraseToAnd(
        if( parseGrowPhraseArray(pParse) ){
          fts5ExprPhraseFree(pPhrase);
        }else{
+          Fts5ExprTerm *p = &pNear->apPhrase[0]->aTerm[ii];
+          Fts5ExprTerm *pTo = &pPhrase->aTerm[0];
          pParse->apPhrase[pParse->nPhrase++] = pPhrase;
          pPhrase->nTerm = 1;
-          pPhrase->aTerm[0].zTerm = sqlite3Fts5Strndup(
-              &pParse->rc, pNear->apPhrase[0]->aTerm[ii].zTerm, -1
-          );
+          pTo->pTerm = sqlite3Fts5Strndup(&pParse->rc, p->pTerm, p->nFullTerm);
+          pTo->nQueryTerm = p->nQueryTerm;
+          pTo->nFullTerm = p->nFullTerm;
          pRet->apChild[ii] = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 
              0, 0, sqlite3Fts5ParseNearset(pParse, 0, pPhrase)
          );
@@ -2485,16 +2495,17 @@ static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){

  /* Determine the maximum amount of space required. */
  for(p=pTerm; p; p=p->pSynonym){
-    nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2;
+    nByte += pTerm->nQueryTerm * 2 + 3 + 2;
  }
  zQuoted = sqlite3_malloc64(nByte);

  if( zQuoted ){
    int i = 0;
    for(p=pTerm; p; p=p->pSynonym){
-      char *zIn = p->zTerm;
+      char *zIn = p->pTerm;
+      char *zEnd = &zIn[p->nQueryTerm];
      zQuoted[i++] = '"';
-      while( *zIn ){
+      while( zIn<zEnd ){
        if( *zIn=='"' ) zQuoted[i++] = '"';
        zQuoted[i++] = *zIn++;
      }
@@ -2572,8 +2583,10 @@ static char *fts5ExprPrintTcl(

      zRet = fts5PrintfAppend(zRet, " {");
      for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){
-        char *zTerm = pPhrase->aTerm[iTerm].zTerm;
-        zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm);
+        Fts5ExprTerm *p = &pPhrase->aTerm[iTerm];
+        zRet = fts5PrintfAppend(zRet, "%s%.*s", iTerm==0?"":" ", 
+            p->nQueryTerm, p->pTerm
+        );
        if( pPhrase->aTerm[iTerm].bPrefix ){
          zRet = fts5PrintfAppend(zRet, "*");
        }
@@ -2974,6 +2987,17 @@ static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){
  return 0;
 }

+/*
+** pToken is a buffer nToken bytes in size that may or may not contain
+** an embedded 0x00 byte. If it does, return the number of bytes in
+** the buffer before the 0x00. If it does not, return nToken.
+*/
+static int fts5QueryTerm(const char *pToken, int nToken){
+  int ii;
+  for(ii=0; ii<nToken && pToken[ii]; ii++){}
+  return ii;
+}
+
 static int fts5ExprPopulatePoslistsCb(
  void *pCtx,                /* Copy of 2nd argument to xTokenize() */
  int tflags,                /* Mask of FTS5_TOKEN_* flags */
@@ -2985,22 +3009,33 @@ static int fts5ExprPopulatePoslistsCb(
  Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
  Fts5Expr *pExpr = p->pExpr;
  int i;
+  int nQuery = nToken;
+  i64 iRowid = pExpr->pRoot->iRowid;

  UNUSED_PARAM2(iUnused1, iUnused2);

-  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
+  if( nQuery>FTS5_MAX_TOKEN_SIZE ) nQuery = FTS5_MAX_TOKEN_SIZE;
+  if( pExpr->pConfig->bTokendata ){
+    nQuery = fts5QueryTerm(pToken, nQuery);
+  }
  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
  for(i=0; i<pExpr->nPhrase; i++){
-    Fts5ExprTerm *pTerm;
+    Fts5ExprTerm *pT;
    if( p->aPopulator[i].bOk==0 ) continue;
-    for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
-      int nTerm = (int)strlen(pTerm->zTerm);
-      if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
-       && memcmp(pTerm->zTerm, pToken, nTerm)==0
+    for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){
+      if( (pT->nQueryTerm==nQuery || (pT->nQueryTerm<nQuery && pT->bPrefix))
+       && memcmp(pT->pTerm, pToken, pT->nQueryTerm)==0
      ){
        int rc = sqlite3Fts5PoslistWriterAppend(
            &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
        );
+        if( rc==SQLITE_OK && pExpr->pConfig->bTokendata && !pT->bPrefix ){
+          int iCol = p->iOff>>32;
+          int iTokOff = p->iOff & 0x7FFFFFFF;
+          rc = sqlite3Fts5IndexIterWriteTokendata(
+              pT->pIter, pToken, nToken, iRowid, iCol, iTokOff
+          );
+        }
        if( rc ) return rc;
        break;
      }
@@ -3135,3 +3170,81 @@ int sqlite3Fts5ExprPhraseCollist(

  return rc;
 }
+
+/*
+** Does the work of the fts5_api.xQueryToken() API method.
+*/
+int sqlite3Fts5ExprQueryToken(
+  Fts5Expr *pExpr, 
+  int iPhrase, 
+  int iToken, 
+  const char **ppOut, 
+  int *pnOut
+){
+  Fts5ExprPhrase *pPhrase = 0;
+
+  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
+    return SQLITE_RANGE;
+  }
+  pPhrase = pExpr->apExprPhrase[iPhrase];
+  if( iToken<0 || iToken>=pPhrase->nTerm ){
+    return SQLITE_RANGE;
+  }
+
+  *ppOut = pPhrase->aTerm[iToken].pTerm;
+  *pnOut = pPhrase->aTerm[iToken].nFullTerm;
+  return SQLITE_OK;
+}
+
+/*
+** Does the work of the fts5_api.xInstToken() API method.
+*/
+int sqlite3Fts5ExprInstToken(
+  Fts5Expr *pExpr, 
+  i64 iRowid,
+  int iPhrase, 
+  int iCol, 
+  int iOff, 
+  int iToken, 
+  const char **ppOut, 
+  int *pnOut
+){
+  Fts5ExprPhrase *pPhrase = 0;
+  Fts5ExprTerm *pTerm = 0;
+  int rc = SQLITE_OK;
+
+  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
+    return SQLITE_RANGE;
+  }
+  pPhrase = pExpr->apExprPhrase[iPhrase];
+  if( iToken<0 || iToken>=pPhrase->nTerm ){
+    return SQLITE_RANGE;
+  }
+  pTerm = &pPhrase->aTerm[iToken];
+  if( pTerm->bPrefix==0 ){
+    if( pExpr->pConfig->bTokendata ){
+      rc = sqlite3Fts5IterToken(
+          pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut
+      );
+    }else{
+      *ppOut = pTerm->pTerm;
+      *pnOut = pTerm->nFullTerm;
+    }
+  }
+  return rc;
+}
+
+/*
+** Clear the token mappings for all Fts5IndexIter objects mannaged by 
+** the expression passed as the only argument.
+*/
+void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){
+  int ii;
+  for(ii=0; ii<pExpr->nPhrase; ii++){
+    Fts5ExprTerm *pT;
+    for(pT=&pExpr->apExprPhrase[ii]->aTerm[0]; pT; pT=pT->pSynonym){
+      sqlite3Fts5IndexIterClearTokendata(pT->pIter);
+    }
+  }
+}
+
@@ -36,10 +36,15 @@ struct Fts5Hash {

 /*
 ** Each entry in the hash table is represented by an object of the 
-** following type. Each object, its key (a nul-terminated string) and 
-** its current data are stored in a single memory allocation. The 
-** key immediately follows the object in memory. The position list
-** data immediately follows the key data in memory.
+** following type. Each object, its key, and its current data are stored 
+** in a single memory allocation. The key immediately follows the object 
+** in memory. The position list data immediately follows the key data 
+** in memory.
+**
+** The key is Fts5HashEntry.nKey bytes in size. It consists of a single
+** byte identifying the index (either the main term index or a prefix-index),
+** followed by the term data. For example: "0token". There is no 
+** nul-terminator - in this case nKey=6.
 **
 ** The data that follows the key is in a similar, but not identical format
 ** to the doclist data stored in the database. It is:
@@ -174,8 +179,7 @@ static int fts5HashResize(Fts5Hash *pHash){
      unsigned int iHash;
      Fts5HashEntry *p = apOld[i];
      apOld[i] = p->pHashNext;
-      iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p),
-                          (int)strlen(fts5EntryKey(p)));
+      iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), p->nKey);
      p->pHashNext = apNew[iHash];
      apNew[iHash] = p;
    }
@@ -259,7 +263,7 @@ int sqlite3Fts5HashWrite(
  for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
    char *zKey = fts5EntryKey(p);
    if( zKey[0]==bByte 
-     && p->nKey==nToken
+     && p->nKey==nToken+1
     && memcmp(&zKey[1], pToken, nToken)==0 
    ){
      break;
@@ -289,9 +293,9 @@ int sqlite3Fts5HashWrite(
    zKey[0] = bByte;
    memcpy(&zKey[1], pToken, nToken);
    assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) );
-    p->nKey = nToken;
+    p->nKey = nToken+1;
    zKey[nToken+1] = '\0';
-    p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry);
+    p->nData = nToken+1 + sizeof(Fts5HashEntry);
    p->pHashNext = pHash->aSlot[iHash];
    pHash->aSlot[iHash] = p;
    pHash->nEntry++;
@@ -408,12 +412,17 @@ static Fts5HashEntry *fts5HashEntryMerge(
      *ppOut = p1;
      p1 = 0;
    }else{
-      int i = 0;
      char *zKey1 = fts5EntryKey(p1);
      char *zKey2 = fts5EntryKey(p2);
-      while( zKey1[i]==zKey2[i] ) i++;
+      int nMin = MIN(p1->nKey, p2->nKey);

-      if( ((u8)zKey1[i])>((u8)zKey2[i]) ){
+      int cmp = memcmp(zKey1, zKey2, nMin);
+      if( cmp==0 ){
+        cmp = p1->nKey - p2->nKey;
+      }
+      assert( cmp!=0 );
+
+      if( cmp>0 ){
        /* p2 is smaller */
        *ppOut = p2;
        ppOut = &p2->pScanNext;
@@ -455,7 +464,7 @@ static int fts5HashEntrySort(
    Fts5HashEntry *pIter;
    for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
      if( pTerm==0 
-       || (pIter->nKey+1>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm))
+       || (pIter->nKey>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm))
      ){
        Fts5HashEntry *pEntry = pIter;
        pEntry->pScanNext = 0;
@@ -494,12 +503,11 @@ int sqlite3Fts5HashQuery(

  for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
    zKey = fts5EntryKey(p);
-    assert( p->nKey+1==(int)strlen(zKey) );
-    if( nTerm==p->nKey+1 && memcmp(zKey, pTerm, nTerm)==0 ) break;
+    if( nTerm==p->nKey && memcmp(zKey, pTerm, nTerm)==0 ) break;
  }

  if( p ){
-    int nHashPre = sizeof(Fts5HashEntry) + nTerm + 1;
+    int nHashPre = sizeof(Fts5HashEntry) + nTerm;
    int nList = p->nData - nHashPre;
    u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10));
    if( pRet ){
@@ -560,19 +568,22 @@ int sqlite3Fts5HashScanEof(Fts5Hash *p){
 void sqlite3Fts5HashScanEntry(
  Fts5Hash *pHash,
  const char **pzTerm,            /* OUT: term (nul-terminated) */
+  int *pnTerm,                    /* OUT: Size of term in bytes */
  const u8 **ppDoclist,           /* OUT: pointer to doclist */
  int *pnDoclist                  /* OUT: size of doclist in bytes */
 ){
  Fts5HashEntry *p;
  if( (p = pHash->pScan) ){
    char *zKey = fts5EntryKey(p);
-    int nTerm = (int)strlen(zKey);
+    int nTerm = p->nKey;
    fts5HashAddPoslistSize(pHash, p, 0);
    *pzTerm = zKey;
-    *ppDoclist = (const u8*)&zKey[nTerm+1];
-    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
+    *pnTerm = nTerm;
+    *ppDoclist = (const u8*)&zKey[nTerm];
+    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm);
  }else{
    *pzTerm = 0;
+    *pnTerm = 0;
    *ppDoclist = 0;
    *pnDoclist = 0;
  }
@@ -118,7 +118,7 @@ struct Fts5FullTable {
  Fts5Global *pGlobal;            /* Global (connection wide) data */
  Fts5Cursor *pSortCsr;           /* Sort data from this cursor */
  int iSavepoint;                 /* Successful xSavepoint()+1 */
-  int bInSavepoint;
+  
 #ifdef SQLITE_DEBUG
  struct Fts5TransactionState ts;
 #endif
@@ -656,12 +656,15 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
  }
  idxStr[iIdxStr] = '\0';

-  /* Set idxFlags flags for the ORDER BY clause */
+  /* Set idxFlags flags for the ORDER BY clause
+  **
+  ** Note that tokendata=1 tables cannot currently handle "ORDER BY rowid DESC".
+  */
  if( pInfo->nOrderBy==1 ){
    int iSort = pInfo->aOrderBy[0].iColumn;
    if( iSort==(pConfig->nCol+1) && bSeenMatch ){
      idxFlags |= FTS5_BI_ORDER_RANK;
-    }else if( iSort==-1 ){
+    }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){
      idxFlags |= FTS5_BI_ORDER_ROWID;
    }
    if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){
@@ -913,6 +916,16 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
  );
  assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) );

+  /* If this cursor uses FTS5_PLAN_MATCH and this is a tokendata=1 table,
+  ** clear any token mappings accumulated at the fts5_index.c level. In
+  ** other cases, specifically FTS5_PLAN_SOURCE and FTS5_PLAN_SORTED_MATCH,
+  ** we need to retain the mappings for the entire query.  */
+  if( pCsr->ePlan==FTS5_PLAN_MATCH 
+   && ((Fts5Table*)pCursor->pVtab)->pConfig->bTokendata 
+  ){
+    sqlite3Fts5ExprClearTokens(pCsr->pExpr);
+  }
+
  if( pCsr->ePlan<3 ){
    int bSkip = 0;
    if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
@@ -2063,12 +2076,6 @@ static int fts5ApiInst(
  ){
    if( iIdx<0 || iIdx>=pCsr->nInstCount ){
      rc = SQLITE_RANGE;
-#if 0
-    }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){
-      *piPhrase = pCsr->aInst[iIdx*3];
-      *piCol = pCsr->aInst[iIdx*3 + 2];
-      *piOff = -1;
-#endif
    }else{
      *piPhrase = pCsr->aInst[iIdx*3];
      *piCol = pCsr->aInst[iIdx*3 + 1];
@@ -2323,13 +2330,56 @@ static int fts5ApiPhraseFirstColumn(
  return rc;
 }

+/*
+** xQueryToken() API implemenetation.
+*/
+static int fts5ApiQueryToken(
+  Fts5Context* pCtx, 
+  int iPhrase, 
+  int iToken, 
+  const char **ppOut, 
+  int *pnOut
+){
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
+  return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut);
+}
+
+/*
+** xInstToken() API implemenetation.
+*/
+static int fts5ApiInstToken(
+  Fts5Context *pCtx,
+  int iIdx, 
+  int iToken,
+  const char **ppOut, int *pnOut
+){
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
+  int rc = SQLITE_OK;
+  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 
+   || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) 
+  ){
+    if( iIdx<0 || iIdx>=pCsr->nInstCount ){
+      rc = SQLITE_RANGE;
+    }else{
+      int iPhrase = pCsr->aInst[iIdx*3];
+      int iCol = pCsr->aInst[iIdx*3 + 1];
+      int iOff = pCsr->aInst[iIdx*3 + 2];
+      i64 iRowid = fts5CursorRowid(pCsr);
+      rc = sqlite3Fts5ExprInstToken(
+          pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut
+      );
+    }
+  }
+  return rc;
+}
+

 static int fts5ApiQueryPhrase(Fts5Context*, int, void*, 
    int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
 );

 static const Fts5ExtensionApi sFts5Api = {
-  2,                            /* iVersion */
+  3,                            /* iVersion */
  fts5ApiUserData,
  fts5ApiColumnCount,
  fts5ApiRowCount,
@@ -2349,6 +2399,8 @@ static const Fts5ExtensionApi sFts5Api = {
  fts5ApiPhraseNext,
  fts5ApiPhraseFirstColumn,
  fts5ApiPhraseNextColumn,
+  fts5ApiQueryToken,
+  fts5ApiInstToken
 };

 /*
@@ -2615,9 +2667,7 @@ static int fts5RenameMethod(
 ){
  int rc;
  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
-  pTab->bInSavepoint = 1;
  rc = sqlite3Fts5StorageRename(pTab->pStorage, zName);
-  pTab->bInSavepoint = 0;
  return rc;
 }

@@ -2634,26 +2684,12 @@ int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
 static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
  int rc = SQLITE_OK;
-  char *zSql = 0;
+
  fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
-
-  if( pTab->bInSavepoint==0 ){
-    zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')",
-        pTab->p.pConfig->zDb, pTab->p.pConfig->zName, pTab->p.pConfig->zName
-    );
-    if( zSql ){
-      pTab->bInSavepoint = 1;
-      rc = sqlite3_exec(pTab->p.pConfig->db, zSql, 0, 0, 0);
-      pTab->bInSavepoint = 0;
-      sqlite3_free(zSql);
-    }else{
-      rc = SQLITE_NOMEM;
-    }
-    if( rc==SQLITE_OK ){
-      pTab->iSavepoint = iSavepoint+1;
-    }
+  rc = sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
+  if( rc==SQLITE_OK ){
+    pTab->iSavepoint = iSavepoint+1;
  }
-
  return rc;
 }

@@ -2914,7 +2950,7 @@ static int fts5ShadowName(const char *zName){
 ** if anything is found amiss.  Return a NULL pointer if everything is
 ** OK.
 */
-static int fts5Integrity(
+static int fts5IntegrityMethod(
  sqlite3_vtab *pVtab,    /* the FTS5 virtual table to check */
  const char *zSchema,    /* Name of schema in which this table lives */
  const char *zTabname,   /* Name of the table itself */
@@ -2972,7 +3008,7 @@ static int fts5Init(sqlite3 *db){
    /* xRelease      */ fts5ReleaseMethod,
    /* xRollbackTo   */ fts5RollbackToMethod,
    /* xShadowName   */ fts5ShadowName,
-    /* xIntegrity    */ fts5Integrity
+    /* xIntegrity    */ fts5IntegrityMethod
  };

  int rc;
@@ -244,6 +244,9 @@ static int SQLITE_TCLAPI xF5tApi(
    { "xGetAuxdataInt",    1, "CLEAR" },              /* 15 */
    { "xPhraseForeach",    4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */
    { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */
+
+    { "xQueryToken",       2, "IPHRASE ITERM" },      /* 18 */
+    { "xInstToken",        2, "IDX ITERM" },          /* 19 */
    { 0, 0, 0}
  };

@@ -500,6 +503,38 @@ static int SQLITE_TCLAPI xF5tApi(
      break;
    }

+    CASE(18, "xQueryToken") {
+      const char *pTerm = 0;
+      int nTerm = 0;
+      int iPhrase = 0;
+      int iTerm = 0;
+
+      if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
+      if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
+      rc = p->pApi->xQueryToken(p->pFts, iPhrase, iTerm, &pTerm, &nTerm);
+      if( rc==SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
+      }
+
+      break;
+    }
+
+    CASE(19, "xInstToken") {
+      const char *pTerm = 0;
+      int nTerm = 0;
+      int iIdx = 0;
+      int iTerm = 0;
+
+      if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ) return TCL_ERROR;
+      if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
+      rc = p->pApi->xInstToken(p->pFts, iIdx, iTerm, &pTerm, &nTerm);
+      if( rc==SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
+      }
+
+      break;
+    }
+
    default: 
      assert( 0 );
      break;
@@ -1117,6 +1152,176 @@ static int SQLITE_TCLAPI f5tRegisterTok(
  return TCL_OK;
 }

+typedef struct OriginTextCtx OriginTextCtx;
+struct OriginTextCtx {
+  sqlite3 *db;
+  fts5_api *pApi;
+};
+
+typedef struct OriginTextTokenizer OriginTextTokenizer;
+struct OriginTextTokenizer {
+  Fts5Tokenizer *pTok;            /* Underlying tokenizer object */
+  fts5_tokenizer tokapi;          /* API implementation for pTok */
+};
+
+/*
+** Delete the OriginTextCtx object indicated by the only argument.
+*/
+static void f5tOrigintextTokenizerDelete(void *pCtx){
+  OriginTextCtx *p = (OriginTextCtx*)pCtx;
+  ckfree(p);
+}
+
+static int f5tOrigintextCreate(
+  void *pCtx, 
+  const char **azArg, 
+  int nArg, 
+  Fts5Tokenizer **ppOut
+){
+  OriginTextCtx *p = (OriginTextCtx*)pCtx;
+  OriginTextTokenizer *pTok = 0;
+  void *pTokCtx = 0;
+  int rc = SQLITE_OK;
+
+  pTok = (OriginTextTokenizer*)sqlite3_malloc(sizeof(OriginTextTokenizer));
+  if( pTok==0 ){
+    rc = SQLITE_NOMEM;
+  }else if( nArg<1 ){
+    rc = SQLITE_ERROR;
+  }else{
+    /* Locate the underlying tokenizer */
+    rc = p->pApi->xFindTokenizer(p->pApi, azArg[0], &pTokCtx, &pTok->tokapi);
+  }
+
+  /* Create the new tokenizer instance */
+  if( rc==SQLITE_OK ){
+    rc = pTok->tokapi.xCreate(pTokCtx, &azArg[1], nArg-1, &pTok->pTok);
+  }
+
+  if( rc!=SQLITE_OK ){
+    sqlite3_free(pTok);
+    pTok = 0;
+  }
+  *ppOut = (Fts5Tokenizer*)pTok;
+  return rc;
+}
+
+static void f5tOrigintextDelete(Fts5Tokenizer *pTokenizer){
+  OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
+  if( p->pTok ){
+    p->tokapi.xDelete(p->pTok);
+  }
+  sqlite3_free(p);
+}
+
+typedef struct OriginTextCb OriginTextCb;
+struct OriginTextCb {
+  void *pCtx;
+  const char *pText; 
+  int nText;
+  int (*xToken)(void *, int, const char *, int, int, int);
+
+  char *aBuf;                     /* Buffer to use */
+  int nBuf;                       /* Allocated size of aBuf[] */
+};
+
+static int xOriginToken(
+  void *pCtx,                     /* Copy of 2nd argument to xTokenize() */
+  int tflags,                     /* Mask of FTS5_TOKEN_* flags */
+  const char *pToken,             /* Pointer to buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStart,                     /* Byte offset of token within input text */
+  int iEnd                        /* Byte offset of end of token within input */
+){
+  OriginTextCb *p = (OriginTextCb*)pCtx;
+  int ret = 0;
+
+  if( nToken==(iEnd-iStart) && 0==memcmp(pToken, &p->pText[iStart], nToken) ){
+    /* Token exactly matches document text. Pass it through as is. */
+    ret = p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd);
+  }else{
+    int nReq = nToken + 1 + (iEnd-iStart);
+    if( nReq>p->nBuf ){
+      sqlite3_free(p->aBuf);
+      p->aBuf = sqlite3_malloc(nReq*2);
+      if( p->aBuf==0 ) return SQLITE_NOMEM;
+      p->nBuf = nReq*2;
+    }
+
+    memcpy(p->aBuf, pToken, nToken);
+    p->aBuf[nToken] = '\0';
+    memcpy(&p->aBuf[nToken+1], &p->pText[iStart], iEnd-iStart);
+    ret = p->xToken(p->pCtx, tflags, p->aBuf, nReq, iStart, iEnd);
+  }
+
+  return ret;
+}
+
+
+static int f5tOrigintextTokenize(
+  Fts5Tokenizer *pTokenizer, 
+  void *pCtx,
+  int flags,                      /* Mask of FTS5_TOKENIZE_* flags */
+  const char *pText, int nText, 
+  int (*xToken)(void *, int, const char *, int, int, int)
+){
+  OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
+  OriginTextCb cb;
+  int ret;
+
+  memset(&cb, 0, sizeof(cb));
+  cb.pCtx = pCtx;
+  cb.pText = pText;
+  cb.nText = nText;
+  cb.xToken = xToken;
+
+  ret = p->tokapi.xTokenize(p->pTok,(void*)&cb,flags,pText,nText,xOriginToken);
+  sqlite3_free(cb.aBuf);
+  return ret;
+}
+
+/*
+**      sqlite3_fts5_register_origintext DB
+**
+** Description...
+*/
+static int SQLITE_TCLAPI f5tRegisterOriginText(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3 *db = 0;
+  fts5_api *pApi = 0;
+  int rc;
+  fts5_tokenizer tok = {0, 0, 0};
+  OriginTextCtx *pCtx = 0;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }
+  if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR;
+
+  pCtx = (OriginTextCtx*)ckalloc(sizeof(OriginTextCtx));
+  pCtx->db = db;
+  pCtx->pApi = pApi;
+
+  tok.xCreate = f5tOrigintextCreate;
+  tok.xDelete = f5tOrigintextDelete;
+  tok.xTokenize = f5tOrigintextTokenize;
+  rc = pApi->xCreateTokenizer(
+      pApi, "origintext", (void*)pCtx, &tok, f5tOrigintextTokenizerDelete
+  );
+
+  Tcl_ResetResult(interp);
+  if( rc!=SQLITE_OK ){
+    Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
+    return TCL_ERROR;
+  }
+  return TCL_OK;
+}
+
 /*
 ** Entry point.
 */
@@ -1133,7 +1338,8 @@ int Fts5tcl_Init(Tcl_Interp *interp){
    { "sqlite3_fts5_may_be_corrupt",     f5tMayBeCorrupt, 0 },
    { "sqlite3_fts5_token_hash",         f5tTokenHash, 0 },
    { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 },
-    { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 }
+    { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 },
+    { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 }
  };
  int i;
  F5tTokenizerContext *pContext;
@@ -629,7 +629,7 @@ static int fts5VocabFilterMethod(
  if( pEq ){
    zTerm = (const char *)sqlite3_value_text(pEq);
    nTerm = sqlite3_value_bytes(pEq);
-    f = 0;
+    f = FTS5INDEX_QUERY_NOTOKENDATA;
  }else{
    if( pGe ){
      zTerm = (const char *)sqlite3_value_text(pGe);
@@ -438,6 +438,20 @@ proc detail_is_none {} { detail_check ; expr {$::detail == "none"} }
 proc detail_is_col {}  { detail_check ; expr {$::detail == "col" } }
 proc detail_is_full {} { detail_check ; expr {$::detail == "full"} }

+proc foreach_tokenizer_mode {prefix script} {
+  set saved $::testprefix
+  foreach {d mapping} {
+    ""              {}
+    "-origintext"   {, tokenize="origintext unicode61", tokendata=1}
+  } {
+    set s [string map [list %TOKENIZER% $mapping] $script]
+    set ::testprefix "$prefix$d"
+    reset_db
+    sqlite3_fts5_register_origintext db
+    uplevel $s
+  }
+  set ::testprefix $saved
+}

 #-------------------------------------------------------------------------
 # Convert a poslist of the type returned by fts5_test_poslist() to a 
@@ -22,6 +22,7 @@ ifcapable !fts5 {
 }

 foreach_detail_mode $::testprefix {
+foreach_tokenizer_mode $::testprefix {

 do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
@@ -44,7 +45,7 @@ do_execsql_test 1.1 {
 #

 do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
 }
 do_execsql_test 2.1 {
  INSERT INTO t1 VALUES('a b c', 'd e f');
@@ -73,8 +74,9 @@ do_execsql_test 2.4 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 3.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
 }
 foreach {i x y} {
   1  {g f d b f} {h h e i a}
@@ -97,8 +99,9 @@ foreach {i x y} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 4.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }
 foreach {i x y} {
@@ -121,8 +124,9 @@ foreach {i x y} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 5.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }
 foreach {i x y} {
@@ -145,8 +149,9 @@ foreach {i x y} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 6.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }

@@ -181,6 +186,7 @@ do_execsql_test 6.6 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 expr srand(0)
 do_execsql_test 7.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x,y,z);
@@ -222,6 +228,7 @@ for {set i 1} {$i <= 10} {incr i} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 8.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3");
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
@@ -236,6 +243,7 @@ do_execsql_test 8.1 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db

 expr srand(0)

@@ -280,8 +288,9 @@ for {set i 1} {$i <= 10} {incr i} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 10.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
 }
 set d10 {
   1  {g f d b f} {h h e i a}
@@ -314,19 +323,19 @@ do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
 #-------------------------------------------------------------------------
 #
 do_catchsql_test 11.1 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL% %TOKENIZER%);
 } {1 {reserved fts5 column name: rank}}
 do_catchsql_test 11.2 {
-  CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL% %TOKENIZER%);
 } {1 {reserved fts5 table name: rank}}
 do_catchsql_test 11.3 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL% %TOKENIZER%);
 } {1 {reserved fts5 column name: rowid}}

 #-------------------------------------------------------------------------
 #
 do_execsql_test 12.1 {
-  CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
 } {}

 do_catchsql_test 12.2 {
@@ -341,8 +350,9 @@ do_test 12.3 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 13.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
 } {}

@@ -365,8 +375,9 @@ do_execsql_test 13.6 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 14.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  WITH d(x,y) AS (
    SELECT NULL, 'xyz xyz xyz xyz xyz xyz'
@@ -449,8 +460,9 @@ do_catchsql_test 16.2 {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 17.1 {
-  CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO b2 VALUES('a');
  INSERT INTO b2 VALUES('b');
  INSERT INTO b2 VALUES('c');
@@ -466,8 +478,9 @@ do_test 17.2 {

 if {[string match n* %DETAIL%]==0} {
  reset_db
+  sqlite3_fts5_register_origintext db
  do_execsql_test 17.3 {
-    CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%);
+    CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
    INSERT INTO c2 VALUES('x x x', 'x x x');
    SELECT rowid FROM c2 WHERE c2 MATCH 'y:x';
  } {1}
@@ -476,8 +489,9 @@ if {[string match n* %DETAIL%]==0} {
 #-------------------------------------------------------------------------
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 17.1 {
-  CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO uio VALUES(NULL);
  INSERT INTO uio SELECT NULL FROM uio;
  INSERT INTO uio SELECT NULL FROM uio;
@@ -524,8 +538,8 @@ do_execsql_test 17.9 {
 #--------------------------------------------------------------------
 #
 do_execsql_test 18.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
-  CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1 VALUES('abc*', NULL);
  INSERT INTO t2 VALUES(1, 'abcdefg');
 }
@@ -540,8 +554,9 @@ do_execsql_test 18.3 {
 # fts5 table in the temp schema.
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 19.0 {
-  CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t1 VALUES('x y z');
  INSERT INTO t1 VALUES('w x 1');
  SELECT rowid FROM t1 WHERE t1 MATCH 'x';
@@ -551,8 +566,9 @@ do_execsql_test 19.0 {
 # Test that 6 and 7 byte varints can be read.
 #
 reset_db
+sqlite3_fts5_register_origintext db
 do_execsql_test 20.0 {
-  CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL% %TOKENIZER%);
 }
 set ::ids [list \
  0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43]
@@ -570,7 +586,7 @@ do_test 20.1 {
 #
 do_execsql_test 21.0 {
  CREATE TEMP TABLE t8(a, b);
-  CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL% %TOKENIZER%);
 }

 do_execsql_test 21.1 {
@@ -581,7 +597,7 @@ do_execsql_test 21.1 {
 }

 do_execsql_test 22.0 {
-  CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t9(rowid, x) VALUES(2, 'bbb');
  BEGIN;
    INSERT INTO t9(rowid, x) VALUES(1, 'aaa');
@@ -596,7 +612,7 @@ do_execsql_test 22.1 {

 #-------------------------------------------------------------------------
 do_execsql_test 23.0 {
-  CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  CREATE TABLE t11(x);
 }
 do_execsql_test 23.1 {
@@ -608,7 +624,7 @@ do_execsql_test 23.2 {

 #-------------------------------------------------------------------------
 do_execsql_test 24.0 {
-  CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
  INSERT INTO t12 VALUES('aaaa');
 }
 do_execsql_test 24.1 {
@@ -618,6 +634,9 @@ do_execsql_test 24.1 {
    INSERT INTO t12 VALUES('aaaa');
  END;
 }
+execsql_pp {
+  SELECT rowid, hex(block) FROM t12_data
+}
 do_execsql_test 24.2 {
  INSERT INTO t12(t12) VALUES('integrity-check');
 }
@@ -627,7 +646,7 @@ do_execsql_test 24.3 {

 #-------------------------------------------------------------------------
 do_execsql_test 25.0 {
-  CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
 }
 do_execsql_test 25.1 {
  BEGIN;
@@ -638,6 +657,7 @@ SELECT * FROM t13('BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
 }


+}
 }

 expand_all_sql db
@@ -966,6 +966,262 @@ do_catchsql_test 6.2 {
  UPDATE t1 SET content=randomblob(500) WHERE t1;
 } {1 {constraint failed}}

+#-------------------------------------------------------------------------
+reset_db
+do_test 7.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+.open --hexdb
+| size 40960 pagesize 4096 filename crash-d8b4a99207c10b.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04   ................
+|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     96: 00 00 00 00 0d 00 00 00 0d 0b 62 00 0f 97 0f 40   ..........b....@
+|    112: 0e d5 0e 75 0e 18 0d c0 0d 66 0d 0f 0c a4 0c 44   ...u.....f.....D
+|    128: 0b ec 0b a7 0b 62 00 00 00 00 00 00 00 00 00 00   .....b..........
+|   2912: 00 00 43 0d 06 17 11 11 08 75 74 61 62 6c 65 74   ..C......utablet
+|   2928: 34 74 34 43 52 45 41 54 45 20 56 49 52 54 55 41   4t4CREATE VIRTUA
+|   2944: 4c 20 54 41 42 4c 45 20 74 34 20 55 53 49 4e 47   L TABLE t4 USING
+|   2960: 20 66 74 73 35 76 6f 63 61 62 28 27 74 32 27 2c    fts5vocab('t2',
+|   2976: 20 27 72 6f 77 27 29 43 0c 06 17 11 11 08 75 74    'row')C......ut
+|   2992: 61 62 6c 65 74 33 74 33 43 52 45 41 54 45 20 56   ablet3t3CREATE V
+|   3008: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 33 20   IRTUAL TABLE t3 
+|   3024: 55 53 49 4e 47 20 66 74 73 35 76 6f 63 61 62 28   USING fts5vocab(
+|   3040: 27 74 31 27 2c 20 27 72 6f 77 27 29 56 0b 06 17   't1', 'row')V...
+|   3056: 1f 1f 01 7d 74 61 62 6c 65 74 32 5f 63 6f 6e 66   ....tablet2_conf
+|   3072: 69 67 74 32 5f 63 6f 6e 66 69 67 0a 43 52 45 41   igt2_config.CREA
+|   3088: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 63 6f 6e   TE TABLE 't2_con
+|   3104: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b   fig'(k PRIMARY K
+|   3120: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52   EY, v) WITHOUT R
+|   3136: 4f 57 49 44 5e 0a 07 17 21 21 01 81 07 74 61 62   OWID^...!!...tab
+|   3152: 6c 65 74 32 5f 63 6f 6e 74 65 6e 74 74 32 5f 63   let2_contentt2_c
+|   3168: 6f 6e 74 65 6e 74 09 43 52 45 41 54 45 20 54 41   ontent.CREATE TA
+|   3184: 42 4c 45 20 27 74 32 5f 63 6f 6e 74 65 6e 74 27   BLE 't2_content'
+|   3200: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d   (id INTEGER PRIM
+|   3216: 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c   ARY KEY, c0, c1,
+|   3232: 20 63 32 29 69 09 07 17 19 19 01 81 2d 74 61 62    c2)i.......-tab
+|   3248: 6c 65 74 32 5f 69 64 78 74 32 5f 69 64 78 08 43   let2_idxt2_idx.C
+|   3264: 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 32 5f   REATE TABLE 't2_
+|   3280: 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d   idx'(segid, term
+|   3296: 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20   , pgno, PRIMARY 
+|   3312: 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29   KEY(segid, term)
+|   3328: 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55   ) WITHOUT ROWIDU
+|   3344: 08 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 32 5f   ........tablet2_
+|   3360: 64 61 74 61 74 32 5f 64 61 74 61 07 43 52 45 41   datat2_data.CREA
+|   3376: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 64 61 74   TE TABLE 't2_dat
+|   3392: 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52   a'(id INTEGER PR
+|   3408: 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b   IMARY KEY, block
+|   3424: 20 42 4c 4f 42 29 58 07 07 17 11 11 08 81 1d 74    BLOB)X........t
+|   3440: 61 62 6c 65 74 32 74 32 43 52 45 41 54 45 20 56   ablet2t2CREATE V
+|   3456: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 32 20   IRTUAL TABLE t2 
+|   3472: 55 53 49 4e 47 20 66 74 73 35 28 27 61 27 2c 5b   USING fts5('a',[
+|   3488: 62 5d 2c 22 63 22 2c 64 65 74 61 69 6c 3d 6e 6f   b],.c.,detail=no
+|   3504: 6e 65 2c 63 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29   ne,columnsize=0)
+|   3520: 56 06 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f   V.......tablet1_
+|   3536: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 06   configt1_config.
+|   3552: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
+|   3568: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41   _config'(k PRIMA
+|   3584: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f   RY KEY, v) WITHO
+|   3600: 55 54 20 52 4f 57 49 44 5b 05 07 17 21 21 01 81   UT ROWID[...!!..
+|   3616: 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65   .tablet1_docsize
+|   3632: 74 31 5f 64 6f 63 73 69 7a 65 05 43 52 45 41 54   t1_docsize.CREAT
+|   3648: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73   E TABLE 't1_docs
+|   3664: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20   ize'(id INTEGER 
+|   3680: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20   PRIMARY KEY, sz 
+|   3696: 42 4c 4f 42 29 5e 04 07 17 21 21 01 81 07 74 61   BLOB)^...!!...ta
+|   3712: 62 6c 65 74 31 5f 63 6f 6e 74 65 6e 74 74 31 5f   blet1_contentt1_
+|   3728: 63 6f 6e 74 65 6e 74 04 43 52 45 41 54 45 20 54   content.CREATE T
+|   3744: 41 42 4c 45 20 27 74 31 5f 63 6f 6e 74 65 6e 74   ABLE 't1_content
+|   3760: 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49   '(id INTEGER PRI
+|   3776: 4d 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31   MARY KEY, c0, c1
+|   3792: 2c 20 63 32 29 69 03 07 17 19 19 01 81 2d 74 61   , c2)i.......-ta
+|   3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03   blet1_idxt1_idx.
+|   3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
+|   3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72   _idx'(segid, ter
+|   3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59   m, pgno, PRIMARY
+|   3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d    KEY(segid, term
+|   3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44   )) WITHOUT ROWID
+|   3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31   U........tablet1
+|   3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45   _datat1_data.CRE
+|   3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61   ATE TABLE 't1_da
+|   3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50   ta'(id INTEGER P
+|   3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63   RIMARY KEY, bloc
+|   3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b   k BLOB)g.......;
+|   4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20   tablet1t1CREATE 
+|   4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31   VIRTUAL TABLE t1
+|   4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 62 20    USING fts5(a,b 
+|   4048: 75 6e 69 6e 64 65 78 65 64 2c 63 2c 74 6f 6b 65   unindexed,c,toke
+|   4064: 6e 69 7a 65 3d 22 70 6f 72 74 65 72 20 61 73 63   nize=.porter asc
+|   4080: 69 69 22 2c 74 6f 6b 65 6e 64 61 74 61 3d 31 29   ii.,tokendata=1)
+| page 2 offset 4096
+|      0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c   ..h............|
+|     16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00   .*..............
+|   3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00   .......0........
+|   3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80   ..........7.....
+|   3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01   ...t.....0a.....
+|   3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01   b.....c.....g...
+|   3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01   ....h.......i...
+|   3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00   ............*...
+|   3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80   ............%...
+|   3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02   .....P.....0g...
+|   3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02   .....h........i.
+|   4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 7f f1 03   ........7.......
+|   4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01   .t.....0a.....b.
+|   4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02   ....c.....g.....
+|   4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 04   ..h.......i.....
+|   4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00   ................
+|   4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
+| page 3 offset 8192
+|      0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c   ................
+|   4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02   ................
+| page 4 offset 12288
+|      0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00   ................
+|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
+|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
+|   4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69    h i.......g h i
+|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
+|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
+| page 5 offset 16384
+|      0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03   ................
+|   4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03   ................
+| page 6 offset 20480
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| page 7 offset 24576
+|      0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00   ................
+|   3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84   ..............A.
+|   4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61   ............4.0a
+|   4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01   .....b.....c....
+|   4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01   .d...e...f...g..
+|   4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06   ...h.....i......
+|   4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f   ................
+|   4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
+| page 8 offset 28672
+|      0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
+| page 9 offset 32768
+|      0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00   ................
+|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
+|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
+|   4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69    h i.......g h i
+|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
+|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
+| page 10 offset 36864
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| end crash-d8b4a99207c10b.db
+}]} {}
+
+do_catchsql_test 7.1 {
+  SELECT snippet(t1, -1, '.', '..', '[', ']'), 
+         highlight(t1, 2, '[', ']') 
+           FROM t1('g + h') 
+           WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank;
+} {1 {database disk image is malformed}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_test 8.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+.open --hexdb
+| size 20480 pagesize 4096 filename crash-d57c01958e48ab.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 05   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 04   ................
+|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     96: 00 00 00 00 0d 00 00 00 05 0e 10 00 0f 97 0f 40   ...............@
+|    112: 0e d5 0e 68 0e 10 01 00 00 00 00 00 00 00 00 00   ...h............
+|   3600: 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f   V.......tablet1_
+|   3616: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 05   configt1_config.
+|   3632: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
+|   3648: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41   _config'(k PRIMA
+|   3664: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f   RY KEY, v) WITHO
+|   3680: 55 54 20 52 4f 57 49 44 6b 04 07 17 21 21 01 81   UT ROWIDk...!!..
+|   3696: 21 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65   !tablet1_docsize
+|   3712: 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 45 41 54   t1_docsize.CREAT
+|   3728: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73   E TABLE 't1_docs
+|   3744: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20   ize'(id INTEGER 
+|   3760: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20   PRIMARY KEY, sz 
+|   3776: 42 4c 4f 42 2c 20 6f 72 69 67 69 6e 20 49 4e 54   BLOB, origin INT
+|   3792: 45 47 45 52 29 69 03 07 17 19 19 01 81 2d 74 61   EGER)i.......-ta
+|   3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03   blet1_idxt1_idx.
+|   3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
+|   3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72   _idx'(segid, ter
+|   3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59   m, pgno, PRIMARY
+|   3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d    KEY(segid, term
+|   3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44   )) WITHOUT ROWID
+|   3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31   U........tablet1
+|   3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45   _datat1_data.CRE
+|   3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61   ATE TABLE 't1_da
+|   3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50   ta'(id INTEGER P
+|   3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63   RIMARY KEY, bloc
+|   3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b   k BLOB)g.......;
+|   4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20   tablet1t1CREATE 
+|   4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31   VIRTUAL TABLE t1
+|   4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 20 62    USING fts5(a, b
+|   4048: 2c 20 63 6f 6e 74 65 6e 74 3d 27 27 2c 20 63 6f   , content='', co
+|   4064: 6e 74 65 6e 74 6c 65 73 73 5f 64 65 6c 65 74 65   ntentless_delete
+|   4080: 3d 31 2c 20 74 6f 6b 65 6e 64 61 74 61 3d 31 29   =1, tokendata=1)
+| page 2 offset 4096
+|      0: 0d 0f eb 00 03 0e 17 00 0f e2 0e 17 0e 31 00 00   .............1..
+|     16: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
+|   3600: 00 00 00 00 00 00 00 18 0a 03 00 36 00 00 00 00   ...........6....
+|   3616: ff 00 00 01 01 01 01 00 01 01 01 01 01 01 00 00   ................
+|   3632: 07 83 29 84 80 80 80 80 01 04 00 86 56 00 00 01   ..).........V...
+|   3648: 96 04 30 61 61 61 01 02 02 01 04 02 04 01 08 02   ..0aaa..........
+|   3664: 04 04 04 01 10 02 04 04 04 04 04 04 04 01 20 02   .............. .
+|   3680: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 01   ................
+|   3696: 40 02 04 04 04 04 04 04 04 04 04 04 04 04 04 04   @...............
+|   3712: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3728: 04 01 81 00 02 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3744: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3760: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3776: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
+|   3792: 04 04 04 04 02 02 62 63 01 06 01 01 02 01 03 62   ......bc.......b
+|   3808: 62 62 02 02 03 01 04 03 06 01 08 03 06 06 06 01   bb..............
+|   3824: 10 03 06 06 06 06 06 06 06 01 20 03 06 06 06 06   .......... .....
+|   3840: 06 06 06 06 06 06 06 06 06 06 06 01 40 03 06 06   ............@...
+|   3856: 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06   ................
+|   3872: 06 06 06 06 06 06 06 06 06 06 16 06 06 02 02 63   ...............c
+|   3888: 64 02 06 01 01 02 01 03 63 63 63 03 02 05 01 04   d.......ccc.....
+|   3904: 05 0a 01 08 05 0a 0a 0a 01 10 05 0a 0a 0a 0a 0a   ................
+|   3920: 0a 0a 01 20 05 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a   ... ............
+|   3936: 0a 0a 0a 0a 02 02 64 65 03 06 01 01 02 01 03 64   ......de.......d
+|   3952: 64 64 04 02 09 01 04 09 12 01 08 09 12 12 12 01   dd..............
+|   3968: 10 09 12 12 12 12 12 12 12 02 02 65 66 04 06 01   ...........ef...
+|   3984: 01 02 01 03 65 65 65 05 02 11 01 04 11 22 01 08   ....eee.........
+|   4000: 11 22 22 22 02 02 66 67 05 06 01 01 02 01 03 66   ......fg.......f
+|   4016: 56 66 06 02 21 01 04 21 42 02 02 67 68 06 06 01   Vf..!..!B..gh...
+|   4032: 01 02 cb 03 67 67 67 07 02 41 02 02 68 69 07 06   ....ggg..A..hi..
+|   4048: 01 01 02 04 81 13 09 50 09 2e 09 1c 09 12 09 0c   .......P........
+|   4064: 09 08 07 01 03 00 14 07 81 77 07 00 00 00 15 22   .........w......
+|   4080: 00 00 00 00 ff 00 00 01 00 00 00 00 00 00 05 0c   ................
+| page 3 offset 8192
+|      0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
+| page 4 offset 12288
+|      0: 0d 00 00 00 07 0f c8 00 0f f8 0f f0 0f e8 0f e0   ................
+|     16: 0f d8 0f d0 0f c8 00 00 00 00 00 00 00 00 00 00   ................
+|   4032: 00 00 00 00 00 00 00 00 06 07 04 00 10 09 7f 01   ................
+|   4048: 06 06 04 00 10 09 3f 01 06 05 04 00 10 09 1f 01   ......?.........
+|   4064: 06 04 04 00 10 09 0f 01 06 03 04 00 10 09 07 01   ................
+|   4080: 06 02 04 00 10 09 03 01 06 01 04 00 10 09 01 01   ................
+| page 5 offset 16384
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| end crash-d57c01958e48ab.db
+}]} {}
+
+do_catchsql_test 8.1 {
+  SELECT rowid FROM t1('a* NOT ý') ;
+} {0 {1 2 3 4 5 6 7}}
+
 sqlite3_fts5_may_be_corrupt 0
 finish_test

@@ -0,0 +1,141 @@
+# 2010 June 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultG
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+set ::testprefix fts5faultH
+
+sqlite3_fts5_register_origintext db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+
+  BEGIN;
+    INSERT INTO t1 VALUES('oNe tWo thRee');
+    INSERT INTO t1 VALUES('One Two Three');
+    INSERT INTO t1 VALUES('onE twO threE');
+  COMMIT;
+  BEGIN;
+    INSERT INTO t1 VALUES('one two three');
+    INSERT INTO t1 VALUES('one two three');
+    INSERT INTO t1 VALUES('one two three');
+  COMMIT;
+}
+
+do_faultsim_test 1 -faults oom* -prep { 
+} -body {
+  execsql {
+    SELECT rowid FROM t1('three');
+  }
+} -test {
+  faultsim_integrity_check
+  faultsim_test_result {0 {1 2 3 4 5 6}}
+}
+
+
+reset_db
+sqlite3_fts5_register_origintext db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+
+  BEGIN;
+    INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(12, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(13, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(14, 'bbb BBB bbb');
+    INSERT INTO t1(rowid, x) VALUES(15, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(16, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(17, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(18, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(19, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(20, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(21, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(22, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(23, 'bbb bbb bbb');
+    INSERT INTO t1(rowid, x) VALUES(24, 'aaa bbb BBB');
+  COMMIT;
+}
+
+do_faultsim_test 2 -faults oom* -prep { 
+} -body {
+  execsql {
+    SELECT rowid FROM t1('BBB AND AAA');
+  }
+} -test {
+  faultsim_integrity_check
+  faultsim_test_result {0 {10 24}}
+}
+
+reset_db
+sqlite3_fts5_register_origintext db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+
+  INSERT INTO t1(rowid, x) VALUES(9, 'bbb Bbb BBB');
+  BEGIN;
+    INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(11, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(12, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(13, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(14, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(15, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(16, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(17, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(18, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(19, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(20, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(21, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(22, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(23, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(24, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(25, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(26, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(27, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(28, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(29, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(30, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(31, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(32, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(33, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(34, 'bbb Bbb BBB');
+    INSERT INTO t1(rowid, x) VALUES(35, 'aaa bbb BBB');
+  COMMIT;
+}
+
+do_faultsim_test 3 -faults oom* -prep { 
+} -body {
+  execsql {
+    SELECT rowid FROM t1('BBB AND AAA');
+  }
+} -test {
+  faultsim_integrity_check
+  faultsim_test_result {0 {10 35}}
+}
+
+
+finish_test
@@ -91,7 +91,6 @@ do_execsql_test 2.2.1 {
  INSERT INTO vt0(c0) VALUES ('xyz');
 }

-breakpoint
 do_execsql_test 2.2.2 {
  ALTER TABLE t0 RENAME TO t1;
 }
@@ -500,6 +499,21 @@ do_execsql_test 17.5 {
  SELECT c0 FROM t0 WHERE c0 GLOB '*faul*';
 } {assertionfaultproblem}

+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 18.0 {
+  BEGIN;
+    CREATE VIRTUAL TABLE t1 USING fts5(text);
+    ALTER TABLE t1 RENAME TO t2;
+}
+
+do_execsql_test 18.1 {
+    DROP TABLE t2;
+}
+
+do_execsql_test 18.2 {
+  COMMIT;
+}

 finish_test

@@ -0,0 +1,297 @@
+# 2014 Jan 08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", detail=%DETAIL%
+  );
+  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO ft VALUES('Hello world');
+}
+
+do_execsql_test 1.2 {
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+proc b {x} { string map [list "\0" "."] $x }
+db func b b
+
+do_execsql_test 1.3 {
+  select b(term) from vocab;
+} {
+  hello.Hello
+  world
+}
+
+do_execsql_test 1.4 {
+  SELECT rowid FROM ft('Hello');
+} {1}
+
+#-------------------------------------------------------------------------
+reset_db
+
+# Return a random integer between 0 and n-1.
+#
+proc random {n} {
+  expr {abs(int(rand()*$n))}
+}
+
+proc select_one {list} {
+  set n [llength $list]
+  lindex $list [random $n]
+}
+
+proc term {} {
+  set first_letter {
+    a b c d e f g h i j k l m n o p q r s t u v w x y z
+    A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
+  }
+
+  set term [select_one $first_letter]
+  append term [random 100]
+}
+
+proc document {} {
+  set nTerm [expr [random 5] + 5]
+  set doc ""
+  for {set ii 0} {$ii < $nTerm} {incr ii} {
+    lappend doc [term]
+  }
+  set doc
+}
+db func document document
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", detail=%DETAIL%
+  );
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 128);
+  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
+}
+
+do_test 2.1 {
+  for {set ii 0} {$ii < 500} {incr ii} {
+    execsql { INSERT INTO ft VALUES( document() ) }
+  }
+} {}
+
+do_execsql_test 2.2 {
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+do_execsql_test 2.3 {
+  INSERT INTO ft(ft, rank) VALUES('merge', 16);
+}
+
+do_execsql_test 2.4 {
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+do_execsql_test 2.5 {
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", detail=%DETAIL%
+  );
+  CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
+
+  INSERT INTO ft(rowid, x) VALUES(1, 'hello');
+  INSERT INTO ft(rowid, x) VALUES(2, 'Hello');
+  INSERT INTO ft(rowid, x) VALUES(3, 'HELLO');
+}
+
+#proc b {x} { string map [list "\0" "."] $x }
+#db func b b
+#execsql_pp { SELECT b(term) FROM vocab }
+
+do_execsql_test 3.1.1 { SELECT rowid FROM ft('hello') } 1
+do_execsql_test 3.1.2 { SELECT rowid FROM ft('Hello') } 2
+do_execsql_test 3.1.3 { SELECT rowid FROM ft('HELLO') } 3
+
+do_execsql_test 3.2 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(x, 
+      tokenize="origintext unicode61", 
+      tokendata=1,
+      detail=%DETAIL%
+  );
+  CREATE VIRTUAL TABLE vocab2 USING fts5vocab(ft2, instance);
+
+  INSERT INTO ft2(rowid, x) VALUES(1, 'hello');
+  INSERT INTO ft2(rowid, x) VALUES(2, 'Hello');
+  INSERT INTO ft2(rowid, x) VALUES(3, 'HELLO');
+
+  INSERT INTO ft2(rowid, x) VALUES(10, 'helloooo');
+}
+
+#proc b {x} { string map [list "\0" "."] $x }
+#db func b b
+#execsql_pp { SELECT b(term) FROM vocab }
+
+do_execsql_test 3.3.1 { SELECT rowid FROM ft2('hello') } {1 2 3}
+do_execsql_test 3.3.2 { SELECT rowid FROM ft2('Hello') } {1 2 3}
+do_execsql_test 3.3.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3}
+
+do_execsql_test 3.3.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+sqlite3_fts5_register_origintext db
+proc querytoken {cmd iPhrase iToken} { 
+  set txt [$cmd xQueryToken $iPhrase $iToken]
+  string map [list "\0" "."] $txt
+}
+sqlite3_fts5_create_function db querytoken querytoken
+
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
+  );
+  INSERT INTO ft VALUES('one two three four');
+}
+
+do_execsql_test 4.1 {
+  SELECT rowid, querytoken(ft, 0, 0) FROM ft('TwO')
+} {1 two.TwO}
+do_execsql_test 4.2 {
+  SELECT rowid, querytoken(ft, 0, 0) FROM ft('one TWO ThreE')
+} {1 one}
+do_execsql_test 4.3 {
+  SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE')
+} {1 two.TWO}
+
+if {"%DETAIL%"=="full"} {
+  # Phrase queries are only supported for detail=full.
+  #
+  do_execsql_test 4.4 {
+    SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"')
+  } {1 three.ThreE}
+  do_catchsql_test 4.5 {
+    SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"')
+  } {1 SQLITE_RANGE}
+  do_catchsql_test 4.6 {
+    SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"')
+  } {1 SQLITE_RANGE}
+  do_catchsql_test 4.7 {
+    SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"')
+  } {1 SQLITE_RANGE}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+sqlite3_fts5_register_origintext db
+proc insttoken {cmd iIdx iToken} { 
+  set txt [$cmd xInstToken $iIdx $iToken]
+  string map [list "\0" "."] $txt
+}
+sqlite3_fts5_create_function db insttoken insttoken
+fts5_aux_test_functions db
+
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
+  );
+  INSERT INTO ft VALUES('one ONE One oNe oNE one');
+}
+
+do_execsql_test 5.1 {
+  SELECT insttoken(ft, 0, 0), 
+         insttoken(ft, 1, 0),
+         insttoken(ft, 2, 0),
+         insttoken(ft, 3, 0),
+         insttoken(ft, 4, 0),
+         insttoken(ft, 5, 0)
+  FROM ft('one');
+} {
+  one one.ONE one.One one.oNe one.oNE one
+}
+
+do_execsql_test 5.2 {
+  SELECT insttoken(ft, 1, 0) FROM ft('one');
+} {
+  one.ONE
+}
+
+do_execsql_test 5.3 {
+  SELECT fts5_test_poslist(ft) FROM ft('one');
+} {
+  {0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5}
+}
+
+#-------------------------------------------------------------------------
+# Test the xInstToken() API with:
+#
+#   * a non tokendata=1 table.
+#   * prefix queries.
+#
+reset_db
+sqlite3_fts5_register_origintext db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, y, tokenize='origintext unicode61', detail=%DETAIL%
+  );
+
+  INSERT INTO ft VALUES('One Two', 'Three two');
+  INSERT INTO ft VALUES('three Three', 'one One');
+}
+proc tokens {cmd} { 
+  set ret [list]
+  for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+    set txt [$cmd xInstToken $iTok 0]
+    set txt [string map [list "\0" "."] $txt]
+    lappend ret $txt
+  }
+  set ret
+}
+sqlite3_fts5_create_function db tokens tokens
+
+do_execsql_test 6.1 {
+  SELECT rowid, tokens(ft) FROM ft('One');
+} {1 one.One 2 one.One}
+
+do_execsql_test 6.2 {
+  SELECT rowid, tokens(ft) FROM ft('on*');
+} {1 {{}} 2 {{} {}}}
+
+do_execsql_test 6.3 {
+  SELECT rowid, tokens(ft) FROM ft('Three*');
+} {1 {{}} 2 {{}}}
+
+}
+
+finish_test
+
@@ -0,0 +1,146 @@
+# 2014 Jan 08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+}
+
+do_execsql_test 1.1 {
+  BEGIN;
+  INSERT INTO ft VALUES('Hello');
+  INSERT INTO ft VALUES('hello');
+  INSERT INTO ft VALUES('HELLO');
+  INSERT INTO ft VALUES('today');
+  INSERT INTO ft VALUES('today');
+  INSERT INTO ft VALUES('today');
+  INSERT INTO ft VALUES('World');
+  INSERT INTO ft VALUES('world');
+  INSERT INTO ft VALUES('WORLD');
+  COMMIT;
+}
+
+do_execsql_test 1.2 { SELECT rowid FROM ft('hello'); } {1 2 3}
+do_execsql_test 1.3 { SELECT rowid FROM ft('today'); } {4 5 6}
+do_execsql_test 1.4 { SELECT rowid FROM ft('world'); } {7 8 9}
+
+do_execsql_test 1.5 {
+  SELECT count(*) FROM ft_data
+} 3
+
+do_execsql_test 1.6 {
+  DELETE FROM ft;
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
+  BEGIN;
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+    )
+    INSERT INTO ft SELECT 'Hello Hello Hello Hello Hello Hello Hello' FROM s;
+    INSERT INTO ft VALUES ('hELLO hELLO hELLO');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('today today today today today today today');
+    INSERT INTO ft VALUES('World World World World World World World');
+    INSERT INTO ft VALUES('world world world world world world world');
+    INSERT INTO ft VALUES('WORLD WORLD WORLD WORLD WORLD WORLD WORLD');
+    INSERT INTO ft VALUES('World World World World World World World');
+    INSERT INTO ft VALUES('world world world world world world world');
+    INSERT INTO ft VALUES('WORLD WORLD WORLD WORLD WORLD WORLD WORLD');
+  COMMIT;
+}
+
+do_execsql_test 1.7 {
+  SELECT count(*) FROM ft_data;
+} 23
+
+do_execsql_test 1.8 { SELECT rowid FROM ft('hello') WHERE rowid>100; } {101}
+
+do_execsql_test 1.9 {
+  DELETE FROM ft;
+  INSERT INTO ft(ft) VALUES('optimize');
+  SELECT count(*) FROM ft_data;
+} {2}
+do_execsql_test 1.10 {
+  BEGIN;
+    INSERT INTO ft VALUES('Hello');
+    INSERT INTO ft VALUES('hello');
+    INSERT INTO ft VALUES('HELLO');
+    INSERT INTO ft VALUES('today');
+    INSERT INTO ft VALUES('today');
+    INSERT INTO ft VALUES('today');
+    INSERT INTO ft VALUES('World');
+    INSERT INTO ft VALUES('world');
+    INSERT INTO ft VALUES('WORLD');
+}
+
+do_execsql_test 1.11 { SELECT rowid FROM ft('hello'); } {1 2 3}
+do_execsql_test 1.12 { SELECT rowid FROM ft('today'); } {4 5 6}
+do_execsql_test 1.13 { SELECT rowid FROM ft('world'); } {7 8 9}
+do_execsql_test 1.14 { SELECT rowid FROM ft('hello') ORDER BY rank; } {1 2 3}
+
+#------------------------------------------------------------------------
+reset_db
+sqlite3_fts5_register_origintext db
+proc tokens {cmd} { 
+  set ret [list]
+  for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+    set txt [$cmd xInstToken $iTok 0]
+    set txt [string map [list "\0" "."] $txt]
+    lappend ret $txt
+  }
+  set ret
+}
+sqlite3_fts5_create_function db tokens tokens
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(
+    v, tokenize="origintext unicode61", tokendata=1, detail=none
+  );
+
+  INSERT INTO x1 VALUES('xxx Xxx XXX yyy YYY yyy');
+  INSERT INTO x1 VALUES('xxx yyy xxx yyy yyy yyy');
+}
+
+do_execsql_test 2.1 {
+  SELECT tokens(x1) FROM x1('xxx');
+} {
+  {xxx xxx.Xxx xxx.XXX} {xxx xxx}
+}
+
+do_execsql_test 2.2 {
+  UPDATE x1_content SET c0 = 'xxx xxX xxx yyy yyy yyy' WHERE id=1;
+}
+
+do_execsql_test 2.3 {
+  SELECT tokens(x1) FROM x1('xxx');
+} {
+  {xxx {} xxx} {xxx xxx}
+}
+
+finish_test
+
@@ -0,0 +1,101 @@
+# 2023 November 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext3
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+  reset_db
+
+  sqlite3_fts5_register_origintext db
+  fts5_aux_test_functions db
+  proc insttoken {cmd iIdx iToken} { 
+    set txt [$cmd xInstToken $iIdx $iToken]
+    string map [list "\0" "."] $txt
+  }
+  sqlite3_fts5_create_function db insttoken insttoken
+  
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE ft USING fts5(
+        x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
+    );
+  }
+  
+  do_execsql_test 1.1 {
+    INSERT INTO ft VALUES('Hello world HELLO WORLD hello');
+  }
+  
+  do_execsql_test 1.2 {
+    SELECT fts5_test_poslist(ft) FROM ft('hello');
+  } {{0.0.0 0.0.2 0.0.4}}
+
+  do_execsql_test 1.3 {
+    SELECT 
+      insttoken(ft, 0, 0),
+      insttoken(ft, 1, 0),
+      insttoken(ft, 2, 0)
+    FROM ft('hello');
+  } {hello.Hello hello.HELLO hello}
+
+  do_execsql_test 1.4 {
+    SELECT 
+      insttoken(ft, 0, 0),
+      insttoken(ft, 1, 0),
+      insttoken(ft, 2, 0)
+    FROM ft('hello') ORDER BY rank;
+  } {hello.Hello hello.HELLO hello}
+
+  do_execsql_test 1.5 {
+    CREATE VIRTUAL TABLE ft2 USING fts5(
+        x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
+    );
+    INSERT INTO ft2(rowid, x) VALUES(1, 'ONE one two three ONE');
+    INSERT INTO ft2(rowid, x) VALUES(2, 'TWO one two three TWO');
+    INSERT INTO ft2(rowid, x) VALUES(3, 'THREE one two three THREE');
+  }
+
+  do_execsql_test 1.6 {
+    SELECT insttoken(ft2, 0, 0), rowid FROM ft2('three') ORDER BY rank;
+  } {three.THREE 3 three 1 three 2}
+
+  do_execsql_test 1.7 {
+    INSERT INTO ft2(rowid, x) VALUES(10, 'aaa bbb BBB');
+    INSERT INTO ft2(rowid, x) VALUES(12, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(13, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(14, 'bbb BBB bbb');
+    INSERT INTO ft2(rowid, x) VALUES(15, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(16, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(17, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(18, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(19, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(20, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(21, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(22, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(23, 'bbb bbb bbb');
+    INSERT INTO ft2(rowid, x) VALUES(24, 'aaa bbb BBB');
+  }
+
+  do_execsql_test 1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24}
+  do_execsql_test 1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24}
+
+}
+
+finish_test
+
@@ -0,0 +1,66 @@
+# 2023 November 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext4
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+sqlite3_fts5_register_origintext db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", tokendata=1
+  );
+}
+
+do_execsql_test 1.1 {
+  BEGIN;
+    INSERT INTO ft SELECT 'the first thing';
+
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<90000
+    )
+    INSERT INTO ft SELECT 'The second thing' FROM s;
+
+    INSERT INTO ft SELECT 'the first thing';
+  COMMIT;
+  INSERT INTO ft(ft) VALUES('optimize');
+}
+
+foreach {tn sql expr} {
+  1 { SELECT rowid FROM ft('the') }       {$mem > 250000}
+  2 { SELECT rowid FROM ft('first') }     {$mem <  50000}
+  3 { SELECT rowid FROM ft('the first') } {$mem <  50000}
+} {
+  db close
+  sqlite3 db test.db
+  sqlite3_fts5_register_origintext db
+
+  execsql $sql
+  do_test 1.2.$tn {
+    set mem [lindex [sqlite3_db_status db CACHE_USED 0] 1]
+    expr $expr
+  } 1
+}
+
+proc b {x} { string map [list "\0" "."] $x }
+db func b b
+# execsql_pp { SELECT segid, b(term), pgno from ft_idx }
+
+finish_test
+
@@ -0,0 +1,273 @@
+# 2023 Dec 04
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests for tables that use both tokendata=1 and contentless_delete=1.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+# Return a random integer between 0 and n-1.
+#
+proc random {n} { expr {abs(int(rand()*$n))} }
+
+# Select an element of the list passed as the only argument at random and
+# return it. 
+#
+proc select_one {list} {
+  set n [llength $list]
+  lindex $list [random $n]
+}
+
+# Given a term that consists entirely of alphabet characters, return all
+# permutations of the term using upper and lower case characters. e.g.
+#
+#    "abc" -> {CBA cBA CbA cbA CBa cBa Cba cba}
+#
+proc casify {term {lRet {{}}}} {
+  if {$term==""} { return $lRet }
+  set t [string range $term 1 end]
+  set f1 [string toupper [string range $term 0 0]]
+  set f2 [string tolower [string range $term 0 0]]
+  set ret [list]
+  foreach x $lRet {
+    lappend ret "$x$f1"
+    lappend ret "$x$f2"
+  }
+  return [casify $t $ret]
+}
+
+proc vocab {} {
+  list abc def ghi jkl mno pqr stu vwx yza
+}
+
+# Return a random 3 letter term.
+#
+proc term {} {
+  if {[info exists ::expanded_vocab]==0} {
+    foreach v [vocab] { lappend ::expanded_vocab {*}[casify $v] }
+  }
+
+  select_one $::expanded_vocab
+}
+
+# Return a document - between 3 and 10 terms.
+#
+proc document {} {
+  set nTerm [expr [random 3] + 7]
+  set doc ""
+  for {set ii 0} {$ii < $nTerm} {incr ii} {
+    lappend doc [term]
+  }
+  set doc
+}
+db func document document
+
+#-------------------------------------------------------------------------
+
+expr srand(6)
+
+set NDOC  200
+set NLOOP 50
+
+sqlite3_fts5_register_origintext db
+
+proc tokens {cmd} { 
+  set ret [list]
+  for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+    set txt [$cmd xInstToken $iTok 0]
+    set txt [string map [list "\0" "."] $txt]
+    lappend ret $txt
+  }
+  set ret
+}
+sqlite3_fts5_create_function db tokens tokens
+
+proc rankfunc {cmd} { 
+  $cmd xRowid
+}
+sqlite3_fts5_create_function db rankfunc rankfunc
+
+proc ctrl_tokens {term args} {
+  set ret [list]
+  set term [string tolower $term]
+  foreach doc $args {
+    foreach a $doc {
+      if {[string tolower $a]==$term} {
+        if {$a==$term} {
+          lappend ret $a
+        } else {
+          lappend ret [string tolower $a].$a
+        }
+      }
+    }
+  }
+  set ret
+}
+db func ctrl_tokens ctrl_tokens
+
+proc do_all_vocab_test {tn} {
+  foreach ::v [concat [vocab] nnn] {
+    set answer [execsql {
+      SELECT id, ctrl_tokens($::v, x) FROM ctrl WHERE x LIKE '%' || $::v || '%'
+    }]
+    do_execsql_test $tn.$::v.1 {
+      SELECT rowid, tokens(ft) FROM ft($::v)
+    } $answer
+    do_execsql_test $tn.$::v.2 {
+      SELECT rowid, tokens(ft) FROM ft($::v) ORDER BY rank
+    } $answer
+  }
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", content=, contentless_delete=1,
+      tokendata=1
+  );
+
+  CREATE TABLE ctrl(id INTEGER PRIMARY KEY, x TEXT);
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
+  INSERT INTO ft(ft, rank) VALUES('rank', 'rankfunc()');
+}
+do_test 1.1 {
+  for {set ii 0} {$ii < $NDOC} {incr ii} {
+    set doc [document]
+    execsql {
+      INSERT INTO ft(rowid, x) VALUES($ii, $doc);
+      INSERT INTO ctrl(id, x) VALUES($ii, $doc);
+    }
+  }
+} {}
+
+#execsql_pp { SELECT * FROM ctrl }
+#execsql_pp { SELECT * FROM ft }
+#fts5_aux_test_functions db
+#execsql_pp { SELECT rowid, tokens(ft), fts5_test_poslist(ft) FROM ft('ghi'); }
+
+do_all_vocab_test 1.2
+
+for {set ii 0} {$ii < $NLOOP} {incr ii} {
+  set lRowid [execsql { SELECT id FROM ctrl WHERE random() % 2 }]
+  foreach r $lRowid {
+    execsql { DELETE FROM ft WHERE rowid = $r }
+    execsql { DELETE FROM ctrl WHERE rowid = $r }
+
+    set doc [document]
+    execsql { INSERT INTO ft(rowid, x) VALUES($r, $doc) }
+    execsql { INSERT INTO ctrl(id, x) VALUES($r, $doc) }
+  }
+  do_all_vocab_test 1.3.$ii
+}
+
+#-------------------------------------------------------------------------
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(
+      x, y, tokenize="origintext unicode61", content=, contentless_delete=1,
+      tokendata=1
+  );
+
+  CREATE TABLE ctrl2(id INTEGER PRIMARY KEY, x TEXT, y TEXT);
+  INSERT INTO ft2(ft2, rank) VALUES('pgsz', 64);
+  INSERT INTO ft2(ft2, rank) VALUES('rank', 'rankfunc()');
+}
+do_test 2.1 {
+  for {set ii 0} {$ii < $NDOC} {incr ii} {
+    set doc1 [document]
+    set doc2 [document]
+    execsql {
+      INSERT INTO ft2(rowid, x, y) VALUES($ii, $doc, $doc2);
+      INSERT INTO ctrl2(id, x, y) VALUES($ii, $doc, $doc2);
+    }
+  }
+} {}
+
+proc do_all_vocab_test2 {tn} {
+  foreach ::v [vocab] {
+    set answer [execsql {
+      SELECT id, ctrl_tokens($::v, x, y) FROM ctrl2
+      WHERE x LIKE '%' || $::v || '%' OR y LIKE '%' || $::v || '%';
+    }]
+    do_execsql_test $tn.$::v.1 {
+      SELECT rowid, tokens(ft2) FROM ft2($::v)
+    } $answer
+    do_execsql_test $tn.$::v.2 {
+      SELECT rowid, tokens(ft2) FROM ft2($::v) ORDER BY rank
+    } $answer
+  }
+}
+
+do_all_vocab_test2 2.2
+
+for {set ii 0} {$ii < $NLOOP} {incr ii} {
+  set lRowid [execsql { SELECT id FROM ctrl2 WHERE random() % 2 }]
+  foreach r $lRowid {
+    execsql { DELETE FROM ft2 WHERE rowid = $r }
+    execsql { DELETE FROM ctrl2 WHERE rowid = $r }
+
+    set doc1 [document]
+    set doc2 [document]
+    execsql { INSERT INTO ft2(rowid, x, y) VALUES($r, $doc, $doc1) }
+    execsql { INSERT INTO ctrl2(id, x, y) VALUES($r, $doc, $doc2) }
+  }
+  do_all_vocab_test 2.3.$ii
+}
+
+#-------------------------------------------------------------------------
+
+unset -nocomplain ::expanded_vocab
+proc vocab {} {
+  list abcde fghij klmno
+}
+
+proc do_all_vocab_test3 {tn} {
+  foreach ::v [concat [vocab] nnn] {
+    set answer [execsql {
+      SELECT rowid, ctrl_tokens($::v, w) FROM ctrl3 WHERE w LIKE '%' || $::v || '%'
+    }]
+    do_execsql_test $tn.$::v.1 {
+      SELECT rowid, tokens(ft3) FROM ft3($::v)
+    } $answer
+    do_execsql_test $tn.$::v.2 {
+      SELECT rowid, tokens(ft3) FROM ft3($::v) ORDER BY rank
+    } $answer
+  }
+}
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft3 USING fts5(
+      w, tokenize="origintext unicode61", content=, contentless_delete=1,
+      tokendata=1
+  );
+  INSERT INTO ft3(ft3, rank) VALUES('rank', 'rankfunc()');
+  CREATE TABLE ctrl3(w);
+}
+
+do_execsql_test 3.1 {
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<2
+  )
+  INSERT INTO ctrl3 SELECT document() FROM s;
+  INSERT INTO ft3(rowid, w) SELECT rowid, w FROM ctrl3;
+}
+
+do_all_vocab_test3 3.2
+
+
+finish_test
+
@@ -343,7 +343,9 @@ do_execsql_test 17.0 {
    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
  COMMIT;
 }
-do_execsql_test 17.1 { SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 }
+do_execsql_test 17.1 { 
+  SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 
+}
 do_execsql_test 17.2 {
  BEGIN;
    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
@@ -280,6 +280,30 @@ do_catchsql_test 5.2 {
  INSERT INTO t1 SELECT randomblob(3000) FROM v1
 } {1 {query aborted}}

+#-------------------------------------------------------------------------
+reset_db
+sqlite3_fts5_may_be_corrupt 1
+
+do_execsql_test 6.0 {
+  BEGIN TRANSACTION;
+    CREATE VIRTUAL TABLE t1 USING fts5(a,b unindexed,c,tokenize="porter ascii",tokendata=1);
+    REPLACE INTO t1_data VALUES(1,X'03090009');
+    REPLACE INTO t1_data VALUES(10,X'000000000103030003010101020101030101');
+    REPLACE INTO t1_data VALUES(137438953473,X'0000002e023061010202010162010203010163010204010167010601020201016801060102030101690106010204040606060808');
+    REPLACE INTO t1_data VALUES(274877906945,X'0000001f013067020802010202010168020803010203010169020804010204040909');
+    REPLACE INTO t1_data VALUES(412316860417,X'0000002e023061030202010162030203010163030204010167030601020201016803060102030101690306010204040606060808');
+  COMMIT;
+}
+
+do_execsql_test 6.1 {
+  CREATE VIRTUAL TABLE t3 USING fts5vocab('t1', 'row');
+}
+
+do_catchsql_test 6.2 {
+  SELECT * FROM t3;
+} {1 {database disk image is malformed}}
+
+sqlite3_fts5_may_be_corrupt 0

 finish_test

@@ -582,6 +582,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
            bNextPage = 1;
          }else{
            iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload);
+            if( nPayload>0x7fffff00 ) nPayload &= 0x3fff;
          }
    
          /* If this is a leaf intkey cell, load the rowid */
@@ -854,7 +854,7 @@ dir.sql := sql
 speedtest1 := ../../speedtest1
 speedtest1.c := ../../test/speedtest1.c
 speedtest1.sql := $(dir.sql)/speedtest1.sql
-speedtest1.cliflags := --size 25 --big-transactions
+speedtest1.cliflags := --size 10 --big-transactions
 $(speedtest1):
 	$(MAKE) -C ../.. speedtest1
 $(speedtest1.sql): $(speedtest1) $(MAKEFILE)
@@ -0,0 +1,86 @@
+<!doctype html>
+<html lang="en-us">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon">
+    <link rel="stylesheet" href="common/testing.css"/>
+    <title>sqlite3-api batch SQL runner for the SAHPool VFS</title>
+  </head>
+  <body>
+    <header id='titlebar'><span>sqlite3-api batch SQL runner for the SAHPool VFS</span></header>
+    <div>
+      <span class='input-wrapper'>
+        <input type='checkbox' class='disable-during-eval' id='cb-reverse-log-order' checked></input>
+        <label for='cb-reverse-log-order' id='lbl-reverse-log-order'>Reverse log order</label>
+      </span>
+    </div>
+    <div id='test-output' class='reverse'></div>
+    <script>
+      (function(){
+        const E = (sel)=>document.querySelector(sel);
+
+        const eOut = E('#test-output');
+        const log2 = function(cssClass,...args){
+          let ln;
+          if(1 || cssClass){
+            ln = document.createElement('div');
+            if(cssClass) ln.classList.add(cssClass);
+            ln.append(document.createTextNode(args.join(' ')));
+          }else{
+            // This doesn't work with the "reverse order" option!
+            ln = document.createTextNode(args.join(' ')+'\n');
+          }
+          eOut.append(ln);
+        };
+        const log = (...args)=>{
+          //console.log(...args);
+          log2('', ...args);
+        };
+        const logErr = function(...args){
+          console.error(...args);
+          log2('error', ...args);
+        };
+        const logWarn = function(...args){
+          console.warn(...args);
+          log2('warning', ...args);
+        };
+
+        const cbReverseLog = E('#cb-reverse-log-order');
+        const lblReverseLog = E('#lbl-reverse-log-order');
+        if(cbReverseLog.checked){
+            lblReverseLog.classList.add('warning');
+            eOut.classList.add('reverse');
+        }
+        cbReverseLog.addEventListener('change', function(){
+            if(this.checked){
+                eOut.classList.add('reverse');
+                lblReverseLog.classList.add('warning');
+            }else{
+                eOut.classList.remove('reverse');
+                lblReverseLog.classList.remove('warning');
+            }
+        }, false);
+
+        const w = new Worker('batch-runner-sahpool.js?sqlite3.dir=jswasm');
+        w.onmessage = function(msg){
+          msg = msg.data;
+          switch(msg.type){
+            case 'stdout': log(...msg.data); break;
+            case 'warn': logWarn(...msg.data); break;
+            case 'error': logErr(...msg.data); break;
+            default:
+              logErr("Unhandled worker message type:",msg);
+              break;
+          }
+        };
+      })();
+    </script>
+    <style>
+      #test-output {
+          white-space: break-spaces;
+          overflow: auto;
+      }
+    </style>
+  </body>
+</html>
@@ -0,0 +1,341 @@
+/*
+  2023-11-30
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  A basic batch SQL runner for the SAHPool VFS. This file must be run in
+  a worker thread. This is not a full-featured app, just a way to get some
+  measurements for batch execution of SQL for the OPFS SAH Pool VFS.
+*/
+'use strict';
+
+const wMsg = function(msgType,...args){
+  postMessage({
+    type: msgType,
+    data: args
+  });
+};
+const toss = function(...args){throw new Error(args.join(' '))};
+const warn = (...args)=>{ wMsg('warn',...args); };
+const error = (...args)=>{ wMsg('error',...args); };
+const log = (...args)=>{ wMsg('stdout',...args); }
+let sqlite3;
+const urlParams = new URL(globalThis.location.href).searchParams;
+const cacheSize = (()=>{
+  if(urlParams.has('cachesize')) return +urlParams.get('cachesize');
+  return 200;
+})();
+
+
+/** Throws if the given sqlite3 result code is not 0. */
+const checkSqliteRc = (dbh,rc)=>{
+  if(rc) toss("Prepare failed:",sqlite3.capi.sqlite3_errmsg(dbh));
+};
+
+const sqlToDrop = [
+  "SELECT type,name FROM sqlite_schema ",
+  "WHERE name NOT LIKE 'sqlite\\_%' escape '\\' ",
+  "AND name NOT LIKE '\\_%' escape '\\'"
+].join('');
+
+const clearDbSqlite = function(db){
+  // This would be SO much easier with the oo1 API, but we specifically want to
+  // inject metrics we can't get via that API, and we cannot reliably (OPFS)
+  // open the same DB twice to clear it using that API, so...
+  const rc = sqlite3.wasm.exports.sqlite3_wasm_db_reset(db.handle);
+  log("reset db rc =",rc,db.id, db.filename);
+};
+
+const App = {
+  db: undefined,
+  cache:Object.create(null),
+  log: log,
+  warn: warn,
+  error: error,
+  metrics: {
+    fileCount: 0,
+    runTimeMs: 0,
+    prepareTimeMs: 0,
+    stepTimeMs: 0,
+    stmtCount: 0,
+    strcpyMs: 0,
+    sqlBytes: 0
+  },
+  fileList: undefined,
+  execSql: async function(name,sql){
+    const db = this.db;
+    const banner = "========================================";
+    this.log(banner,
+             "Running",name,'('+sql.length,'bytes)');
+    const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm;
+    let pStmt = 0, pSqlBegin;
+    const metrics = db.metrics = Object.create(null);
+    metrics.prepTotal = metrics.stepTotal = 0;
+    metrics.stmtCount = 0;
+    metrics.malloc = 0;
+    metrics.strcpy = 0;
+    if(this.gotErr){
+      this.error("Cannot run SQL: error cleanup is pending.");
+      return;
+    }
+    // Run this async so that the UI can be updated for the above header...
+    const endRun = ()=>{
+      metrics.evalSqlEnd = performance.now();
+      metrics.evalTimeTotal = (metrics.evalSqlEnd - metrics.evalSqlStart);
+      this.log("metrics:",JSON.stringify(metrics, undefined, ' '));
+      this.log("prepare() count:",metrics.stmtCount);
+      this.log("Time in prepare_v2():",metrics.prepTotal,"ms",
+               "("+(metrics.prepTotal / metrics.stmtCount),"ms per prepare())");
+      this.log("Time in step():",metrics.stepTotal,"ms",
+               "("+(metrics.stepTotal / metrics.stmtCount),"ms per step())");
+      this.log("Total runtime:",metrics.evalTimeTotal,"ms");
+      this.log("Overhead (time - prep - step):",
+               (metrics.evalTimeTotal - metrics.prepTotal - metrics.stepTotal)+"ms");
+      this.log(banner,"End of",name);
+      this.metrics.prepareTimeMs += metrics.prepTotal;
+      this.metrics.stepTimeMs += metrics.stepTotal;
+      this.metrics.stmtCount += metrics.stmtCount;
+      this.metrics.strcpyMs += metrics.strcpy;
+      this.metrics.sqlBytes += sql.length;
+    };
+
+    const runner = function(resolve, reject){
+      ++this.metrics.fileCount;
+      metrics.evalSqlStart = performance.now();
+      const stack = wasm.scopedAllocPush();
+      try {
+        let t, rc;
+        let sqlByteLen = sql.byteLength;
+        const [ppStmt, pzTail] = wasm.scopedAllocPtr(2);
+        t = performance.now();
+        pSqlBegin = wasm.scopedAlloc( sqlByteLen + 1/*SQL + NUL*/) || toss("alloc(",sqlByteLen,") failed");
+        metrics.malloc = performance.now() - t;
+        metrics.byteLength = sqlByteLen;
+        let pSql = pSqlBegin;
+        const pSqlEnd = pSqlBegin + sqlByteLen;
+        t = performance.now();
+        wasm.heap8().set(sql, pSql);
+        wasm.poke(pSql + sqlByteLen, 0);
+        //log("SQL:",wasm.cstrToJs(pSql));
+        metrics.strcpy = performance.now() - t;
+        let breaker = 0;
+        while(pSql && wasm.peek8(pSql)){
+          wasm.pokePtr(ppStmt, 0);
+          wasm.pokePtr(pzTail, 0);
+          t = performance.now();
+          rc = capi.sqlite3_prepare_v2(
+            db.handle, pSql, sqlByteLen, ppStmt, pzTail
+          );
+          metrics.prepTotal += performance.now() - t;
+          checkSqliteRc(db.handle, rc);
+          pStmt = wasm.peekPtr(ppStmt);
+          pSql = wasm.peekPtr(pzTail);
+          sqlByteLen = pSqlEnd - pSql;
+          if(!pStmt) continue/*empty statement*/;
+          ++metrics.stmtCount;
+          t = performance.now();
+          rc = capi.sqlite3_step(pStmt);
+          capi.sqlite3_finalize(pStmt);
+          pStmt = 0;
+          metrics.stepTotal += performance.now() - t;
+          switch(rc){
+            case capi.SQLITE_ROW:
+            case capi.SQLITE_DONE: break;
+            default: checkSqliteRc(db.handle, rc); toss("Not reached.");
+          }
+        }
+        resolve(this);
+      }catch(e){
+        if(pStmt) capi.sqlite3_finalize(pStmt);
+        this.gotErr = e;
+        reject(e);
+      }finally{
+        capi.sqlite3_exec(db.handle,"rollback;",0,0,0);
+        wasm.scopedAllocPop(stack);
+      }
+    }.bind(this);
+    const p = new Promise(runner);
+    return p.catch(
+      (e)=>this.error("Error via execSql("+name+",...):",e.message)
+    ).finally(()=>{
+      endRun();
+    });
+  },
+
+  /**
+     Loads batch-runner.list and populates the selection list from
+     it. Returns a promise which resolves to nothing in particular
+     when it completes. Only intended to be run once at the start
+     of the app.
+  */
+  loadSqlList: async function(){
+    const infile = 'batch-runner.list';
+    this.log("Loading list of SQL files:", infile);
+    let txt;
+    try{
+      const r = await fetch(infile);
+      if(404 === r.status){
+        toss("Missing file '"+infile+"'.");
+      }
+      if(!r.ok) toss("Loading",infile,"failed:",r.statusText);
+      txt = await r.text();
+    }catch(e){
+      this.error(e.message);
+      throw e;
+    }
+    App.fileList = txt.split(/\n+/).filter(x=>!!x);
+    this.log("Loaded",infile);
+  },
+
+  /** Fetch ./fn and return its contents as a Uint8Array. */
+  fetchFile: async function(fn, cacheIt=false){
+    if(cacheIt && this.cache[fn]) return this.cache[fn];
+    this.log("Fetching",fn,"...");
+    let sql;
+    try {
+      const r = await fetch(fn);
+      if(!r.ok) toss("Fetch failed:",r.statusText);
+      sql = new Uint8Array(await r.arrayBuffer());
+    }catch(e){
+      this.error(e.message);
+      throw e;
+    }
+    this.log("Fetched",sql.length,"bytes from",fn);
+    if(cacheIt) this.cache[fn] = sql;
+    return sql;
+  }/*fetchFile()*/,
+
+  /**
+     Converts this.metrics() to a form which is suitable for easy conversion to
+     CSV. It returns an array of arrays. The first sub-array is the column names.
+     The 2nd and subsequent are the values, one per test file (only the most recent
+     metrics are kept for any given file).
+  */
+  metricsToArrays: function(){
+    const rc = [];
+    Object.keys(this.dbs).sort().forEach((k)=>{
+      const d = this.dbs[k];
+      const m = d.metrics;
+      delete m.evalSqlStart;
+      delete m.evalSqlEnd;
+      const mk = Object.keys(m).sort();
+      if(!rc.length){
+        rc.push(['db', ...mk]);
+      }
+      const row = [k.split('/').pop()/*remove dir prefix from filename*/];
+      rc.push(row);
+      row.push(...mk.map((kk)=>m[kk]));
+    });
+    return rc;
+  },
+
+  metricsToBlob: function(colSeparator='\t'){
+    const ar = [], ma = this.metricsToArrays();
+    if(!ma.length){
+      this.error("Metrics are empty. Run something.");
+      return;
+    }
+    ma.forEach(function(row){
+      ar.push(row.join(colSeparator),'\n');
+    });
+    return new Blob(ar);
+  },
+
+  /**
+     Fetch file fn and eval it as an SQL blob. This is an async
+     operation and returns a Promise which resolves to this
+     object on success.
+  */
+  evalFile: async function(fn){
+    const sql = await this.fetchFile(fn);
+    return this.execSql(fn,sql);
+  }/*evalFile()*/,
+
+  /**
+     Fetches the handle of the db associated with
+     this.e.selImpl.value, opening it if needed.
+  */
+  initDb: function(){
+    const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm;
+    const stack = wasm.scopedAllocPush();
+    let pDb = 0;
+    const d = Object.create(null);
+    d.filename = "/batch.db";
+    try{
+      const oFlags = capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE;
+      const ppDb = wasm.scopedAllocPtr();
+      const rc = capi.sqlite3_open_v2(d.filename, ppDb, oFlags, this.PoolUtil.vfsName);
+      pDb = wasm.peekPtr(ppDb)
+      if(rc) toss("sqlite3_open_v2() failed with code",rc);
+      capi.sqlite3_exec(pDb, "PRAGMA cache_size="+cacheSize, 0, 0, 0);
+      this.log("cache_size =",cacheSize);
+    }catch(e){
+      if(pDb) capi.sqlite3_close_v2(pDb);
+      throw e;
+    }finally{
+      wasm.scopedAllocPop(stack);
+    }
+    d.handle = pDb;
+    this.log("Opened db:",d.filename,'@',d.handle);
+    return d;
+  },
+
+  closeDb: function(){
+    if(this.db.handle){
+      this.sqlite3.capi.sqlite3_close_v2(this.db.handle);
+      this.db.handle = undefined;
+    }
+  },
+
+  run: async function(sqlite3){
+    delete this.run;
+    this.sqlite3 = sqlite3;
+    const capi = sqlite3.capi, wasm = sqlite3.wasm;
+    this.log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid());
+    this.log("WASM heap size =",wasm.heap8().length);
+    let timeStart;
+    sqlite3.installOpfsSAHPoolVfs({
+      clearOnInit: true, initialCapacity: 4,
+      name: 'batch-sahpool',
+      verbosity: 2
+    }).then(PoolUtil=>{
+      App.PoolUtil = PoolUtil;
+      App.db = App.initDb();
+    })
+      .then(async ()=>this.loadSqlList())
+      .then(async ()=>{
+        timeStart = performance.now();
+        for(let i = 0; i < App.fileList.length; ++i){
+          const fn = App.fileList[i];
+          await App.evalFile(fn);
+          if(App.gotErr) throw App.gotErr;
+        }
+      })
+      .then(()=>{
+        App.metrics.runTimeMs = performance.now() - timeStart;
+        App.log("total metrics:",JSON.stringify(App.metrics, undefined, ' '));
+        App.log("Reload the page to run this again.");
+        App.closeDb();
+        App.PoolUtil.removeVfs();
+      })
+      .catch(e=>this.error("ERROR:",e));
+  }/*run()*/
+}/*App*/;
+
+let sqlite3Js = 'sqlite3.js';
+if(urlParams.has('sqlite3.dir')){
+  sqlite3Js = urlParams.get('sqlite3.dir') + '/' + sqlite3Js;
+}
+importScripts(sqlite3Js);
+globalThis.sqlite3InitModule().then(async function(sqlite3_){
+  log("Done initializing. Running batch runner...");
+  sqlite3 = sqlite3_;
+  App.run(sqlite3_);
+});
@@ -72,7 +72,6 @@
    App.logHtml("reset db rc =",rc,db.id, db.filename);
  };

-  
  const E = (s)=>document.querySelector(s);
  const App = {
    e: {
@@ -91,6 +90,15 @@
    db: Object.create(null),
    dbs: Object.create(null),
    cache:{},
+    metrics: {
+      fileCount: 0,
+      runTimeMs: 0,
+      prepareTimeMs: 0,
+      stepTimeMs: 0,
+      stmtCount: 0,
+      strcpyMs: 0,
+      sqlBytes: 0
+    },
    log: console.log.bind(console),
    warn: console.warn.bind(console),
    cls: function(){this.e.output.innerHTML = ''},
@@ -117,7 +125,6 @@
                   "Running",name,'('+sql.length,'bytes) using',db.id);
      const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm;
      let pStmt = 0, pSqlBegin;
-      const stack = wasm.scopedAllocPush();
      const metrics = db.metrics = Object.create(null);
      metrics.prepTotal = metrics.stepTotal = 0;
      metrics.stmtCount = 0;
@@ -142,6 +149,11 @@
        this.logHtml("Overhead (time - prep - step):",
                     (metrics.evalTimeTotal - metrics.prepTotal - metrics.stepTotal)+"ms");
        this.logHtml(banner,"End of",name);
+        this.metrics.prepareTimeMs += metrics.prepTotal;
+        this.metrics.stepTimeMs += metrics.stepTotal;
+        this.metrics.stmtCount += metrics.stmtCount;
+        this.metrics.strcpyMs += metrics.strcpy;
+        this.metrics.sqlBytes += sql.length;
      };

      let runner;
@@ -214,7 +226,9 @@
        }.bind(this);
      }else{/*sqlite3 db...*/
        runner = function(resolve, reject){
+          ++this.metrics.fileCount;
          metrics.evalSqlStart = performance.now();
+          const stack = wasm.scopedAllocPush();
          try {
            let t;
            let sqlByteLen = sql.byteLength;
@@ -269,7 +283,7 @@
      let p;
      if(1){
        p = new Promise(function(res,rej){
-          setTimeout(()=>runner(res, rej), 50)/*give UI a chance to output the "running" banner*/;
+          setTimeout(()=>runner(res, rej), 0)/*give UI a chance to output the "running" banner*/;
        });
      }else{
        p = new Promise(runner);
@@ -401,7 +415,7 @@
      });
      return new Blob(ar);
    },
-    
+
    downloadMetrics: function(){
      const b = this.metricsToBlob();
      if(!b) return;
@@ -576,6 +590,8 @@
        const timeTotal = performance.now() - timeStart;
        who.logHtml("Run-remaining time:",timeTotal,"ms ("+(timeTotal/1000/60)+" minute(s))");
        who.clearStorage();
+        App.metrics.runTimeMs = timeTotal;
+        who.logHtml("Total metrics:",JSON.stringify(App.metrics,undefined,'  '));
      }, false);
    }/*run()*/
  }/*App*/;
@@ -20,7 +20,7 @@
     the main (UI) thread.
  */
  let logHtml;
-  if(self.window === self /* UI thread */){
+  if(globalThis.window === globalThis /* UI thread */){
    console.log("Running demo from main UI thread.");
    logHtml = function(cssClass,...args){
      const ln = document.createElement('div');
@@ -250,7 +250,7 @@
  }/*demo1()*/;

  log("Loading and initializing sqlite3 module...");
-  if(self.window!==self) /*worker thread*/{
+  if(globalThis.window!==globalThis) /*worker thread*/{
    /*
      If sqlite3.js is in a directory other than this script, in order
      to get sqlite3.js to resolve sqlite3.wasm properly, we have to
@@ -262,19 +262,20 @@
      that's not needed.

      URL arguments passed as part of the filename via importScripts()
-      are simply lost, and such scripts see the self.location of
+      are simply lost, and such scripts see the globalThis.location of
      _this_ script.
    */
    let sqlite3Js = 'sqlite3.js';
-    const urlParams = new URL(self.location.href).searchParams;
+    const urlParams = new URL(globalThis.location.href).searchParams;
    if(urlParams.has('sqlite3.dir')){
      sqlite3Js = urlParams.get('sqlite3.dir') + '/' + sqlite3Js;
    }
    importScripts(sqlite3Js);
  }
-  self.sqlite3InitModule({
-    // We can redirect any stdout/stderr from the module
-    // like so...
+  globalThis.sqlite3InitModule({
+    /* We can redirect any stdout/stderr from the module like so, but
+       note that doing so makes use of Emscripten-isms, not
+       well-defined sqlite APIs. */
    print: log,
    printErr: error
  }).then(function(sqlite3){
@@ -46,6 +46,9 @@
          file:// URLs.</li>
        <li>Any OPFS-related pages or tests require:
          <ul>
+            <li>An OPFS-capable browser released after February
+              2023. Some tests will work with Chromium-based browsers
+              going back to around v102.</li>
            <li>That the web server emit the so-called
              <a href='https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cross-Origin-Opener-Policy'>COOP</a>
              and
@@ -53,12 +56,6 @@
              headers. <a href='https://sqlite.org/althttpd'>althttpd</a> requires the
              <code>-enable-sab</code> flag for that.
            </li>
-            <li>A very recent version of a Chromium-based browser
-              (v102 at least, possibly newer). OPFS support in the
-              other major browsers is pending. Development and testing
-              is currently done against a dev-channel release of
-              Chrome (v111 as of 2023-02-10).
-            </li>
          </ul>
        </li>
      </ul>
@@ -31,6 +31,9 @@
          file:// URLs.</li>
        <li>Any OPFS-related pages or tests require:
          <ul>
+            <li>An OPFS-capable browser released after February
+              2023. Some tests will work with Chromium-based browsers
+              going back to around v102.</li>
            <li>That the web server emit the so-called
              <a href='https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cross-Origin-Opener-Policy'>COOP</a>
              and
@@ -38,12 +41,6 @@
              headers. <a href='https://sqlite.org/althttpd'>althttpd</a> requires the
              <code>-enable-sab</code> flag for that.
            </li>
-            <li>A very recent version of a
-              Chromium-based browser (v102 at least, possibly newer). OPFS
-              support in the other major browsers is pending. Development
-              and testing is currently done against a dev-channel release
-              of Chrome (v111 as of 2023-02-10).
-            </li>
          </ul>
        </li>
      </ul>
@@ -350,7 +350,7 @@
                eControls.classList.remove('hidden');
                break;
            case 'stdout': log(msg.data); break;
-            case 'stdout': logErr(msg.data); break;
+            case 'stderr': logErr(msg.data); break;
            case 'run-start':
                eControls.disabled = true;
                log("Running speedtest1 with argv =",msg.data.join(' '));
@@ -7,9 +7,9 @@
  }
  importScripts(speedtestJs);
  /**
-     If this environment contains OPFS, this function initializes it and
-     returns the name of the dir on which OPFS is mounted, else it returns
-     an empty string.
+     If this build includes WASMFS, this function initializes it and
+     returns the name of the dir on which OPFS is mounted, else it
+     returns an empty string.
  */
  const wasmfsDir = function f(wasmUtil){
    if(undefined !== f._) return f._;
@@ -47,6 +47,7 @@
  };
  const log = (...args)=>logMsg('stdout',args);
  const logErr = (...args)=>logMsg('stderr',args);
+  const realSahName = 'opfs-sahpool-speedtest1';

  const runSpeedtest = async function(cliFlagsArray){
    const scope = App.wasm.scopedAllocPush();
@@ -57,7 +58,6 @@
      ];
      App.logBuffer.length = 0;
      const ndxSahPool = argv.indexOf('opfs-sahpool');
-      const realSahName = 'opfs-sahpool-speedtest1';
      if(ndxSahPool>0){
        argv[ndxSahPool] = realSahName;
        log("Updated argv for opfs-sahpool: --vfs",realSahName);
@@ -73,7 +73,7 @@
          clearOnInit: true,
          verbosity: 2
        }).then(PoolUtil=>{
-          log("opfs-sahpool successfully installed as",realSahName);
+          log("opfs-sahpool successfully installed as",PoolUtil.vfsName);
          App.sqlite3.$SAHPoolUtil = PoolUtil;
          //console.log("sqlite3.oo1.OpfsSAHPoolDb =", App.sqlite3.oo1.OpfsSAHPoolDb);
        });
@@ -102,19 +102,6 @@
    }
  };

-  const sahpSanityChecks = function(sqlite3){
-    log("Attempting OpfsSAHPoolDb sanity checks...");
-    const db = new sqlite3.oo1.OpfsSAHPoolDb('opfs-sahpoool.db');
-    const fn = db.filename;
-    db.exec([
-      'create table t(a);',
-      'insert into t(a) values(1),(2),(3);'
-    ]);
-    db.close();
-    sqlite3.wasm.sqlite3_wasm_vfs_unlink(sqlite3_vfs_find("opfs-sahpool"), fn);
-    log("SAH sanity checks done.");
-  };
-
  const EmscriptenModule = {
    print: log,
    printErr: logErr,
@@ -1,12 +1,12 @@
-C Enhance\sjson_set()\sand\sjson_insert()\sso\sthat\sthey\screate\smissing\nsubstructure.
-D 2023-11-30T19:06:27.003
+C Add\sNEVER\sto\stwo\sunreachable\sbranches\sin\sJSON.
+D 2023-12-12T17:13:10.122
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
 F Makefile.in 890caf094636c308bc981032baf8f9208bf307755f9197ae4218a9fbcec2e449
 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6
 F Makefile.msc 59bb36dba001f0b38212be0794fb838f25371008b180929bcf08aa799694c168
-F README.md 963d30019abf0cc06b263cd2824bce022893f3f93a531758f6f04ff2194a16a8
+F README.md 6358805260a03ebead84e168bbf3740ddf3f683b477e478567186aa7afb490d3
 F VERSION 73573d4545343f001bf5dc5461173a7c78c203dd046cabcf99153878cf25d3a6
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
@@ -88,28 +88,28 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
-F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
-F ext/fts5/fts5.h 05501612cc655504c5dce8ba765ab621d50fc478490089beaa0d75e00b23e520
-F ext/fts5/fts5Int.h 78a63cc0795186cde5384816a9403a68c65774b35d952e05b81a1b4b158e07c8
+F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e
+F ext/fts5/fts5.h 9d7867cc63631296462c90bafe57c4881b56e480fa510ffaee6a77049258c714
+F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880e9834
 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
-F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081
-F ext/fts5/fts5_expr.c bd3b81ce669c4104e34ffe66570af1999a317b142c15fccb112de9fb0caa57a6
-F ext/fts5/fts5_hash.c 076058f93327051952a752dc765df1acfe783eb11b419b30652aa1fc1f987902
-F ext/fts5/fts5_index.c 458cbed8a3e17617cbf7e80cdfb7612000b9bb3781f286b345fb9655858658cf
-F ext/fts5/fts5_main.c a07ed863b8bd9e6fefb62db2fd40a3518eb30a5f7dcfda5be915dd2db45efa2f
+F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf
+F ext/fts5/fts5_expr.c b1ec526371b9ffde82341423a5b9753c42cbea629a41b69f26fa377d13b95a8e
+F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
+F ext/fts5/fts5_index.c ed206045ff0f2226d870fa41fba45f738c0cc953ab74ba68477091b9a574ccd3
+F ext/fts5/fts5_main.c b908696c52410e8383019ac0657c8a5cd0c8f60e78edc169e9b3c4b93f24c933
 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
-F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
+F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b
 F ext/fts5/fts5_tokenize.c 83cfcede3898001cab84432a36ce1503e3080cf9b1c682b022ec82e267ea4c13
 F ext/fts5/fts5_unicode2.c eca63dbc797f8ff0572e97caf4631389c0ab900d6364861b915bdd4735973f00
 F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80
-F ext/fts5/fts5_vocab.c aed56169ae5c1aa9b8189c779ffeef04ed516d3c712c06914e6d91a6759f4e4a
+F ext/fts5/fts5_vocab.c 209e0c151e108d5f3621fa24b91e9b02f3750ee6c3f9ccec312df39481b68a09
 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
-F ext/fts5/test/fts5_common.tcl a9de9c2209cc4e7ae3c753e783504e67206c6c1467d08f209cd0c5923d3e8d8b
-F ext/fts5/test/fts5aa.test ba5158eba7d61359becdfca895ef471072c7bf7b20e5e60dcb4d024c8419c926
+F ext/fts5/test/fts5_common.tcl 8b1848ac2baad10e444e4183034a52050b52d20b3796d9d30e78f01ab0d05583
+F ext/fts5/test/fts5aa.test 4db81519863244a3cab35795fe65ab6b592e7970c7409eba098b23ebbfc08d95
 F ext/fts5/test/fts5ab.test bd932720c748383277456b81f91bc00453de2174f9762cd05f95d0495dc50390
 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d
 F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de
@@ -145,7 +145,7 @@ F ext/fts5/test/fts5corrupt.test b6d4034b682bb3387bc44c510c71b3c67d4349e4df13949
 F ext/fts5/test/fts5corrupt2.test 99e7e23a58b4d89eb7167c6de1669cbc595cd3c79ab333e0eb56405473319e77
 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
 F ext/fts5/test/fts5corrupt4.test f4c08e2182a48d8b70975fd869ee5391855c06d8a0ff87b6a2529e7c5a88a1d3
-F ext/fts5/test/fts5corrupt5.test 38a238df26c4de471e1c4b98f8de6c902bc692577a1c08d0ff4f2251f3559644
+F ext/fts5/test/fts5corrupt5.test f9dbed6785c38d891c0b947fd6e973e9c4eb44991e1322f8d7e5281a019276f2
 F ext/fts5/test/fts5corrupt6.test bf8eeae07825b088b9665d9d8e4accbd8dc9bf3cb85b6c64cf6c9e18ccc420a4
 F ext/fts5/test/fts5corrupt7.test 80ad7f683a8bda2404731bb77e8c3dbbb620c1f6cc583cca8239f6accd6338c0
 F ext/fts5/test/fts5delete.test 619295b20dbc1d840b403ee07c878f52378849c3c02e44f2ee143b3e978a0aa7
@@ -170,6 +170,7 @@ F ext/fts5/test/fts5faultD.test e7ed7895abfe6bc98a5e853826f6b74956e7ba7f594f1860
 F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e4710c77eb8ce7075
 F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1
 F ext/fts5/test/fts5faultG.test d2e5a4d9a34e08dcaadcaeafef74d10cbc2abdd11aa2659a18af0294bf2812d3
+F ext/fts5/test/fts5faultH.test 57f53c87ffd59be0265840f2b54a16811f9cb9012db86aad9b41d0d14d85dfe3
 F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b996159f6909dc8079
 F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717
 F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e
@@ -182,7 +183,7 @@ F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fc
 F ext/fts5/test/fts5matchinfo.test 10c9a6f7fe61fb132299c4183c012770b10c4d5c2f2edb6df0b6607f683d737a
 F ext/fts5/test/fts5merge.test e92a8db28b45931e7a9c7b1bbd36101692759d00274df74d83fd29d25d53b3a6
 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2
-F ext/fts5/test/fts5misc.test 5ca82f2a5ee016b0842043155d1382f98a34d0d86b2791165a44d7807f6e0f54
+F ext/fts5/test/fts5misc.test dd97c86c9cbc3e587067e640f6ce88842cfbf5d23bb0e0fbb7f6707623b2d505
 F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581
 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45
 F ext/fts5/test/fts5near.test 211477940142d733ac04fad97cb24095513ab2507073a99c2765c3ddd2ef58bd
@@ -190,6 +191,11 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618
 F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1
 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785
 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca
+F ext/fts5/test/fts5origintext.test d2796fa08ee7aecfabdc0c45bb8a2fb16a00ea8757e63fbc153b718dbe430a39
+F ext/fts5/test/fts5origintext2.test f3b9436de540828d01f0672df855b09ebc0863e126d5b56234701d71dfa73634
+F ext/fts5/test/fts5origintext3.test 0d25933506600452a5ab3873cbb418ed5f2de2446c3672b9997b1ea104b0e7f0
+F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871
+F ext/fts5/test/fts5origintext5.test a037bdf7235a22033c4663837bdb12d9738245464a3ac2f60c71fc40d07ede7d
 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b
 F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a
 F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15
@@ -212,7 +218,7 @@ F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114
 F ext/fts5/test/fts5secure8.test eb3579e9d58b0acad97e8082dee1f99b2d393198f03500b453c2b25761c0c298
 F ext/fts5/test/fts5securefault.test dbca2b6a1c16700017f5051138991b705410889933f2a37c57ae8a23b296b10b
 F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b
-F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0
+F ext/fts5/test/fts5simple2.test 8dd2389ee75e21a1429fe87e5f8c7d9a97ad1470304a8a2d3ba4b8c3c345fecd
 F ext/fts5/test/fts5simple3.test d5c74a9d3ca71bd5dd5cacb7c55b86ea12cdddfc8b1910e3de2995206898380f
 F ext/fts5/test/fts5synonym.test 1651815b8008de170e8e600dcacc17521d765482ea8f074ae82cfa870d8bb7fb
 F ext/fts5/test/fts5synonym2.test 8f891fc49cc1e8daed727051e77e1f42849c784a6a54bef82564761b2cb3e016
@@ -232,7 +238,7 @@ F ext/fts5/test/fts5unindexed.test 9021af86a0fb9fc616f7a69a996db0116e7936d0db638
 F ext/fts5/test/fts5update.test b8affd796e45c94a4d19ad5c26606ea06065a0f162a9562d9f005b5a80ccf0bc
 F ext/fts5/test/fts5version.test d6e5a5897550afeccc2f8531d87404dc1c289ee89385dd4318dbdd75e71d7a67
 F ext/fts5/test/fts5vocab.test 7ed80d9af1ddaaa1637da05e406327b5aac250848bc604c1c1cc667908b87760
-F ext/fts5/test/fts5vocab2.test 681980e92e031c9f3fe8d9c149189e876c108da2fb0fb3a25bd8a9b94bff8f68
+F ext/fts5/test/fts5vocab2.test 1b1f0059f762ffb404213d35dac013e010621f08128589b6ec7bec59d9a710f3
 F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85
 F ext/fts5/tool/fts5txt2db.tcl c0d43c8590656f8240e622b00957b3a0facc49482411a9fdc2870b45c0c82f9f
 F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093
@@ -460,7 +466,7 @@ F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69
 F ext/rbu/sqlite3rbu.c d4ddf8f0e93772556e452a6c2814063cf47efb760a0834391a9d0cd9859fa4b9
 F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304
 F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055
-F ext/recover/dbdata.c fc7147a68422cbbbaa481ee92ae1752cc25f5a24302bece1c70dcb76345bd736
+F ext/recover/dbdata.c b7746c2ec801b453840831311be4b31f8c8f9dd97791060a69bbf12392c78949
 F ext/recover/recover1.test c484d01502239f11b61f23c1cee9f5dd19fa17617f8974e42e74d64639c524cf
 F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a
 F ext/recover/recoverbuild.test c74170e0f7b02456af41838afeb5353fdb985a48cc2331d661bbabbca7c6b8e3
@@ -568,7 +574,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb
 F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c
-F ext/wasm/GNUmakefile 0e362f3fc04eab6628cbe4f1e35f4ab4a200881f6b5f753b27fb45eabeddd9d2
+F ext/wasm/GNUmakefile 57439eec2b8b4d4074e5d861e93aab3f32bb0f44339a2b472c23f4e638b7e8a3
 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576
 F ext/wasm/README.md a8a2962c3aebdf8d2104a9102e336c5554e78fc6072746e5daf9c61514e7d193
 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff
@@ -598,8 +604,10 @@ F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 46c4afa6c50d7369252c104f274ad977a97e91cc
 F ext/wasm/api/sqlite3-wasm.c d0e09eb5ed3743c00294e30019e591c3aa150572ae7ffe8a8994568a7377589f
 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 569d4e859968e65f55dec5fac0b879828a23c8b179162cc7812fec19f844dd21
 F ext/wasm/api/sqlite3-worker1.c-pp.js a541112aa51e16705f13a99bb943c64efe178aa28c86704a955f8fd9afe4ba37
+F ext/wasm/batch-runner-sahpool.html e9a38fdeb36a13eac7b50241dfe7ae066fe3f51f5c0b0151e7baee5fce0d07a7
+F ext/wasm/batch-runner-sahpool.js 54a3ac228e6c4703fe72fb65c897e19156263a51fe9b7e21d2834a45e876aabd
 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8
-F ext/wasm/batch-runner.js 0dad6a02ad796f1003d3b7048947d275c4d6277f63767b8e685c27df8fdac93e
+F ext/wasm/batch-runner.js 05ec254f5dbfe605146d9640b3db17d6ef8c3fbef6aa8396051ca72bb5884e3f
 F ext/wasm/c-pp.c 6d80d8569d85713effe8b0818a3cf51dc779e3f0bf8dc88771b8998552ee25b4
 F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51
 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15
@@ -607,7 +615,7 @@ F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a7
 F ext/wasm/common/whwasmutil.js 4c64594eecc7af4ae64259e95a71ba2a7edf118881aaff0bba86d0c7164e78e4
 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed
 F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508
-F ext/wasm/demo-123.js 38aa8faec4d0ace1c973bc8a7a1533584463ebeecd4c420daa7d9687beeb9cb5
+F ext/wasm/demo-123.js c7b3cca50c55841c381a9ca4f9396e5bbdc6114273d0b10a43e378e32e7be5bf
 F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e
 F ext/wasm/demo-jsstorage.js 44e3ae7ec2483b6c511384c3c290beb6f305c721186bcf5398ca4e00004a06b8
 F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98ab22f5786620b3354ed15f
@@ -621,8 +629,8 @@ F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d695
 F ext/wasm/fiddle/fiddle-worker.js e0153f9af6500805c6f09c0b3cfdb7d857e9d6863dbee9d50d1628fccf5f4b4d
 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715
 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2
-F ext/wasm/index-dist.html 22379774f0ad4edcaaa8cf9c674c82e794cc557719a8addabed74eb8069d412e
-F ext/wasm/index.html 4e7847b909f4ae0da8c829b150b79454050e53b3658431f138636257729cd42b
+F ext/wasm/index-dist.html e91d76e4581185238fd3d42ed86ec600f7023ed3e3a944c5c356f25304bf1263
+F ext/wasm/index.html b31ce41c0da476d5ffcef23069b9d3415b419d65af5779096ebcfbcbade453a9
 F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54
 F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb
 F ext/wasm/module-symbols.html dc476b403369b26a1a23773e13b80f41b9a49f0825e81435fe3600a7cfbbe337
@@ -630,8 +638,8 @@ F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d
 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63
 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d
 F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe
-F ext/wasm/speedtest1-worker.html e33e2064bda572c0c3ebaec7306c35aa758d9d27e245d67e807f8cc4a9351cc5
-F ext/wasm/speedtest1-worker.js 315d26198c46be7c85e26fda15d80ef882424276abde25ffd8b026fb02a35d8c
+F ext/wasm/speedtest1-worker.html 864b65ed78ce24847a348c180e7f267621a02ca027068a1863ec1c90187c1852
+F ext/wasm/speedtest1-worker.js 4d2ea70a3c24e05bdca78025202841f33d298c4fa9541a0070c3228661f89ecd
 F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da
 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
 F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
@@ -670,7 +678,7 @@ F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522
 F src/btree.c f3b09c5414de3a11db73e11e1d66f4c5e53c9e89876ff3b531a887ab656ca303 x
 F src/btree.h 03e3356f5208bcab8eed4e094240fdac4a7f9f5ddf5e91045ce589f67d47c240
 F src/btreeInt.h 3e2589726c4f105e653461814f65857465da68be1fac688de340c43b873f4062
-F src/build.c 189e4517d67f09f0a3e0d8e1faa6e2ef0c2e95f6ac82e33c912cb7efa2a359cc
+F src/build.c 386eadecabe2e99a3783eb802ca01e665f8e0c2af0e0aab28161fd7def219a9d
 F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 23331529e654be40ca97d171cbbffe9b3d4c71cc53b78fe5501230675952da8b
@@ -678,20 +686,20 @@ F src/date.c 3b8d02977d160e128469de38493b4085f7c5cf4073193459909a6af3cf6d7c91
 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782
 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43
 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500
-F src/expr.c e9a491c7f156e5b25641c28af11b735a424e108a21b9f83b6f3e51c99a8141d9
+F src/expr.c 05278def9c186b5875d6903ea26148c7461b9ce0344f0fd7be9a0dfea0a4538a
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00
 F src/func.c 472f6dcfa39cf54f89a6aec76c79c225fb880a6c14469c15d361331662b9bf43
-F src/global.c 29f56a330ed9d1b5cd9b79ac0ca36f97ac3afc730ff8bfa987b0db9e559d684d
+F src/global.c 765a0656d6cbf043cb272ff0ae38f39cc46713539ffe6793258ed3eb4b188b52
 F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220
 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
-F src/json.c 92cc05dcf87cdace6fbf2a76e0cb2b030f04ddb1819e024af4ce8885261b4cef
+F src/json.c cd270629b4b99d065522b45c3d2b5185491b1ab3544843c6f780aa50755d54dc
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36
-F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9
+F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4
 F src/malloc.c f016922435dc7d1f1f5083a03338a3e91f8c67ce2c5bdcfa4cdef62e612f5fcc
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2
@@ -724,16 +732,16 @@ F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00
 F src/pragma.c b5b4cff830575e6188cd56a295a57448d2b9dbc53f0dae58e22b97354cda3781
 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c
-F src/printf.c 9da63b9ae1c14789bcae12840f5d800fd9302500cd2d62733fac77f0041b4750
+F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c d017bad7ba8e778617701a0e986fdeb393d67d6afa84fb28ef4e8b8ad2acf916
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
-F src/select.c 85857bedd2913d888aa571755b48c54cd2e6e7fcb0087e19b226ee0368cfda1e
-F src/shell.c.in 7bb83293775e1a5586d65212997442bc7acc70a2f1b781745da64ec3c2e4ea97
-F src/sqlite.h.in d93a4821d2f792467a60f7dc81268d1bb8634f40c31694ef254cab4f9921f96a
+F src/select.c 9f55c9f3307b9888f62abe709eec245e98ff217bd14c044f93d72810bb7dc445
+F src/shell.c.in 0cd2ef4b3c814dded5436625ab664d9a973cbc4266a1768e3aa4cbf11bb41eca
+F src/sqlite.h.in adcc7dbfeea1e69d6d487139a7e90db8a48fe998f3f5bb0f85c683e6a6fa68ca
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
-F src/sqliteInt.h 6b82eb99a9d2887e873fb29e56befb7c50cf4624df615d23a28f071dc8abf5f6
+F src/sqliteInt.h 615524fff083abf9353aa4a3a8adca2d32b3e61e014e62b1a5335c454a386783
 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6
 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -813,7 +821,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c e5247a3406531b705b44630e9ccf9ca0e5c74955ef19c06fbb146d765c500c20
 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452
 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2
-F src/where.c 1fdc69ce1333e9bd6d7d3df9fa5af1373a3f5bfdd52108d1dbc0ca85a55f777e
+F src/where.c dad4b472b8dc886c0962409e70978568c06b5108bcf3dcd4938a9305a3540440
 F src/whereInt.h 4b38c5889514e3aead3f27d0ee9a26e47c3f150efc59e2a8b4e3bc8835e4d7a1
 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1
 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00
@@ -1221,7 +1229,7 @@ F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c
 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634
 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830
 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2
-F test/fuzzcheck.c 69b8549e112fb815931a8c14c7955a0c407ae91a79356eecb82458384f2cb989
+F test/fuzzcheck.c 442b0b02146bb905c47c4ee1388949e7fd9d27acf22d9535e9abf216e582b956
 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517
 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f
 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba
@@ -1322,7 +1330,7 @@ F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd7
 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa
 F test/json/README.md 63e3e589e1df8fd3cc1588ba1faaff659214003f8b77a15af5c6452b35e30ee2
 F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f
-F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b72447a147a462c
+F test/json/json-q1-b.txt 1e180fe6491efab307e318b22879e3a736ac9a96539bbde7911a13ee5b33abc7
 F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307
 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x
 F test/json101.test 70587d7d35ef9e2126364ba70f0c951f70827cfbd28649d779ff3df7e8f87547
@@ -1331,7 +1339,7 @@ F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a888011
 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1
 F test/json105.test e64a8d73677fbae67886642cd5076e2ef3efe89f8483b87595cf9c030216c9bd
 F test/json501.test ab168a12eb6eb14d479f8c1cdae3ac062fd5a4679f17f976e96f1af518408330
-F test/json502.test 98c38e3c4573841028a1381dfb81d4c3f9b105d39668167da10d055e503f6d0b
+F test/json502.test 3c697e506fc38ccb455b49660b21b6e62e08ede0f2d0c869a7d171e17809093c
 F test/jsonb01.test cace70765b36a36aec9a85a41ea65667d3bbf647d4400ddc3ac76f8fe7d94f90
 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff
 F test/kvtest.c 6e0228409ea7ca0497dad503fbd109badb5e59545d131014b6aaac68b56f484a
@@ -2145,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 04c0d5644372446c924a2e31a26edf51ddc563a1990d170b0ed4739e3e8b239b
-R 4a2861d7ab7860f80e426ce44e32c50f
+P be1864eac4eb75cc30bf98f73092c8608467f4bd956240df6a0cbea9f1e09e85
+R 6e221a94cd5525fe9f8a7e94ea467b60
 U drh
-Z 1fffc27f10fd5ca429d5618946ebb513
+Z 2ea9e875faf360824d045f495f8867aa
 # Remove this line to create a well-formed Fossil manifest.
@@ -1 +1 @@
-cc7a641ab5ae739d31c24f0ad0caeb15a481a63fa8f13720718ea922c25862ff
+c96ebb086feb89341565cc52b970ae7799ce1327fe1ad4fc790f1b0dcaa6e229
@@ -720,7 +720,7 @@ void sqlite3ColumnSetExpr(
 */
 Expr *sqlite3ColumnExpr(Table *pTab, Column *pCol){
  if( pCol->iDflt==0 ) return 0;
-  if( NEVER(!IsOrdinaryTable(pTab)) ) return 0;
+  if( !IsOrdinaryTable(pTab) ) return 0;
  if( NEVER(pTab->u.tab.pDfltList==0) ) return 0;
  if( NEVER(pTab->u.tab.pDfltList->nExpr<pCol->iDflt) ) return 0;
  return pTab->u.tab.pDfltList->a[pCol->iDflt-1].pExpr;
@@ -873,6 +873,9 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){
  if( db->pnBytesFreed==0 && (--pTable->nTabRef)>0 ) return;
  deleteTable(db, pTable);
 }
+void sqlite3DeleteTableGeneric(sqlite3 *db, void *pTable){
+  sqlite3DeleteTable(db, (Table*)pTable);
+}


 /*
@@ -1410,7 +1413,8 @@ void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){
 /*
 ** Clean up the data structures associated with the RETURNING clause.
 */
-static void sqlite3DeleteReturning(sqlite3 *db, Returning *pRet){
+static void sqlite3DeleteReturning(sqlite3 *db, void *pArg){
+  Returning *pRet = (Returning*)pArg;
  Hash *pHash;
  pHash = &(db->aDb[1].pSchema->trigHash);
  sqlite3HashInsert(pHash, pRet->zName, 0);
@@ -1452,8 +1456,7 @@ void sqlite3AddReturning(Parse *pParse, ExprList *pList){
  pParse->u1.pReturning = pRet;
  pRet->pParse = pParse;
  pRet->pReturnEL = pList;
-  sqlite3ParserAddCleanup(pParse,
-     (void(*)(sqlite3*,void*))sqlite3DeleteReturning, pRet);
+  sqlite3ParserAddCleanup(pParse, sqlite3DeleteReturning, pRet);
  testcase( pParse->earlyCleanup );
  if( db->mallocFailed ) return;
  sqlite3_snprintf(sizeof(pRet->zName), pRet->zName,
@@ -5692,4 +5695,7 @@ void sqlite3WithDelete(sqlite3 *db, With *pWith){
    sqlite3DbFree(db, pWith);
  }
 }
+void sqlite3WithDeleteGeneric(sqlite3 *db, void *pWith){
+  sqlite3WithDelete(db, (With*)pWith);
+}
 #endif /* !defined(SQLITE_OMIT_CTE) */
@@ -1223,9 +1223,7 @@ void sqlite3ExprAddFunctionOrderBy(
  assert( ExprUseXList(pExpr) );
  if( pExpr->x.pList==0 || NEVER(pExpr->x.pList->nExpr==0) ){
    /* Ignore ORDER BY on zero-argument aggregates */
-    sqlite3ParserAddCleanup(pParse,
-        (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
-        pOrderBy);
+    sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, pOrderBy);
    return;
  }
  if( IsWindowFunc(pExpr) ){
@@ -1406,6 +1404,9 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){
 void sqlite3ExprDelete(sqlite3 *db, Expr *p){
  if( p ) sqlite3ExprDeleteNN(db, p);
 }
+void sqlite3ExprDeleteGeneric(sqlite3 *db, void *p){
+  if( ALWAYS(p) ) sqlite3ExprDeleteNN(db, (Expr*)p);
+}

 /*
 ** Clear both elements of an OnOrUsing object
@@ -1431,9 +1432,7 @@ void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){
 ** pExpr to the pParse->pConstExpr list with a register number of 0.
 */
 void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){
-  sqlite3ParserAddCleanup(pParse,
-    (void(*)(sqlite3*,void*))sqlite3ExprDelete,
-    pExpr);
+  sqlite3ParserAddCleanup(pParse, sqlite3ExprDeleteGeneric, pExpr);
 }

 /* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the
@@ -2239,6 +2238,9 @@ static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){
 void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){
  if( pList ) exprListDeleteNN(db, pList);
 }
+void sqlite3ExprListDeleteGeneric(sqlite3 *db, void *pList){
+  if( ALWAYS(pList) ) exprListDeleteNN(db, (ExprList*)pList);
+}

 /*
 ** Return the bitwise-OR of all Expr.flags fields in the given
@@ -244,6 +244,9 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = {
   0,                         /* bSmallMalloc */
   1,                         /* bExtraSchemaChecks */
   sizeof(LONGDOUBLE_TYPE)>8, /* bUseLongDouble */
+#ifdef SQLITE_DEBUG
+   0,                         /* bJsonSelfcheck */
+#endif
   0x7ffffffe,                /* mxStrlen */
   0,                         /* neverCorrupt */
   SQLITE_DEFAULT_LOOKASIDE,  /* szLookaside, nLookaside */
@@ -4659,6 +4659,28 @@ int sqlite3_test_control(int op, ...){
      break;
    }
 #endif
+
+    /* sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &onOff);
+    **
+    ** Activate or deactivate validation of JSONB that is generated from
+    ** text.  Off by default, as the validation is slow.  Validation is
+    ** only available if compiled using SQLITE_DEBUG.
+    **
+    ** If onOff is initially 1, then turn it on.  If onOff is initially
+    ** off, turn it off.  If onOff is initially -1, then change onOff
+    ** to be the current setting.
+    */
+    case SQLITE_TESTCTRL_JSON_SELFCHECK: {
+#if defined(SQLITE_DEBUG)
+      int *pOnOff = va_arg(ap, int*);
+      if( *pOnOff<0 ){
+        *pOnOff = sqlite3Config.bJsonSelfcheck;
+      }else{
+        sqlite3Config.bJsonSelfcheck = (u8)((*pOnOff)&0xff);
+      }
+#endif
+      break;
+    }
  }
  va_end(ap);
 #endif /* SQLITE_UNTESTABLE */
@@ -1369,7 +1369,7 @@ void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){


 /*****************************************************************************
-** Reference counted string storage
+** Reference counted string/blob storage
 *****************************************************************************/

 /*
@@ -184,6 +184,9 @@ Select *sqlite3SelectNew(
 void sqlite3SelectDelete(sqlite3 *db, Select *p){
  if( OK_IF_ALWAYS_TRUE(p) ) clearSelect(db, p, 1);
 }
+void sqlite3SelectDeleteGeneric(sqlite3 *db, void *p){
+  if( ALWAYS(p) ) clearSelect(db, (Select*)p, 1);
+}

 /*
 ** Return a pointer to the right-most SELECT statement in a compound.
@@ -3204,9 +3207,7 @@ multi_select_end:
  pDest->iSdst = dest.iSdst;
  pDest->nSdst = dest.nSdst;
  if( pDelete ){
-    sqlite3ParserAddCleanup(pParse,
-        (void(*)(sqlite3*,void*))sqlite3SelectDelete,
-        pDelete);
+    sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pDelete);
  }
  return rc;
 }
@@ -3757,8 +3758,7 @@ static int multiSelectOrderBy(
  /* Make arrangements to free the 2nd and subsequent arms of the compound
  ** after the parse has finished */
  if( pSplit->pPrior ){
-    sqlite3ParserAddCleanup(pParse,
-       (void(*)(sqlite3*,void*))sqlite3SelectDelete, pSplit->pPrior);
+    sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pSplit->pPrior);
  }
  pSplit->pPrior = pPrior;
  pPrior->pNext = pSplit;
@@ -4579,9 +4579,7 @@ static int flattenSubquery(
    Table *pTabToDel = pSubitem->pTab;
    if( pTabToDel->nTabRef==1 ){
      Parse *pToplevel = sqlite3ParseToplevel(pParse);
-      sqlite3ParserAddCleanup(pToplevel,
-         (void(*)(sqlite3*,void*))sqlite3DeleteTable,
-         pTabToDel);
+      sqlite3ParserAddCleanup(pToplevel, sqlite3DeleteTableGeneric, pTabToDel);
      testcase( pToplevel->earlyCleanup );
    }else{
      pTabToDel->nTabRef--;
@@ -5628,8 +5626,7 @@ static struct Cte *searchWith(
 With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){
  if( pWith ){
    if( bFree ){
-      pWith = (With*)sqlite3ParserAddCleanup(pParse,
-                      (void(*)(sqlite3*,void*))sqlite3WithDelete,
+      pWith = (With*)sqlite3ParserAddCleanup(pParse, sqlite3WithDeleteGeneric,
                      pWith);
      if( pWith==0 ) return 0;
    }
@@ -7037,7 +7034,8 @@ static SrcItem *isSelfJoinView(
 /*
 ** Deallocate a single AggInfo object
 */
-static void agginfoFree(sqlite3 *db, AggInfo *p){
+static void agginfoFree(sqlite3 *db, void *pArg){
+  AggInfo *p = (AggInfo*)pArg;
  sqlite3DbFree(db, p->aCol);
  sqlite3DbFree(db, p->aFunc);
  sqlite3DbFreeNN(db, p);
@@ -7291,9 +7289,8 @@ int sqlite3Select(
        sqlite3TreeViewExprList(0, p->pOrderBy, 0, "ORDERBY");
      }
 #endif   
-      sqlite3ParserAddCleanup(pParse,
-        (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
-        p->pOrderBy);
+      sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric,
+                              p->pOrderBy);
      testcase( pParse->earlyCleanup );
      p->pOrderBy = 0;
    }
@@ -7485,9 +7482,8 @@ int sqlite3Select(
    ){
      TREETRACE(0x800,pParse,p,
                ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1));
-      sqlite3ParserAddCleanup(pParse,
-         (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
-         pSub->pOrderBy);
+      sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric,
+                              pSub->pOrderBy);
      pSub->pOrderBy = 0;
    }

@@ -8016,8 +8012,7 @@ int sqlite3Select(
    */
    pAggInfo = sqlite3DbMallocZero(db, sizeof(*pAggInfo) );
    if( pAggInfo ){
-      sqlite3ParserAddCleanup(pParse,
-          (void(*)(sqlite3*,void*))agginfoFree, pAggInfo);
+      sqlite3ParserAddCleanup(pParse, agginfoFree, pAggInfo);
      testcase( pParse->earlyCleanup );
    }
    if( db->mallocFailed ){
@@ -10756,6 +10756,7 @@ static int do_meta_command(char *zLine, ShellState *p){
    {"fk_no_action",       SQLITE_TESTCTRL_FK_NO_ACTION, 0, "BOOLEAN"       },
    {"imposter",         SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"},
    {"internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,""          },
+    {"json_selfcheck",     SQLITE_TESTCTRL_JSON_SELFCHECK ,0,"BOOLEAN"      },
    {"localtime_fault",    SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN"      },
    {"never_corrupt",      SQLITE_TESTCTRL_NEVER_CORRUPT,1, "BOOLEAN"       },
    {"optimizations",      SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK"   },
@@ -10974,6 +10975,16 @@ static int do_meta_command(char *zLine, ShellState *p){
            isOk = 3;
          }
          break;
+        case SQLITE_TESTCTRL_JSON_SELFCHECK:
+          if( nArg==2 ){
+            rc2 = -1;
+            isOk = 1;
+          }else{
+            rc2 = booleanValue(azArg[2]);
+            isOk = 3;
+          }
+          sqlite3_test_control(testctrl, &rc2);
+          break;
      }
    }
    if( isOk==0 && iCtrl>=0 ){
@@ -12061,10 +12072,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
    }else if( cli_strcmp(z,"-init")==0 ){
      zInitFile = cmdline_option_value(argc, argv, ++i);
    }else if( cli_strcmp(z,"-interactive")==0 ){
-      /* Need to check for interactive override here to so that it can
-      ** affect console setup (for Windows only) and testing thereof.
-      */
-      stdin_is_interactive = 1;
    }else if( cli_strcmp(z,"-batch")==0 ){
      /* Need to check for batch mode here to so we can avoid printing
      ** informational messages (like from process_sqliterc) before
@@ -12338,7 +12345,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
            sqlite3_libversion(), sqlite3_sourceid(), 8*(int)sizeof(char*));
      return 0;
    }else if( cli_strcmp(z,"-interactive")==0 ){
-      /* already handled */
+      /* Need to check for interactive override here to so that it can
+      ** affect console setup (for Windows only) and testing thereof.
+      */
+      stdin_is_interactive = 1;
    }else if( cli_strcmp(z,"-batch")==0 ){
      /* already handled */
    }else if( cli_strcmp(z,"-utf8")==0 ){
@@ -8300,6 +8300,7 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_ASSERT                  12
 #define SQLITE_TESTCTRL_ALWAYS                  13
 #define SQLITE_TESTCTRL_RESERVE                 14  /* NOT USED */
+#define SQLITE_TESTCTRL_JSON_SELFCHECK          14
 #define SQLITE_TESTCTRL_OPTIMIZATIONS           15
 #define SQLITE_TESTCTRL_ISKEYWORD               16  /* NOT USED */
 #define SQLITE_TESTCTRL_SCRATCHMALLOC           17  /* NOT USED */
@@ -4116,6 +4116,9 @@ struct sqlite3_str {
 **
 **   3.  Make a (read-only) copy of a read-only RCStr string using
 **       sqlite3RCStrRef().
+**
+** "String" is in the name, but an RCStr object can also be used to hold
+** binary data.
 */
 struct RCStr {
  u64 nRCRef;            /* Number of references */
@@ -4174,6 +4177,9 @@ struct Sqlite3Config {
  u8 bSmallMalloc;                  /* Avoid large memory allocations if true */
  u8 bExtraSchemaChecks;            /* Verify type,name,tbl_name in schema */
  u8 bUseLongDouble;                /* Make use of long double */
+#ifdef SQLITE_DEBUG
+  u8 bJsonSelfcheck;                /* Double-check JSON parsing */
+#endif
  int mxStrlen;                     /* Maximum string length */
  int neverCorrupt;                 /* Database is always well-formed */
  int szLookaside;                  /* Default lookaside buffer size */
@@ -4800,6 +4806,7 @@ void sqlite3ExprOrderByAggregateError(Parse*,Expr*);
 void sqlite3ExprFunctionUsable(Parse*,const Expr*,const FuncDef*);
 void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32);
 void sqlite3ExprDelete(sqlite3*, Expr*);
+void sqlite3ExprDeleteGeneric(sqlite3*,void*);
 void sqlite3ExprDeferredDelete(Parse*, Expr*);
 void sqlite3ExprUnmapAndDelete(Parse*, Expr*);
 ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*);
@@ -4809,6 +4816,7 @@ void sqlite3ExprListSetSortOrder(ExprList*,int,int);
 void sqlite3ExprListSetName(Parse*,ExprList*,const Token*,int);
 void sqlite3ExprListSetSpan(Parse*,ExprList*,const char*,const char*);
 void sqlite3ExprListDelete(sqlite3*, ExprList*);
+void sqlite3ExprListDeleteGeneric(sqlite3*,void*);
 u32 sqlite3ExprListFlags(const ExprList*);
 int sqlite3IndexHasDuplicateRootPage(Index*);
 int sqlite3Init(sqlite3*, char**);
@@ -4899,6 +4907,7 @@ void sqlite3CreateView(Parse*,Token*,Token*,Token*,ExprList*,Select*,int,int);
 void sqlite3DropTable(Parse*, SrcList*, int, int);
 void sqlite3CodeDropTable(Parse*, Table*, int, int);
 void sqlite3DeleteTable(sqlite3*, Table*);
+void sqlite3DeleteTableGeneric(sqlite3*, void*);
 void sqlite3FreeIndex(sqlite3*, Index*);
 #ifndef SQLITE_OMIT_AUTOINCREMENT
  void sqlite3AutoincrementBegin(Parse *pParse);
@@ -4935,6 +4944,7 @@ int sqlite3Select(Parse*, Select*, SelectDest*);
 Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*,
                         Expr*,ExprList*,u32,Expr*);
 void sqlite3SelectDelete(sqlite3*, Select*);
+void sqlite3SelectDeleteGeneric(sqlite3*,void*);
 Table *sqlite3SrcListLookup(Parse*, SrcList*);
 int sqlite3IsReadOnly(Parse*, Table*, Trigger*);
 void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
@@ -5507,6 +5517,7 @@ const char *sqlite3JournalModename(int);
  void sqlite3CteDelete(sqlite3*,Cte*);
  With *sqlite3WithAdd(Parse*,With*,Cte*);
  void sqlite3WithDelete(sqlite3*,With*);
+  void sqlite3WithDeleteGeneric(sqlite3*,void*);
  With *sqlite3WithPush(Parse*, With*, u8);
 #else
 # define sqlite3CteNew(P,T,E,S)   ((void*)0)
@@ -2020,7 +2020,8 @@ static int whereRangeScanEst(
          ** sample, then assume they are 4x more selective.  This brings
          ** the estimated selectivity more in line with what it would be
          ** if estimated without the use of STAT4 tables. */
-          if( iLwrIdx==iUprIdx ) nNew -= 20;  assert( 20==sqlite3LogEst(4) );
+          if( iLwrIdx==iUprIdx ){ nNew -= 20; }
+          assert( 20==sqlite3LogEst(4) );
        }else{
          nNew = 10;        assert( 10==sqlite3LogEst(2) );
        }
@@ -1267,6 +1267,8 @@ int runCombinedDbSqlInput(
  }
  sqlite3_limit(cx.db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH, 100);
  sqlite3_hard_heap_limit64(heapLimit);
+  rc = 1;
+  sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &rc);

  if( nDb>=20 && aDb[18]==2 && aDb[19]==2 ){
    aDb[18] = aDb[19] = 1;
@@ -1,6 +1,6 @@
 .mode qbox
 .timer on
-.param set $label '$.q87'
+.param set $label 'q87'
 SELECT rowid, x->>$label FROM data1 WHERE x->>$label IS NOT NULL;

 CREATE TEMP TABLE t2(x JSON TEXT);
@@ -14,8 +14,7 @@ WITH RECURSIVE
  ),
  c2(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c2 WHERE n<5)
 INSERT INTO t2(x)
-  SELECT jsonb(json_object('a',n,'b',n*2,'c',y,'d',3,'e',5,'f',6))
-    FROM array1, c2;
+  SELECT jsonb_object('a',n,'b',n*2,'c',y,'d',3,'e',5,'f',6) FROM array1, c2;
 CREATE INDEX t2x1 ON t2(x->>'a');
 CREATE INDEX t2x2 ON t2(x->>'b');
 CREATE INDEX t2x3 ON t2(x->>'e');
@@ -36,5 +36,30 @@ do_catchsql_test 2.3 {
  SELECT '{a:null,{"h":[1,[1,2,3]],"j":"abc"}:true}'->'$h[#-1]';
 } {1 {malformed JSON}}

+# Verify that escaped label names are compared correctly.
+# 
+do_execsql_test 3.1 {
+  SELECT '{"a\x62c":123}' ->> 'abc';
+} 123
+do_execsql_test 3.2 {
+  SELECT '{"abc":123}' ->> 'a\x62c';
+} 123
+
+db null null
+do_execsql_test 3.3 {
+  DROP TABLE t1;
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES(json_insert('{}','$.a\',111,'$."b\\"',222));
+  INSERT INTO t1 VALUES(jsonb_insert('{}','$.a\',111,'$."b\\"',222));
+  SELECT x->'$.a\', x->'$.a\\', x->'$."a\\"', x->'$."b\\"' FROM t1;
+} {111 null 111 222 111 null 111 222}
+
+do_execsql_test 3.4 {
+  SELECT json_patch('{"a\x62c":123}','{"ab\x63":456}') ->> 'abc';
+} 456
+
+do_execsql_test 4.1 {
+  SELECT * FROM json_tree('{"\u0017":1}','$."\x17"');
+} {{\x17} 1 integer 1 1 null {$."\x17"} {$}}

 finish_test