Show the creation of IN-operator Bloom filters in the EXPLAIN QUERY PLAN output.

FossilOrigin-Name: c10a1b99d47a4c93bdb16e646f6a21add570d6dbb34fd91a32f8abacdb28712b
Add a new sqlite3FaultSim() call to OP_NotFound to use for testing purposes.
2024-07-03 20:10:42 +00:00 · 2024-07-03 18:56:56 +00:00 · 2024-07-03 17:51:48 +00:00 · 2024-07-02 13:54:46 +00:00 · 2024-07-02 12:16:29 +00:00 · 2024-07-02 11:30:10 +00:00
175 changed files with 3223 additions and 1009 deletions
@@ -454,6 +454,7 @@ TESTSRC += \
  $(TOP)/ext/misc/remember.c \
  $(TOP)/ext/misc/series.c \
  $(TOP)/ext/misc/spellfix.c \
+  $(TOP)/ext/misc/stmtrand.c \
  $(TOP)/ext/misc/totype.c \
  $(TOP)/ext/misc/unionvtab.c \
  $(TOP)/ext/misc/wholenumber.c \
@@ -1596,6 +1596,7 @@ TESTEXT = \
  $(TOP)\ext\misc\remember.c \
  $(TOP)\ext\misc\series.c \
  $(TOP)\ext\misc\spellfix.c \
+  $(TOP)\ext\misc\stmtrand.c \
  $(TOP)\ext\misc\totype.c \
  $(TOP)\ext\misc\unionvtab.c \
  $(TOP)\ext\misc\wholenumber.c \
@@ -1 +1 @@
-3.46.0
+3.47.0
@@ -19,7 +19,7 @@ dnl	to configure the system for the local environment.
 # so that we create the export library with the dll.
 #-----------------------------------------------------------------------

-AC_INIT([sqlite],[3.46.0])
+AC_INIT([sqlite],[3.47.0])

 #--------------------------------------------------------------------
 # Call TEA_INIT as the first TEA_ macro to set up initial vars.
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.46.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.47.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.46.0'
-PACKAGE_STRING='sqlite 3.46.0'
+PACKAGE_VERSION='3.47.0'
+PACKAGE_STRING='sqlite 3.47.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''

@@ -1472,7 +1472,7 @@ if test "$ac_init_help" = "long"; then
  # Omit some internal or obsolete options to make the list less imposing.
  # This message is too long to be a string in the A/UX 3.1 sh.
  cat <<_ACEOF
-\`configure' configures sqlite 3.46.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.47.0 to adapt to many kinds of systems.

 Usage: $0 [OPTION]... [VAR=VALUE]...

@@ -1537,7 +1537,7 @@ fi

 if test -n "$ac_init_help"; then
  case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.46.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.47.0:";;
   esac
  cat <<\_ACEOF

@@ -1668,7 +1668,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
  cat <<\_ACEOF
-sqlite configure 3.46.0
+sqlite configure 3.47.0
 generated by GNU Autoconf 2.69

 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2087,7 +2087,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.

-It was created by sqlite $as_me 3.46.0, which was
+It was created by sqlite $as_me 3.47.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  $ $0 $@
@@ -12481,7 +12481,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.46.0, which was
+This file was extended by sqlite $as_me 3.47.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  CONFIG_FILES    = $CONFIG_FILES
@@ -12547,7 +12547,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.46.0
+sqlite config.status 3.47.0
 configured by $0, generated by GNU Autoconf 2.69,
  with options \\"\$ac_cs_config\\"

@@ -59,8 +59,16 @@ canonical source on a new Windows 11 PC, as of 2023-11-01:
      <li>  `nmake /f makefile.msc sqlite3.c`
      <li>  `nmake /f makefile.msc devtest`
      <li>  `nmake /f makefile.msc releasetest`
+      <li>  `nmake /f makefile.msc sqlite3.exe`
      </ul>

+  7.  For a debugging build of the CLI, where the ".treetrace" and ".wheretrace"
+      commands work, add the DEBUG=3 argument to nmake.  Like this:
+      <ul>
+      <li> `nmake /f makefile.msc DEBUG=3 clean sqlite3.exe`
+      </ul>
+   
+
 ## 32-bit Builds

 Doing a 32-bit build is just like doing a 64-bit build with the
@@ -238,6 +238,10 @@ struct Fts5PhraseIter {
 **   (i.e. if it is a contentless table), then this API always iterates
 **   through an empty set (all calls to xPhraseFirst() set iCol to -1).
 **
+**   In all cases, matches are visited in (column ASC, offset ASC) order.
+**   i.e. all those in column 0, sorted by offset, followed by those in 
+**   column 1, etc.
+**
 ** xPhraseNext()
 **   See xPhraseFirst above.
 **
@@ -142,6 +142,15 @@ struct Fts5Colset {
 */

 typedef struct Fts5Config Fts5Config;
+typedef struct Fts5TokenizerConfig Fts5TokenizerConfig;
+
+struct Fts5TokenizerConfig {
+  Fts5Tokenizer *pTok;
+  fts5_tokenizer *pTokApi;
+  const char **azArg;
+  int nArg;
+  int ePattern;                   /* FTS_PATTERN_XXX constant */
+};

 /*
 ** An instance of the following structure encodes all information that can
@@ -184,6 +193,7 @@ typedef struct Fts5Config Fts5Config;
 */
 struct Fts5Config {
  sqlite3 *db;                    /* Database handle */
+  Fts5Global *pGlobal;            /* Global fts5 object for handle db */
  char *zDb;                      /* Database holding FTS index (e.g. "main") */
  char *zName;                    /* Name of FTS index */
  int nCol;                       /* Number of columns */
@@ -199,10 +209,8 @@ struct Fts5Config {
  int bTokendata;                 /* "tokendata=" option value (dflt==0) */
  int eDetail;                    /* FTS5_DETAIL_XXX value */
  char *zContentExprlist;
-  Fts5Tokenizer *pTok;
-  fts5_tokenizer *pTokApi;
+  Fts5TokenizerConfig t;
  int bLock;                      /* True when table is preparing statement */
-  int ePattern;                   /* FTS_PATTERN_XXX constant */

  /* Values loaded from the %_config table */
  int iVersion;                   /* fts5 file format 'version' */
@@ -597,13 +605,7 @@ struct Fts5Table {
  Fts5Index *pIndex;              /* Full-text index */
 };

-int sqlite3Fts5GetTokenizer(
-  Fts5Global*, 
-  const char **azArg,
-  int nArg,
-  Fts5Config*,
-  char **pzErr
-);
+int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig);

 Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);

@@ -866,6 +868,7 @@ int sqlite3Fts5TokenizerPattern(
    int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
    Fts5Tokenizer *pTok
 );
+int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig*);
 /*
 ** End of interface to code in fts5_tokenizer.c.
 **************************************************************************/
@@ -234,7 +234,6 @@ static int fts5ConfigSetEnum(
 ** eventually free any such error message using sqlite3_free().
 */
 static int fts5ConfigParseSpecial(
-  Fts5Global *pGlobal,
  Fts5Config *pConfig,            /* Configuration object to update */
  const char *zCmd,               /* Special command to parse */
  const char *zArg,               /* Argument to parse */
@@ -298,12 +297,11 @@ static int fts5ConfigParseSpecial(
  if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
    const char *p = (const char*)zArg;
    sqlite3_int64 nArg = strlen(zArg) + 1;
-    char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
-    char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
-    char *pSpace = pDel;
+    char **azArg = sqlite3Fts5MallocZero(&rc, (sizeof(char*) + 2) * nArg);

-    if( azArg && pSpace ){
-      if( pConfig->pTok ){
+    if( azArg ){
+      char *pSpace = (char*)&azArg[nArg];
+      if( pConfig->t.azArg ){
        *pzErr = sqlite3_mprintf("multiple tokenize=... directives");
        rc = SQLITE_ERROR;
      }else{
@@ -326,16 +324,14 @@ static int fts5ConfigParseSpecial(
          *pzErr = sqlite3_mprintf("parse error in tokenize directive");
          rc = SQLITE_ERROR;
        }else{
-          rc = sqlite3Fts5GetTokenizer(pGlobal, 
-              (const char**)azArg, (int)nArg, pConfig,
-              pzErr
-          );
+          pConfig->t.azArg = (const char**)azArg;
+          pConfig->t.nArg = nArg;
+          azArg = 0;
        }
      }
    }
-
    sqlite3_free(azArg);
-    sqlite3_free(pDel);
+
    return rc;
  }

@@ -412,16 +408,6 @@ static int fts5ConfigParseSpecial(
  return SQLITE_ERROR;
 }

-/*
-** Allocate an instance of the default tokenizer ("simple") at 
-** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
-** code if an error occurs.
-*/
-static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
-  assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
-  return sqlite3Fts5GetTokenizer(pGlobal, 0, 0, pConfig, 0);
-}
-
 /*
 ** Gobble up the first bareword or quoted word from the input buffer zIn.
 ** Return a pointer to the character immediately following the last in
@@ -554,6 +540,7 @@ int sqlite3Fts5ConfigParse(
  *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
  if( pRet==0 ) return SQLITE_NOMEM;
  memset(pRet, 0, sizeof(Fts5Config));
+  pRet->pGlobal = pGlobal;
  pRet->db = db;
  pRet->iCookie = -1;

@@ -602,7 +589,7 @@ int sqlite3Fts5ConfigParse(
        rc = SQLITE_ERROR;
      }else{
        if( bOption ){
-          rc = fts5ConfigParseSpecial(pGlobal, pRet, 
+          rc = fts5ConfigParseSpecial(pRet, 
            ALWAYS(zOne)?zOne:"",
            zTwo?zTwo:"",
            pzErr
@@ -640,13 +627,6 @@ int sqlite3Fts5ConfigParse(
    rc = SQLITE_ERROR;
  }

-  /* If a tokenizer= option was successfully parsed, the tokenizer has
-  ** already been allocated. Otherwise, allocate an instance of the default
-  ** tokenizer (unicode61) now.  */
-  if( rc==SQLITE_OK && pRet->pTok==0 ){
-    rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
-  }
-
  /* If no zContent option was specified, fill in the default values. */
  if( rc==SQLITE_OK && pRet->zContent==0 ){
    const char *zTail = 0;
@@ -688,9 +668,10 @@ int sqlite3Fts5ConfigParse(
 void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
  if( pConfig ){
    int i;
-    if( pConfig->pTok ){
-      pConfig->pTokApi->xDelete(pConfig->pTok);
+    if( pConfig->t.pTok ){
+      pConfig->t.pTokApi->xDelete(pConfig->t.pTok);
    }
+    sqlite3_free((char*)pConfig->t.azArg);
    sqlite3_free(pConfig->zDb);
    sqlite3_free(pConfig->zName);
    for(i=0; i<pConfig->nCol; i++){
@@ -765,10 +746,18 @@ int sqlite3Fts5Tokenize(
  void *pCtx,                     /* Context passed to xToken() */
  int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
 ){
-  if( pText==0 ) return SQLITE_OK;
-  return pConfig->pTokApi->xTokenize(
-      pConfig->pTok, pCtx, flags, pText, nText, xToken
-  );
+  int rc = SQLITE_OK;
+  if( pText ){
+    if( pConfig->t.pTok==0 ){
+      rc = sqlite3Fts5LoadTokenizer(pConfig);
+    }
+    if( rc==SQLITE_OK ){
+      rc = pConfig->t.pTokApi->xTokenize(
+          pConfig->t.pTok, pCtx, flags, pText, nText, xToken
+      );
+    }
+  }
+  return rc;
 }

 /*
@@ -324,7 +324,11 @@ int sqlite3Fts5ExprNew(
  }

  sqlite3_free(sParse.apPhrase);
-  *pzErr = sParse.zErr;
+  if( 0==*pzErr ){
+    *pzErr = sParse.zErr;
+  }else{
+    sqlite3_free(sParse.zErr);
+  }
  return sParse.rc;
 }

@@ -1871,6 +1875,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
    }else if( sCtx.pPhrase->nTerm ){
      sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix;
    }
+    assert( pParse->apPhrase!=0 );
    pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
  }

@@ -1890,7 +1895,7 @@ int sqlite3Fts5ExprClonePhrase(
  Fts5ExprPhrase *pOrig = 0;      /* The phrase extracted from pExpr */
  Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
  TokenCtx sCtx = {0,0,0};        /* Context object for fts5ParseTokenize */
-  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
+  if( !pExpr || iPhrase<0 || iPhrase>=pExpr->nPhrase ){
    rc = SQLITE_RANGE;
  }else{
    pOrig = pExpr->apExprPhrase[iPhrase];
@@ -2465,6 +2470,8 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
        );

    if( pRight->eType==FTS5_EOF ){
+      assert( pParse->apPhrase!=0 );
+      assert( pParse->nPhrase>0 );
      assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] );
      sqlite3Fts5ParseNodeFree(pRight);
      pRet = pLeft;
@@ -377,8 +377,12 @@ static int fts5InitVtab(
    assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 );
  }
  if( rc==SQLITE_OK ){
+    pConfig->pzErrmsg = pzErr;
    pTab->p.pConfig = pConfig;
    pTab->pGlobal = pGlobal;
+    if( bCreate || sqlite3Fts5TokenizerPreload(&pConfig->t) ){
+      rc = sqlite3Fts5LoadTokenizer(pConfig);
+    }
  }

  /* Open the index sub-system */
@@ -400,11 +404,8 @@ static int fts5InitVtab(

  /* Load the initial configuration */
  if( rc==SQLITE_OK ){
-    assert( pConfig->pzErrmsg==0 );
-    pConfig->pzErrmsg = pzErr;
    rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
    sqlite3Fts5IndexRollback(pTab->p.pIndex);
-    pConfig->pzErrmsg = 0;
  }

  if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
@@ -414,6 +415,7 @@ static int fts5InitVtab(
    rc = sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS);
  }

+  if( pConfig ) pConfig->pzErrmsg = 0;
  if( rc!=SQLITE_OK ){
    fts5FreeVtab(pTab);
    pTab = 0;
@@ -481,10 +483,10 @@ static int fts5UsePatternMatch(
 ){
  assert( FTS5_PATTERN_GLOB==SQLITE_INDEX_CONSTRAINT_GLOB );
  assert( FTS5_PATTERN_LIKE==SQLITE_INDEX_CONSTRAINT_LIKE );
-  if( pConfig->ePattern==FTS5_PATTERN_GLOB && p->op==FTS5_PATTERN_GLOB ){
+  if( pConfig->t.ePattern==FTS5_PATTERN_GLOB && p->op==FTS5_PATTERN_GLOB ){
    return 1;
  }
-  if( pConfig->ePattern==FTS5_PATTERN_LIKE 
+  if( pConfig->t.ePattern==FTS5_PATTERN_LIKE 
   && (p->op==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB)
  ){
    return 1;
@@ -531,10 +533,10 @@ static int fts5UsePatternMatch(
 ** This function ensures that there is at most one "r" or "=". And that if
 ** there exists an "=" then there is no "<" or ">".
 **
-** Costs are assigned as follows:
+** If an unusable MATCH operator is present in the WHERE clause, then
+** SQLITE_CONSTRAINT is returned.
 **
-**  a) If an unusable MATCH operator is present in the WHERE clause, the
-**     cost is unconditionally set to 1e50 (a really big number).
+** Costs are assigned as follows:
 **
 **  a) If a MATCH operator is present, the cost depends on the other
 **     constraints also present. As follows:
@@ -567,7 +569,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
  int bSeenEq = 0;
  int bSeenGt = 0;
  int bSeenLt = 0;
-  int bSeenMatch = 0;
+  int nSeenMatch = 0;
  int bSeenRank = 0;


@@ -598,18 +600,15 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
      /* A MATCH operator or equivalent */
      if( p->usable==0 || iCol<0 ){
        /* As there exists an unusable MATCH constraint this is an 
-        ** unusable plan. Set a prohibitively high cost. */
-        pInfo->estimatedCost = 1e50;
-        assert( iIdxStr < pInfo->nConstraint*6 + 1 );
-        idxStr[iIdxStr] = 0;
-        return SQLITE_OK;
+        ** unusable plan. Return SQLITE_CONSTRAINT. */
+        return SQLITE_CONSTRAINT;
      }else{
        if( iCol==nCol+1 ){
          if( bSeenRank ) continue;
          idxStr[iIdxStr++] = 'r';
          bSeenRank = 1;
        }else if( iCol>=0 ){
-          bSeenMatch = 1;
+          nSeenMatch++;
          idxStr[iIdxStr++] = 'M';
          sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol);
          idxStr += strlen(&idxStr[iIdxStr]);
@@ -626,6 +625,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
        idxStr += strlen(&idxStr[iIdxStr]);
        pInfo->aConstraintUsage[i].argvIndex = ++iCons;
        assert( idxStr[iIdxStr]=='\0' );
+        nSeenMatch++;
      }else if( bSeenEq==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol<0 ){
        idxStr[iIdxStr++] = '=';
        bSeenEq = 1;
@@ -662,7 +662,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
  */
  if( pInfo->nOrderBy==1 ){
    int iSort = pInfo->aOrderBy[0].iColumn;
-    if( iSort==(pConfig->nCol+1) && bSeenMatch ){
+    if( iSort==(pConfig->nCol+1) && nSeenMatch>0 ){
      idxFlags |= FTS5_BI_ORDER_RANK;
    }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){
      idxFlags |= FTS5_BI_ORDER_ROWID;
@@ -677,14 +677,17 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){

  /* Calculate the estimated cost based on the flags set in idxFlags. */
  if( bSeenEq ){
-    pInfo->estimatedCost = bSeenMatch ? 100.0 : 10.0;
-    if( bSeenMatch==0 ) fts5SetUniqueFlag(pInfo);
+    pInfo->estimatedCost = nSeenMatch ? 1000.0 : 10.0;
+    if( nSeenMatch==0 ) fts5SetUniqueFlag(pInfo);
  }else if( bSeenLt && bSeenGt ){
-    pInfo->estimatedCost = bSeenMatch ? 500.0 : 250000.0;
+    pInfo->estimatedCost = nSeenMatch ? 5000.0 : 250000.0;
  }else if( bSeenLt || bSeenGt ){
-    pInfo->estimatedCost = bSeenMatch ? 750.0 : 750000.0;
+    pInfo->estimatedCost = nSeenMatch ? 7500.0 : 750000.0;
  }else{
-    pInfo->estimatedCost = bSeenMatch ? 1000.0 : 1000000.0;
+    pInfo->estimatedCost = nSeenMatch ? 10000.0 : 1000000.0;
+  }
+  for(i=1; i<nSeenMatch; i++){
+    pInfo->estimatedCost *= 0.4;
  }

  pInfo->idxNum = idxFlags;
@@ -960,6 +963,7 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
          }
        }else{
          rc = SQLITE_OK;
+          CsrFlagSet(pCsr, FTS5CSR_REQUIRE_DOCSIZE);
        }
        break;
      }
@@ -1433,9 +1437,13 @@ static i64 fts5CursorRowid(Fts5Cursor *pCsr){
  assert( pCsr->ePlan==FTS5_PLAN_MATCH 
       || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH 
       || pCsr->ePlan==FTS5_PLAN_SOURCE 
+       || pCsr->ePlan==FTS5_PLAN_SCAN 
+       || pCsr->ePlan==FTS5_PLAN_ROWID 
  );
  if( pCsr->pSorter ){
    return pCsr->pSorter->iRowid;
+  }else if( pCsr->ePlan>=FTS5_PLAN_SCAN ){
+    return sqlite3_column_int64(pCsr->pStmt, 0);
  }else{
    return sqlite3Fts5ExprRowid(pCsr->pExpr);
  }
@@ -1452,20 +1460,10 @@ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
  int ePlan = pCsr->ePlan;
  
  assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
-  switch( ePlan ){
-    case FTS5_PLAN_SPECIAL:
-      *pRowid = 0;
-      break;
-
-    case FTS5_PLAN_SOURCE:
-    case FTS5_PLAN_MATCH:
-    case FTS5_PLAN_SORTED_MATCH:
-      *pRowid = fts5CursorRowid(pCsr);
-      break;
-
-    default:
-      *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
-      break;
+  if( ePlan==FTS5_PLAN_SPECIAL ){
+    *pRowid = 0;
+  }else{
+    *pRowid = fts5CursorRowid(pCsr);
  }

  return SQLITE_OK;
@@ -1700,6 +1698,7 @@ static int fts5UpdateMethod(
        rc = SQLITE_ERROR;
      }else{
        rc = fts5SpecialDelete(pTab, apVal);
+        bUpdateOrDelete = 1;
      }
    }else{
      rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]);
@@ -2860,7 +2859,7 @@ static int fts5FindTokenizer(
  return rc;
 }

-int sqlite3Fts5GetTokenizer(
+int fts5GetTokenizer(
  Fts5Global *pGlobal, 
  const char **azArg,
  int nArg,
@@ -2874,29 +2873,42 @@ int sqlite3Fts5GetTokenizer(
  if( pMod==0 ){
    assert( nArg>0 );
    rc = SQLITE_ERROR;
-    *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
+    if( pzErr ) *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
  }else{
    rc = pMod->x.xCreate(
-        pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->pTok
+        pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->t.pTok
    );
-    pConfig->pTokApi = &pMod->x;
+    pConfig->t.pTokApi = &pMod->x;
    if( rc!=SQLITE_OK ){
-      if( pzErr ) *pzErr = sqlite3_mprintf("error in tokenizer constructor");
+      if( pzErr && rc!=SQLITE_NOMEM ){
+        *pzErr = sqlite3_mprintf("error in tokenizer constructor");
+      }
    }else{
-      pConfig->ePattern = sqlite3Fts5TokenizerPattern(
-          pMod->x.xCreate, pConfig->pTok
+      pConfig->t.ePattern = sqlite3Fts5TokenizerPattern(
+          pMod->x.xCreate, pConfig->t.pTok
      );
    }
  }

  if( rc!=SQLITE_OK ){
-    pConfig->pTokApi = 0;
-    pConfig->pTok = 0;
+    pConfig->t.pTokApi = 0;
+    pConfig->t.pTok = 0;
  }

  return rc;
 }

+/*
+** Attempt to instantiate the tokenizer.
+*/
+int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig){
+  return fts5GetTokenizer(
+      pConfig->pGlobal, pConfig->t.azArg, pConfig->t.nArg, 
+      pConfig, pConfig->pzErrmsg
+  );
+}
+
+
 static void fts5ModuleDestroy(void *pCtx){
  Fts5TokenizerModule *pTok, *pNextTok;
  Fts5Auxiliary *pAux, *pNextAux;
@@ -2975,17 +2987,23 @@ static int fts5IntegrityMethod(

  assert( pzErr!=0 && *pzErr==0 );
  UNUSED_PARAM(isQuick);
+  assert( pTab->p.pConfig->pzErrmsg==0 );
+  pTab->p.pConfig->pzErrmsg = pzErr;
  rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0);
-  if( (rc&0xff)==SQLITE_CORRUPT ){
-    *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s",
-                zSchema, zTabname);
-     rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM;
-  }else if( rc!=SQLITE_OK ){
-    *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
-                             " FTS5 table %s.%s: %s",
-                zSchema, zTabname, sqlite3_errstr(rc));
+  if( *pzErr==0 && rc!=SQLITE_OK ){
+    if( (rc&0xff)==SQLITE_CORRUPT ){
+      *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s",
+          zSchema, zTabname);
+      rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM;
+    }else{
+      *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
+          " FTS5 table %s.%s: %s",
+          zSchema, zTabname, sqlite3_errstr(rc));
+    }
  }
+
  sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
+  pTab->p.pConfig->pzErrmsg = 0;

  return rc;
 }
@@ -1428,6 +1428,16 @@ int sqlite3Fts5TokenizerPattern(
  return FTS5_PATTERN_NONE;
 }

+/*
+** Return true if the tokenizer described by p->azArg[] is the trigram
+** tokenizer. This tokenizer needs to be loaded before xBestIndex is
+** called for the first time in order to correctly handle LIKE/GLOB.
+*/
+int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig *p){
+  return (p->nArg>=1 && 0==sqlite3_stricmp(p->azArg[0], "trigram"));
+}
+
+
 /*
 ** Register all built-in tokenizers with FTS5.
 */
@@ -64,6 +64,7 @@ struct Fts5VocabCursor {

  int nLeTerm;                    /* Size of zLeTerm in bytes */
  char *zLeTerm;                  /* (term <= $zLeTerm) paramater, or NULL */
+  int colUsed;                    /* Copy of sqlite3_index_info.colUsed */

  /* These are used by 'col' tables only */
  int iCol;
@@ -90,9 +91,11 @@ struct Fts5VocabCursor {
 /*
 ** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
 */
-#define FTS5_VOCAB_TERM_EQ 0x01
-#define FTS5_VOCAB_TERM_GE 0x02
-#define FTS5_VOCAB_TERM_LE 0x04
+#define FTS5_VOCAB_TERM_EQ 0x0100
+#define FTS5_VOCAB_TERM_GE 0x0200
+#define FTS5_VOCAB_TERM_LE 0x0400
+
+#define FTS5_VOCAB_COLUSED_MASK 0xFF


 /*
@@ -269,11 +272,13 @@ static int fts5VocabBestIndexMethod(
  int iTermEq = -1;
  int iTermGe = -1;
  int iTermLe = -1;
-  int idxNum = 0;
+  int idxNum = (int)pInfo->colUsed;
  int nArg = 0;

  UNUSED_PARAM(pUnused);

+  assert( (pInfo->colUsed & FTS5_VOCAB_COLUSED_MASK)==pInfo->colUsed );
+
  for(i=0; i<pInfo->nConstraint; i++){
    struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
    if( p->usable==0 ) continue;
@@ -365,7 +370,7 @@ static int fts5VocabOpenMethod(
      if( rc==SQLITE_OK ){
        pVTab->zErrMsg = sqlite3_mprintf(
            "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
-            );
+        );
        rc = SQLITE_ERROR;
      }
    }else{
@@ -525,9 +530,19 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){

        switch( pTab->eType ){
          case FTS5_VOCAB_ROW:
-            if( eDetail==FTS5_DETAIL_FULL ){
-              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
-                pCsr->aCnt[0]++;
+            /* Do not bother counting the number of instances if the "cnt"
+            ** column is not being read (according to colUsed).  */
+            if( eDetail==FTS5_DETAIL_FULL && (pCsr->colUsed & 0x04) ){
+              while( iPos<nPos ){
+                u32 ii;
+                fts5FastGetVarint32(pPos, iPos, ii);
+                if( ii==1 ){ 
+                  /* New column in the position list */
+                  fts5FastGetVarint32(pPos, iPos, ii);
+                }else{
+                  /* An instance - increment pCsr->aCnt[] */
+                  pCsr->aCnt[0]++;
+                }
              }
            }
            pCsr->aDoc[0]++;
@@ -625,6 +640,7 @@ static int fts5VocabFilterMethod(
  if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++];
  if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++];
  if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++];
+  pCsr->colUsed = (idxNum & FTS5_VOCAB_COLUSED_MASK);

  if( pEq ){
    zTerm = (const char *)sqlite3_value_text(pEq);
@@ -114,6 +114,10 @@ proc fts5_test_rowcount {cmd} {
  $cmd xRowCount
 }

+proc fts5_test_rowid {cmd} {
+  $cmd xRowid
+}
+
 proc test_queryphrase_cb {cnt cmd} {
  upvar $cnt L 
  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
@@ -167,6 +171,7 @@ proc fts5_aux_test_functions {db} {
    fts5_test_collist
    fts5_test_tokenize
    fts5_test_rowcount
+    fts5_test_rowid
    fts5_test_all

    fts5_test_queryphrase
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ab

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ac

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -18,7 +18,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ad

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ae

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -18,7 +18,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5af

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ag

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ah

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ai

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -19,7 +19,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5aj

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ak

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5al

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5alter

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5auto

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5aux

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -377,4 +377,28 @@ do_catchsql_test 12.3.3 {
  SELECT fts5_collist(t1, 1) FROM t1('one AND two');
 } {0 1}

+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 13.1 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii);
+  INSERT INTO t1 VALUES('a b c'), ('d e f');
+  PRAGMA integrity_check;
+} {ok}
+
+do_catchsql_test 13.2 {
+  SELECT highlight(t1, 0, '[', ']') FROM t1
+} {0 {{a b c} {d e f}}}
+
+do_execsql_test 13.3 {
+  PRAGMA writable_schema = 1;
+  UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)'
+  WHERE name = 't1';
+}
+
+db close
+sqlite3 db test.db
+do_catchsql_test 13.4 {
+  SELECT highlight(t1, 0, '[', ']') FROM t1
+} {1 {SQL logic error}}
+
 finish_test
@@ -0,0 +1,71 @@
+# 2024 June 24
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focusing on the auxiliary function APIs.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5aux
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
+  INSERT INTO x1 VALUES('a b', 'c d');
+  INSERT INTO x1 VALUES('d e', 'a b');
+  INSERT INTO x1 VALUES('a b', 'e f');
+  INSERT INTO x1 VALUES('d e', 'c d');
+}
+
+fts5_aux_test_functions db
+do_execsql_test 1.1 {
+  SELECT fts5_test_all(x1) FROM x1 WHERE rowid=2
+} [list [list {*}{
+  columnsize {2 2}
+  columntext {{d e} {a b}}
+  columntotalsize {8 8}
+  poslist {}
+  tokenize {{d e} {a b}}
+  rowcount 4
+}]]
+
+do_execsql_test 1.2 {
+  SELECT fts5_test_columntext(x1) FROM x1
+} {
+  {{a b} {c d}}
+  {{d e} {a b}}
+  {{a b} {e f}}
+  {{d e} {c d}}
+}
+
+do_execsql_test 1.3 {
+  SELECT fts5_test_rowid(x1) FROM x1
+} {
+  1 2 3 4
+}
+do_execsql_test 1.4 {
+  SELECT fts5_test_phrasecount(x1) FROM x1
+} {
+  0 0 0 0
+}
+do_catchsql_test 1.5 {
+  SELECT fts5_queryphrase(x1, 0) FROM x1
+} {1 SQLITE_RANGE} 
+do_execsql_test 1.6 {
+  SELECT fts5_test_rowcount(x1) FROM x1
+} {4 4 4 4}
+
+
+finish_test
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5auxdata

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5bigpl

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -79,7 +79,7 @@ foreach_detail_mode $::testprefix {

  do_catchsql_test 4.1 {
    SELECT * FROM t1 WHERE rowid MATCH 'a'
-  } {1 {unable to use function MATCH in the requested context}}
+  } {1 {no query solution}}
 }

 #-------------------------------------------------------------------------
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5columnsize

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5config

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5content

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless2

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless3

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless4

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless5

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt2

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt3

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -14,7 +14,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt4

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt5

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt6

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt7

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5dlidx

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5doclist

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ea

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5eb

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5fault6

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5faultG

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5faultG

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5full

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5hash

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5integrity

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -380,5 +380,32 @@ do_execsql_test 12.3 {
 } {ok}


+#-------------------------------------------------------------------
+reset_db
+do_execsql_test 13.1 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii);
+  INSERT INTO t1 VALUES('a b c'), ('d e f');
+  PRAGMA integrity_check;
+} {ok}
+
+db close
+sqlite3 db test.db
+do_catchsql_test 13.2 {
+  PRAGMA integrity_check;
+} {0 ok}
+
+do_execsql_test 13.3 {
+  PRAGMA writable_schema = 1;
+  UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)'
+  WHERE name = 't1';
+}
+
+db close
+sqlite3 db test.db
+breakpoint
+do_catchsql_test 13.4 {
+  PRAGMA integrity_check;
+} {1 {SQL logic error}}
+

 finish_test
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5lastrowid

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -517,7 +517,7 @@ fts5_aux_test_functions db
 do_execsql_test 15.3 {
  SELECT fts5_test_all(t1) FROM t1 LIMIT 1;
 } {
-  {columnsize {0 0} columntext {c d} columntotalsize {2 2} poslist {} tokenize {c d} rowcount 2}
+  {columnsize {1 1} columntext {c d} columntotalsize {2 2} poslist {} tokenize {c d} rowcount 2}
 }

 finish_test
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5merge

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -535,5 +535,36 @@ do_execsql_test 19.0 {
  COMMIT;
 }

+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 20.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a);
+  INSERT INTO x1(rowid, a) VALUES
+      (1, 'a b c d'),
+      (2, 'x b c d'),
+      (3, 'x y z d'),
+      (4, 'a y c x');
+}
+
+do_execsql_test 20.1 {
+  SELECT rowid FROM x1 WHERE x1 MATCH 'a' AND x1 MATCH 'b';
+} {1}
+
+do_execsql_test 20.2 {
+  SELECT rowid FROM x1 WHERE x1 MATCH 'a' AND x1 MATCH 'y';
+} {4}
+
+do_execsql_test 20.3 {
+  SELECT rowid FROM x1 WHERE x1 MATCH 'a' OR x1 MATCH 'y';
+} {1 4 3}
+
+do_execsql_test 20.4 {
+  SELECT rowid FROM x1 WHERE x1 MATCH 'a' OR (x1 MATCH 'y' AND x1 MATCH 'd');
+} {1 4 3}
+
+do_execsql_test 20.5 {
+  SELECT rowid FROM x1 WHERE x1 MATCH 'z' OR (x1 MATCH 'a' AND x1 MATCH 'd');
+} {3 1}
+
 finish_test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5near

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -14,7 +14,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5optimize

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5optimize2

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5optimize2

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5origintext

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5origintext2

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5origintext3

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5origintext4

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5origintext

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5phrase

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5plan

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5porter

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -18,7 +18,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5porter2

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5prefix

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5prefix2

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5query

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5rank

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -14,7 +14,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5rebuild

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5restart

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5rowid

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5savepoint

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -42,6 +42,22 @@ do_execsql_test 1.2 {
  PRAGMA integrity_check;
 } {ok}

+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE xyz USING fts5 (
+  	name,
+        content=''
+  );
+
+  INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 1);
+  INSERT INTO xyz (rowid, name) VALUES(1, 'A');
+  INSERT INTO xyz (rowid, name) VALUES(2, 'A');
+  INSERT INTO xyz(xyz, rowid, name) VALUES('delete', 2, 'A');
+}
+
+do_execsql_test 2.1 {
+  pragma quick_check;
+} {ok}
+



@@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5securefault

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 return_if_no_fts5

 do_execsql_test 1.0 {
@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5simple

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5simple2

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5simple3

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5synonym

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5synonym2

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5tokenizer

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -300,5 +300,75 @@ set ::flags [list]
 do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {}
 do_test 9.5.2 { set ::flags } {query}

+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 10.1 {
+  CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=unicode61);
+  PRAGMA writable_schema = 1;
+  UPDATE sqlite_schema 
+  SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize="unicode61 error");'
+  WHERE name = 'x1';
+}
+
+db close
+sqlite3 db test.db
+
+do_catchsql_test 10.2 {
+  SELECT * FROM x1('abc');
+} {1 {error in tokenizer constructor}}
+
+do_catchsql_test 10.3 {
+  INSERT INTO x1 VALUES('abc');
+} {1 {error in tokenizer constructor}}
+
+do_execsql_test 10.4 {
+  PRAGMA writable_schema = 1;
+  UPDATE sqlite_schema 
+  SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize="nosuch error");'
+  WHERE name = 'x1';
+}
+
+db close
+sqlite3 db test.db
+
+do_catchsql_test 10.5 {
+  SELECT * FROM x1('abc');
+} {1 {no such tokenizer: nosuch}}
+do_catchsql_test 10.6 {
+  INSERT INTO x1 VALUES('abc');
+} {1 {no such tokenizer: nosuch}}
+
+do_execsql_test 10.7 {
+  DROP TABLE x1;
+  SELECT * FROM sqlite_schema;
+}
+
+reset_db
+do_execsql_test 10.8 {
+  CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=unicode61);
+  INSERT INTO x1 VALUES('a b c'), ('d e f'), ('a b c');
+  CREATE VIRTUAL TABLE x1v USING fts5vocab(x1, row);
+
+  PRAGMA writable_schema = 1;
+  UPDATE sqlite_schema 
+  SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=simplify);'
+  WHERE name = 'x1';
+}
+
+do_execsql_test 10.9 {
+  SELECT * FROM x1v
+} {
+  a 2 2 b 2 2 c 2 2 d 1 1 e 1 1 f 1 1
+}
+
+db close
+sqlite3 db test.db
+
+do_execsql_test 10.10 {
+  SELECT * FROM x1v
+} {
+  a 2 2 b 2 2 c 2 2 d 1 1 e 1 1 f 1 1
+}

 finish_test
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5tokenizer2

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -101,9 +101,20 @@ do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE t4 USING fts5(z, tokenize=trigram);
 } {}

-breakpoint
 do_execsql_test 4.1 {
  INSERT INTO t4 VALUES('ABCD');
+  INSERT INTO t4 VALUES('DEFG');
 } {}

+db close
+sqlite3 db test.db
+
+do_eqp_test 4.1 {
+  SELECT rowid FROM t4 WHERE z LIKE '%abc%'
+} {VIRTUAL TABLE INDEX 0:L0}
+
+do_execsql_test 4.2 {
+  SELECT rowid FROM t4 WHERE z LIKE '%abc%'
+} {1}
+
 finish_test
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ubsan

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5unicode

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5unicode2

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -14,7 +14,7 @@

 source [file join [file dirname [info script]] fts5_common.tcl]

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -14,7 +14,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5unicode4

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5unindexed

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5version

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5vocab

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
--- a/Show More
+++ b/Show More
@@ -1 +1 @@
 .46.0
 .47.0