Prevent tokenize-blobs from being used with non-contentless tables. Fix some other issues with the new code on this branch.

FossilOrigin-Name: 6a640ea4d8938e2b0c73b5aa35d1ce96eea1d1304a26c98dc054e9949d1b7c8c

Makefile.in

@@ -454,7 +454,6 @@ TESTSRC += \
   $(TOP)/ext/misc/remember.c \
   $(TOP)/ext/misc/series.c \
   $(TOP)/ext/misc/spellfix.c \
-  $(TOP)/ext/misc/stmtrand.c \
   $(TOP)/ext/misc/totype.c \
   $(TOP)/ext/misc/unionvtab.c \
   $(TOP)/ext/misc/wholenumber.c \

Makefile.msc

@@ -1596,7 +1596,6 @@ TESTEXT = \
   $(TOP)\ext\misc\remember.c \
   $(TOP)\ext\misc\series.c \
   $(TOP)\ext\misc\spellfix.c \
-  $(TOP)\ext\misc\stmtrand.c \
   $(TOP)\ext\misc\totype.c \
   $(TOP)\ext\misc\unionvtab.c \
   $(TOP)\ext\misc\wholenumber.c \

VERSION

@@ -1 +1 @@
-3.47.0
+3.46.0

autoconf/tea/configure.ac

@@ -19,7 +19,7 @@ dnl	to configure the system for the local environment.
 # so that we create the export library with the dll.
 #-----------------------------------------------------------------------
 
-AC_INIT([sqlite],[3.47.0])
+AC_INIT([sqlite],[3.46.0])
 
 #--------------------------------------------------------------------
 # Call TEA_INIT as the first TEA_ macro to set up initial vars.

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.47.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.46.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.47.0'
-PACKAGE_STRING='sqlite 3.47.0'
+PACKAGE_VERSION='3.46.0'
+PACKAGE_STRING='sqlite 3.46.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1472,7 +1472,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures sqlite 3.47.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.46.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1537,7 +1537,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.47.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.46.0:";;
    esac
   cat <<\_ACEOF
 
@@ -1668,7 +1668,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-sqlite configure 3.47.0
+sqlite configure 3.46.0
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2087,7 +2087,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by sqlite $as_me 3.47.0, which was
+It was created by sqlite $as_me 3.46.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -12481,7 +12481,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.47.0, which was
+This file was extended by sqlite $as_me 3.46.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -12547,7 +12547,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.47.0
+sqlite config.status 3.46.0
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

doc/compile-for-windows.md

@@ -59,16 +59,8 @@ canonical source on a new Windows 11 PC, as of 2023-11-01:
       <li>  `nmake /f makefile.msc sqlite3.c`
       <li>  `nmake /f makefile.msc devtest`
       <li>  `nmake /f makefile.msc releasetest`
-      <li>  `nmake /f makefile.msc sqlite3.exe`
       </ul>
 
-  7.  For a debugging build of the CLI, where the ".treetrace" and ".wheretrace"
-      commands work, add the DEBUG=3 argument to nmake.  Like this:
-      <ul>
-      <li> `nmake /f makefile.msc DEBUG=3 clean sqlite3.exe`
-      </ul>
-   
-
 ## 32-bit Builds
 
 Doing a 32-bit build is just like doing a 64-bit build with the

ext/fts5/fts5.h

@@ -27,6 +27,8 @@
 extern "C" {
 #endif
 
+#define SQLITE_FTS5_TOKENIZE_SUBTYPE ((unsigned int)'T')
+
 /*************************************************************************
 ** CUSTOM AUXILIARY FUNCTIONS
 **
@@ -238,10 +240,6 @@ struct Fts5PhraseIter {
 **   (i.e. if it is a contentless table), then this API always iterates
 **   through an empty set (all calls to xPhraseFirst() set iCol to -1).
 **
-**   In all cases, matches are visited in (column ASC, offset ASC) order.
-**   i.e. all those in column 0, sorted by offset, followed by those in 
-**   column 1, etc.
-**
 ** xPhraseNext()
 **   See xPhraseFirst above.
 **

ext/fts5/fts5Int.h

@@ -142,14 +142,17 @@ struct Fts5Colset {
 */
 
 typedef struct Fts5Config Fts5Config;
-typedef struct Fts5TokenizerConfig Fts5TokenizerConfig;
 
-struct Fts5TokenizerConfig {
+/*
+** All instantiated tokenizers are stored in a list of the following objects,
+** starting at Fts5Config.pTokList.
+*/
+typedef struct Fts5TokenizerInst Fts5TokenizerInst;
+struct Fts5TokenizerInst {
+  char *zSpec;                    /* Tokenizer specification */
   Fts5Tokenizer *pTok;
   fts5_tokenizer *pTokApi;
-  const char **azArg;
-  int nArg;
-  int ePattern;                   /* FTS_PATTERN_XXX constant */
+  Fts5TokenizerInst *pNext;
 };
 
 /*
@@ -193,7 +196,7 @@ struct Fts5TokenizerConfig {
 */
 struct Fts5Config {
   sqlite3 *db;                    /* Database handle */
-  Fts5Global *pGlobal;            /* Global fts5 object for handle db */
+  Fts5Global *pGlobal;            /* Database wide data */
   char *zDb;                      /* Database holding FTS index (e.g. "main") */
   char *zName;                    /* Name of FTS index */
   int nCol;                       /* Number of columns */
@@ -209,8 +212,9 @@ struct Fts5Config {
   int bTokendata;                 /* "tokendata=" option value (dflt==0) */
   int eDetail;                    /* FTS5_DETAIL_XXX value */
   char *zContentExprlist;
-  Fts5TokenizerConfig t;
+  Fts5TokenizerInst *pTokList;
   int bLock;                      /* True when table is preparing statement */
+  int ePattern;                   /* FTS_PATTERN_XXX constant */
 
   /* Values loaded from the %_config table */
   int iVersion;                   /* fts5 file format 'version' */
@@ -266,6 +270,21 @@ int sqlite3Fts5Tokenize(
   int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
 );
 
+int sqlite3Fts5ConfigFindTokenizer(
+  Fts5Config *pConfig, 
+  const char *z, 
+  Fts5TokenizerInst **ppOut
+);
+
+int sqlite3Fts5SpecTokenize(
+  Fts5Config *pConfig,            /* FTS5 Configuration object */
+  const char *zSpec,              /* Tokenizer specification */
+  int flags,                      /* FTS5_TOKENIZE_* flags */
+  const char *pText, int nText,   /* Text to tokenize */
+  void *pCtx,                     /* Context passed to xToken() */
+  int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
+);
+
 void sqlite3Fts5Dequote(char *z);
 
 /* Load the contents of the %_config table */
@@ -605,7 +624,10 @@ struct Fts5Table {
   Fts5Index *pIndex;              /* Full-text index */
 };
 
-int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig);
+int sqlite3Fts5GetTokenizer(
+  Fts5Config*,
+  const char *zSpec
+);
 
 Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);
 
@@ -716,6 +738,14 @@ int sqlite3Fts5StorageOptimize(Fts5Storage *p);
 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
 int sqlite3Fts5StorageReset(Fts5Storage *p);
 
+int sqlite3Fts5UnpackTokenizeBlob(
+  Fts5Config *pConfig,
+  sqlite3_value *pVal,
+  Fts5TokenizerInst **ppTok,
+  char **pzText,
+  int *pbDel
+);
+
 /*
 ** End of interface to code in fts5_storage.c.
 **************************************************************************/
@@ -739,6 +769,7 @@ struct Fts5Token {
 /* Parse a MATCH expression. */
 int sqlite3Fts5ExprNew(
   Fts5Config *pConfig, 
+  Fts5TokenizerInst*,
   int bPhraseToAnd,
   int iCol,                       /* Column on LHS of MATCH operator */
   const char *zExpr,
@@ -868,7 +899,6 @@ int sqlite3Fts5TokenizerPattern(
     int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
     Fts5Tokenizer *pTok
 );
-int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig*);
 /*
 ** End of interface to code in fts5_tokenizer.c.
 **************************************************************************/

ext/fts5/fts5_config.c

@@ -224,6 +224,38 @@ static int fts5ConfigSetEnum(
   return iVal<0 ? SQLITE_ERROR : SQLITE_OK;
 }
 
+/*
+** Locate a tokenizer instance with a specification matching the second
+** argument. Create a new tokenizer if one can not be found. Return SQLITE_OK
+** if successful, or an SQLite error code otherwise.
+*/
+int sqlite3Fts5ConfigFindTokenizer(
+  Fts5Config *pConfig,            /* Table configuration */
+  const char *z,                  /* Requested tokenizer specification */
+  Fts5TokenizerInst **ppOut       /* OUT: Tokenizer instance */
+){
+  Fts5TokenizerInst *pRet = 0;
+  int rc = SQLITE_OK;
+
+  assert( pConfig->pzErrmsg );
+
+  /* Search for an existing tokenizer that matches this spec */
+  for(pRet=pConfig->pTokList; pRet; pRet=pRet->pNext){
+    if( strcmp(pRet->zSpec, z)==0 ) break;
+  }
+
+  if( pRet==0 ){
+    /* No tokenizer found - create one. */
+    rc = sqlite3Fts5GetTokenizer(pConfig, z);
+    if( rc==SQLITE_OK ){
+      pRet = pConfig->pTokList->pNext;
+    }
+  }
+
+  if( ppOut ) *ppOut = pRet;
+  return rc;
+}
+
 /*
 ** Parse a "special" CREATE VIRTUAL TABLE directive and update
 ** configuration object pConfig as appropriate.
@@ -234,6 +266,7 @@ static int fts5ConfigSetEnum(
 ** eventually free any such error message using sqlite3_free().
 */
 static int fts5ConfigParseSpecial(
+  Fts5Global *pGlobal,
   Fts5Config *pConfig,            /* Configuration object to update */
   const char *zCmd,               /* Special command to parse */
   const char *zArg,               /* Argument to parse */
@@ -295,43 +328,15 @@ static int fts5ConfigParseSpecial(
   }
 
   if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
-    const char *p = (const char*)zArg;
-    sqlite3_int64 nArg = strlen(zArg) + 1;
-    char **azArg = sqlite3Fts5MallocZero(&rc, (sizeof(char*) + 2) * nArg);
-
-    if( azArg ){
-      char *pSpace = (char*)&azArg[nArg];
-      if( pConfig->t.azArg ){
-        *pzErr = sqlite3_mprintf("multiple tokenize=... directives");
-        rc = SQLITE_ERROR;
-      }else{
-        for(nArg=0; p && *p; nArg++){
-          const char *p2 = fts5ConfigSkipWhitespace(p);
-          if( *p2=='\'' ){
-            p = fts5ConfigSkipLiteral(p2);
-          }else{
-            p = fts5ConfigSkipBareword(p2);
-          }
-          if( p ){
-            memcpy(pSpace, p2, p-p2);
-            azArg[nArg] = pSpace;
-            sqlite3Fts5Dequote(pSpace);
-            pSpace += (p - p2) + 1;
-            p = fts5ConfigSkipWhitespace(p);
-          }
-        }
-        if( p==0 ){
-          *pzErr = sqlite3_mprintf("parse error in tokenize directive");
-          rc = SQLITE_ERROR;
-        }else{
-          pConfig->t.azArg = (const char**)azArg;
-          pConfig->t.nArg = nArg;
-          azArg = 0;
-        }
-      }
+    if( pConfig->pTokList ){
+      *pzErr = sqlite3_mprintf("multiple tokenize=... directives");
+      rc = SQLITE_ERROR;
+    }else{
+      assert( pConfig->pzErrmsg==0 );
+      pConfig->pzErrmsg = pzErr;
+      rc = sqlite3Fts5GetTokenizer(pConfig, zArg);
+      pConfig->pzErrmsg = 0;
     }
-    sqlite3_free(azArg);
-
     return rc;
   }
 
@@ -540,8 +545,8 @@ int sqlite3Fts5ConfigParse(
   *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
   if( pRet==0 ) return SQLITE_NOMEM;
   memset(pRet, 0, sizeof(Fts5Config));
-  pRet->pGlobal = pGlobal;
   pRet->db = db;
+  pRet->pGlobal = pGlobal;
   pRet->iCookie = -1;
 
   nByte = nArg * (sizeof(char*) + sizeof(u8));
@@ -589,7 +594,7 @@ int sqlite3Fts5ConfigParse(
         rc = SQLITE_ERROR;
       }else{
         if( bOption ){
-          rc = fts5ConfigParseSpecial(pRet, 
+          rc = fts5ConfigParseSpecial(pGlobal, pRet, 
             ALWAYS(zOne)?zOne:"",
             zTwo?zTwo:"",
             pzErr
@@ -627,6 +632,13 @@ int sqlite3Fts5ConfigParse(
     rc = SQLITE_ERROR;
   }
 
+  /* If a tokenizer= option was successfully parsed, the tokenizer has
+  ** already been allocated. Otherwise, allocate an instance of the default
+  ** tokenizer (unicode61) now.  */
+  if( rc==SQLITE_OK && pRet->pTokList==0 ){
+    rc = sqlite3Fts5GetTokenizer(pRet, 0);
+  }
+
   /* If no zContent option was specified, fill in the default values. */
   if( rc==SQLITE_OK && pRet->zContent==0 ){
     const char *zTail = 0;
@@ -662,18 +674,30 @@ int sqlite3Fts5ConfigParse(
   return rc;
 }
 
+/*
+** Free all tokenizer instances in the list starting at Fts5Config.pTokList.
+*/
+static void fts5ConfigFreeTokenizers(Fts5Config *pConfig){
+  Fts5TokenizerInst *p = pConfig->pTokList;
+  while( p ){
+    Fts5TokenizerInst *pNext = p->pNext;
+    p->pTokApi->xDelete(p->pTok);
+    sqlite3_free(p);
+    p = pNext;
+  }
+  pConfig->pTokList = 0;
+}
+
 /*
 ** Free the configuration object passed as the only argument.
 */
 void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
   if( pConfig ){
     int i;
-    if( pConfig->t.pTok ){
-      pConfig->t.pTokApi->xDelete(pConfig->t.pTok);
-    }
-    sqlite3_free((char*)pConfig->t.azArg);
+    fts5ConfigFreeTokenizers(pConfig);
     sqlite3_free(pConfig->zDb);
     sqlite3_free(pConfig->zName);
+    fts5ConfigFreeTokenizers(pConfig);
     for(i=0; i<pConfig->nCol; i++){
       sqlite3_free(pConfig->azCol[i]);
     }
@@ -746,18 +770,34 @@ int sqlite3Fts5Tokenize(
   void *pCtx,                     /* Context passed to xToken() */
   int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
 ){
-  int rc = SQLITE_OK;
   if( pText ){
-    if( pConfig->t.pTok==0 ){
-      rc = sqlite3Fts5LoadTokenizer(pConfig);
-    }
-    if( rc==SQLITE_OK ){
-      rc = pConfig->t.pTokApi->xTokenize(
-          pConfig->t.pTok, pCtx, flags, pText, nText, xToken
-      );
-    }
+    Fts5TokenizerInst *p = pConfig->pTokList;
+    return p->pTokApi->xTokenize(p->pTok, pCtx, flags, pText, nText, xToken);
   }
-  return rc;
+  return SQLITE_OK;
+}
+
+/*
+** Like sqlite3Fts5Tokenize(), but using the tokenizer defined by 
+** specification zSpec.
+*/
+int sqlite3Fts5SpecTokenize(
+  Fts5Config *pConfig,            /* FTS5 Configuration object */
+  const char *zSpec,              /* Tokenizer specification */
+  int flags,                      /* FTS5_TOKENIZE_* flags */
+  const char *pText, int nText,   /* Text to tokenize */
+  void *pCtx,                     /* Context passed to xToken() */
+  int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
+){
+  if( pText ){
+    Fts5TokenizerInst *p = pConfig->pTokList;
+    if( zSpec ){
+      int rc = sqlite3Fts5ConfigFindTokenizer(pConfig, zSpec, &p);
+      if( rc!=SQLITE_OK ) return rc;
+    }
+    return p->pTokApi->xTokenize(p->pTok, pCtx, flags, pText, nText, xToken);
+  }
+  return SQLITE_OK;
 }
 
 /*

ext/fts5/fts5_expr.c

@@ -141,6 +141,7 @@ struct Fts5Parse {
   Fts5ExprPhrase **apPhrase;      /* Array of all phrases */
   Fts5ExprNode *pExpr;            /* Result of a successful parse */
   int bPhraseToAnd;               /* Convert "a+b" to "a AND b" */
+  Fts5TokenizerInst *pTok;
 };
 
 /*
@@ -255,6 +256,7 @@ static void fts5ParseFree(void *p){ sqlite3_free(p); }
 
 int sqlite3Fts5ExprNew(
   Fts5Config *pConfig,            /* FTS5 Configuration */
+  Fts5TokenizerInst *pTok,        /* Tokenizer to use, or NULL */
   int bPhraseToAnd,
   int iCol,
   const char *zExpr,              /* Expression text */
@@ -272,6 +274,7 @@ int sqlite3Fts5ExprNew(
   *pzErr = 0;
   memset(&sParse, 0, sizeof(sParse));
   sParse.bPhraseToAnd = bPhraseToAnd;
+  sParse.pTok = pTok ? pTok : pConfig->pTokList;
   pEngine = sqlite3Fts5ParserAlloc(fts5ParseAlloc);
   if( pEngine==0 ){ return SQLITE_NOMEM; }
   sParse.pConfig = pConfig;
@@ -324,11 +327,7 @@ int sqlite3Fts5ExprNew(
   }
 
   sqlite3_free(sParse.apPhrase);
-  if( 0==*pzErr ){
-    *pzErr = sParse.zErr;
-  }else{
-    sqlite3_free(sParse.zErr);
-  }
+  *pzErr = sParse.zErr;
   return sParse.rc;
 }
 
@@ -411,7 +410,9 @@ int sqlite3Fts5ExprPattern(
         }
       }
       zExpr[iOut] = '\0';
-      rc = sqlite3Fts5ExprNew(pConfig, bAnd, iCol, zExpr, pp,pConfig->pzErrmsg);
+      rc = sqlite3Fts5ExprNew(
+          pConfig, 0, bAnd, iCol, zExpr, pp,pConfig->pzErrmsg
+      );
     }else{
       *pp = 0;
     }
@@ -1847,11 +1848,12 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
 
   rc = fts5ParseStringFromToken(pToken, &z);
   if( rc==SQLITE_OK ){
+    Fts5TokenizerInst *p = pParse->pTok;
     int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0);
     int n;
     sqlite3Fts5Dequote(z);
     n = (int)strlen(z);
-    rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize);
+    rc = p->pTokApi->xTokenize(p->pTok, &sCtx, flags, z, n, fts5ParseTokenize);
   }
   sqlite3_free(z);
   if( rc || (rc = sCtx.rc) ){
@@ -1875,7 +1877,6 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
     }else if( sCtx.pPhrase->nTerm ){
       sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix;
     }
-    assert( pParse->apPhrase!=0 );
     pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
   }
 
@@ -1895,7 +1896,7 @@ int sqlite3Fts5ExprClonePhrase(
   Fts5ExprPhrase *pOrig = 0;      /* The phrase extracted from pExpr */
   Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
   TokenCtx sCtx = {0,0,0};        /* Context object for fts5ParseTokenize */
-  if( !pExpr || iPhrase<0 || iPhrase>=pExpr->nPhrase ){
+  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
     rc = SQLITE_RANGE;
   }else{
     pOrig = pExpr->apExprPhrase[iPhrase];
@@ -2470,8 +2471,6 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
         );
 
     if( pRight->eType==FTS5_EOF ){
-      assert( pParse->apPhrase!=0 );
-      assert( pParse->nPhrase>0 );
       assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] );
       sqlite3Fts5ParseNodeFree(pRight);
       pRet = pLeft;
@@ -2784,7 +2783,7 @@ static void fts5ExprFunction(
 
   rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
   if( rc==SQLITE_OK ){
-    rc = sqlite3Fts5ExprNew(pConfig, 0, pConfig->nCol, zExpr, &pExpr, &zErr);
+    rc = sqlite3Fts5ExprNew(pConfig, 0, 0, pConfig->nCol, zExpr, &pExpr, &zErr);
   }
   if( rc==SQLITE_OK ){
     char *zText;

ext/fts5/fts5_main.c

@@ -115,7 +115,6 @@ struct Fts5TokenizerModule {
 struct Fts5FullTable {
   Fts5Table p;                    /* Public class members from fts5Int.h */
   Fts5Storage *pStorage;          /* Document store */
-  Fts5Global *pGlobal;            /* Global (connection wide) data */
   Fts5Cursor *pSortCsr;           /* Sort data from this cursor */
   int iSavepoint;                 /* Successful xSavepoint()+1 */
   
@@ -377,12 +376,7 @@ static int fts5InitVtab(
     assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 );
   }
   if( rc==SQLITE_OK ){
-    pConfig->pzErrmsg = pzErr;
     pTab->p.pConfig = pConfig;
-    pTab->pGlobal = pGlobal;
-    if( bCreate || sqlite3Fts5TokenizerPreload(&pConfig->t) ){
-      rc = sqlite3Fts5LoadTokenizer(pConfig);
-    }
   }
 
   /* Open the index sub-system */
@@ -404,8 +398,11 @@ static int fts5InitVtab(
 
   /* Load the initial configuration */
   if( rc==SQLITE_OK ){
+    assert( pConfig->pzErrmsg==0 );
+    pConfig->pzErrmsg = pzErr;
     rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
     sqlite3Fts5IndexRollback(pTab->p.pIndex);
+    pConfig->pzErrmsg = 0;
   }
 
   if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
@@ -415,7 +412,6 @@ static int fts5InitVtab(
     rc = sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS);
   }
 
-  if( pConfig ) pConfig->pzErrmsg = 0;
   if( rc!=SQLITE_OK ){
     fts5FreeVtab(pTab);
     pTab = 0;
@@ -483,10 +479,10 @@ static int fts5UsePatternMatch(
 ){
   assert( FTS5_PATTERN_GLOB==SQLITE_INDEX_CONSTRAINT_GLOB );
   assert( FTS5_PATTERN_LIKE==SQLITE_INDEX_CONSTRAINT_LIKE );
-  if( pConfig->t.ePattern==FTS5_PATTERN_GLOB && p->op==FTS5_PATTERN_GLOB ){
+  if( pConfig->ePattern==FTS5_PATTERN_GLOB && p->op==FTS5_PATTERN_GLOB ){
     return 1;
   }
-  if( pConfig->t.ePattern==FTS5_PATTERN_LIKE 
+  if( pConfig->ePattern==FTS5_PATTERN_LIKE 
    && (p->op==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB)
   ){
     return 1;
@@ -533,11 +529,11 @@ static int fts5UsePatternMatch(
 ** This function ensures that there is at most one "r" or "=". And that if
 ** there exists an "=" then there is no "<" or ">".
 **
-** If an unusable MATCH operator is present in the WHERE clause, then
-** SQLITE_CONSTRAINT is returned.
-**
 ** Costs are assigned as follows:
 **
+**  a) If an unusable MATCH operator is present in the WHERE clause, the
+**     cost is unconditionally set to 1e50 (a really big number).
+**
 **  a) If a MATCH operator is present, the cost depends on the other
 **     constraints also present. As follows:
 **
@@ -569,7 +565,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
   int bSeenEq = 0;
   int bSeenGt = 0;
   int bSeenLt = 0;
-  int nSeenMatch = 0;
+  int bSeenMatch = 0;
   int bSeenRank = 0;
 
 
@@ -600,15 +596,18 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
       /* A MATCH operator or equivalent */
       if( p->usable==0 || iCol<0 ){
         /* As there exists an unusable MATCH constraint this is an 
-        ** unusable plan. Return SQLITE_CONSTRAINT. */
-        return SQLITE_CONSTRAINT;
+        ** unusable plan. Set a prohibitively high cost. */
+        pInfo->estimatedCost = 1e50;
+        assert( iIdxStr < pInfo->nConstraint*6 + 1 );
+        idxStr[iIdxStr] = 0;
+        return SQLITE_OK;
       }else{
         if( iCol==nCol+1 ){
           if( bSeenRank ) continue;
           idxStr[iIdxStr++] = 'r';
           bSeenRank = 1;
         }else if( iCol>=0 ){
-          nSeenMatch++;
+          bSeenMatch = 1;
           idxStr[iIdxStr++] = 'M';
           sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol);
           idxStr += strlen(&idxStr[iIdxStr]);
@@ -625,7 +624,6 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
         idxStr += strlen(&idxStr[iIdxStr]);
         pInfo->aConstraintUsage[i].argvIndex = ++iCons;
         assert( idxStr[iIdxStr]=='\0' );
-        nSeenMatch++;
       }else if( bSeenEq==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol<0 ){
         idxStr[iIdxStr++] = '=';
         bSeenEq = 1;
@@ -662,7 +660,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
   */
   if( pInfo->nOrderBy==1 ){
     int iSort = pInfo->aOrderBy[0].iColumn;
-    if( iSort==(pConfig->nCol+1) && nSeenMatch>0 ){
+    if( iSort==(pConfig->nCol+1) && bSeenMatch ){
       idxFlags |= FTS5_BI_ORDER_RANK;
     }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){
       idxFlags |= FTS5_BI_ORDER_ROWID;
@@ -677,17 +675,14 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
 
   /* Calculate the estimated cost based on the flags set in idxFlags. */
   if( bSeenEq ){
-    pInfo->estimatedCost = nSeenMatch ? 1000.0 : 10.0;
-    if( nSeenMatch==0 ) fts5SetUniqueFlag(pInfo);
+    pInfo->estimatedCost = bSeenMatch ? 100.0 : 10.0;
+    if( bSeenMatch==0 ) fts5SetUniqueFlag(pInfo);
   }else if( bSeenLt && bSeenGt ){
-    pInfo->estimatedCost = nSeenMatch ? 5000.0 : 250000.0;
+    pInfo->estimatedCost = bSeenMatch ? 500.0 : 250000.0;
   }else if( bSeenLt || bSeenGt ){
-    pInfo->estimatedCost = nSeenMatch ? 7500.0 : 750000.0;
+    pInfo->estimatedCost = bSeenMatch ? 750.0 : 750000.0;
   }else{
-    pInfo->estimatedCost = nSeenMatch ? 10000.0 : 1000000.0;
-  }
-  for(i=1; i<nSeenMatch; i++){
-    pInfo->estimatedCost *= 0.4;
+    pInfo->estimatedCost = bSeenMatch ? 1000.0 : 1000000.0;
   }
 
   pInfo->idxNum = idxFlags;
@@ -696,7 +691,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
 
 static int fts5NewTransaction(Fts5FullTable *pTab){
   Fts5Cursor *pCsr;
-  for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
+  for(pCsr=pTab->p.pConfig->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
     if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK;
   }
   return sqlite3Fts5StorageReset(pTab->pStorage);
@@ -717,7 +712,7 @@ static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
     nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
     pCsr = (Fts5Cursor*)sqlite3_malloc64(nByte);
     if( pCsr ){
-      Fts5Global *pGlobal = pTab->pGlobal;
+      Fts5Global *pGlobal = pConfig->pGlobal;
       memset(pCsr, 0, (size_t)nByte);
       pCsr->aColumnSize = (int*)&pCsr[1];
       pCsr->pNext = pGlobal->pCsr;
@@ -804,7 +799,7 @@ static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
 
     fts5FreeCursorComponents(pCsr);
     /* Remove the cursor from the Fts5Global.pCsr list */
-    for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
+    for(pp=&pTab->p.pConfig->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
     *pp = pCsr->pNext;
 
     sqlite3_free(pCsr);
@@ -857,7 +852,7 @@ static int fts5SorterNext(Fts5Cursor *pCsr){
 */
 static void fts5TripCursors(Fts5FullTable *pTab){
   Fts5Cursor *pCsr;
-  for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
+  for(pCsr=pTab->p.pConfig->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
     if( pCsr->ePlan==FTS5_PLAN_MATCH
      && pCsr->base.pVtab==(sqlite3_vtab*)pTab 
     ){
@@ -963,7 +958,6 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
           }
         }else{
           rc = SQLITE_OK;
-          CsrFlagSet(pCsr, FTS5CSR_REQUIRE_DOCSIZE);
         }
         break;
       }
@@ -1110,7 +1104,7 @@ static int fts5SpecialMatch(
 static Fts5Auxiliary *fts5FindAuxiliary(Fts5FullTable *pTab, const char *zName){
   Fts5Auxiliary *pAux;
 
-  for(pAux=pTab->pGlobal->pAux; pAux; pAux=pAux->pNext){
+  for(pAux=pTab->p.pConfig->pGlobal->pAux; pAux; pAux=pAux->pNext){
     if( sqlite3_stricmp(zName, pAux->zFunc)==0 ) return pAux;
   }
 
@@ -1281,7 +1275,15 @@ static int fts5FilterMethod(
         pRank = apVal[i];
         break;
       case 'M': {
-        const char *zText = (const char*)sqlite3_value_text(apVal[i]);
+        Fts5TokenizerInst *pInst = 0;
+        char *zText = 0;
+        int bDel = 0;
+
+        rc = sqlite3Fts5UnpackTokenizeBlob(
+            pConfig, apVal[i], &pInst, &zText, &bDel
+        );
+        if( rc!=SQLITE_OK ) goto filter_out;
+
         if( zText==0 ) zText = "";
         iCol = 0;
         do{
@@ -1294,17 +1296,19 @@ static int fts5FilterMethod(
           ** indicates that the MATCH expression is not a full text query,
           ** but a request for an internal parameter.  */
           rc = fts5SpecialMatch(pTab, pCsr, &zText[1]);
+          if( bDel ) sqlite3_free(zText);
           goto filter_out;
         }else{
           char **pzErr = &pTab->p.base.zErrMsg;
-          rc = sqlite3Fts5ExprNew(pConfig, 0, iCol, zText, &pExpr, pzErr);
+          rc = sqlite3Fts5ExprNew(pConfig, pInst, 0, iCol, zText, &pExpr,pzErr);
           if( rc==SQLITE_OK ){
             rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr);
             pExpr = 0;
           }
-          if( rc!=SQLITE_OK ) goto filter_out;
         }
 
+        if( bDel ) sqlite3_free(zText);
+        if( rc!=SQLITE_OK ) goto filter_out;
         break;
       }
       case 'L':
@@ -1437,13 +1441,9 @@ static i64 fts5CursorRowid(Fts5Cursor *pCsr){
   assert( pCsr->ePlan==FTS5_PLAN_MATCH 
        || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH 
        || pCsr->ePlan==FTS5_PLAN_SOURCE 
-       || pCsr->ePlan==FTS5_PLAN_SCAN 
-       || pCsr->ePlan==FTS5_PLAN_ROWID 
   );
   if( pCsr->pSorter ){
     return pCsr->pSorter->iRowid;
-  }else if( pCsr->ePlan>=FTS5_PLAN_SCAN ){
-    return sqlite3_column_int64(pCsr->pStmt, 0);
   }else{
     return sqlite3Fts5ExprRowid(pCsr->pExpr);
   }
@@ -1460,10 +1460,20 @@ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
   int ePlan = pCsr->ePlan;
   
   assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
-  if( ePlan==FTS5_PLAN_SPECIAL ){
-    *pRowid = 0;
-  }else{
-    *pRowid = fts5CursorRowid(pCsr);
+  switch( ePlan ){
+    case FTS5_PLAN_SPECIAL:
+      *pRowid = 0;
+      break;
+
+    case FTS5_PLAN_SOURCE:
+    case FTS5_PLAN_MATCH:
+    case FTS5_PLAN_SORTED_MATCH:
+      *pRowid = fts5CursorRowid(pCsr);
+      break;
+
+    default:
+      *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
+      break;
   }
 
   return SQLITE_OK;
@@ -1698,7 +1708,6 @@ static int fts5UpdateMethod(
         rc = SQLITE_ERROR;
       }else{
         rc = fts5SpecialDelete(pTab, apVal);
-        bUpdateOrDelete = 1;
       }
     }else{
       rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]);
@@ -2859,56 +2868,101 @@ static int fts5FindTokenizer(
   return rc;
 }
 
-int fts5GetTokenizer(
-  Fts5Global *pGlobal, 
-  const char **azArg,
-  int nArg,
+/*
+** Add a tokenizer with specification zSpec to the list at Fts5Config.pTokList.
+** Return SQLITE_OK if successful, or an SQLite error code otherwise.
+*/
+int sqlite3Fts5GetTokenizer(
   Fts5Config *pConfig,
-  char **pzErr
+  const char *zSpec
 ){
-  Fts5TokenizerModule *pMod;
   int rc = SQLITE_OK;
+  char **pzErr = pConfig->pzErrmsg;
+  const char **azArg = 0;
+  char *pDel = 0;
+  sqlite3_int64 nArg = 0;
 
-  pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]);
-  if( pMod==0 ){
-    assert( nArg>0 );
-    rc = SQLITE_ERROR;
-    if( pzErr ) *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
-  }else{
-    rc = pMod->x.xCreate(
-        pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->t.pTok
-    );
-    pConfig->t.pTokApi = &pMod->x;
-    if( rc!=SQLITE_OK ){
-      if( pzErr && rc!=SQLITE_NOMEM ){
-        *pzErr = sqlite3_mprintf("error in tokenizer constructor");
+  assert( pzErr || (zSpec==0 && pConfig->pTokList==0) );
+  if( zSpec ){
+    const char *p = (const char*)zSpec;
+    char *pSpace = 0;
+
+    nArg = strlen(zSpec) + 1;
+    pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
+    pSpace = pDel;
+    azArg = (const char**)sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
+
+    if( azArg && pSpace ){
+      for(nArg=0; p && *p; nArg++){
+        const char *p2 = fts5ConfigSkipWhitespace(p);
+        if( *p2=='\'' ){
+          p = fts5ConfigSkipLiteral(p2);
+        }else{
+          p = fts5ConfigSkipBareword(p2);
+        }
+        if( p ){
+          memcpy(pSpace, p2, p-p2);
+          azArg[nArg] = pSpace;
+          sqlite3Fts5Dequote(pSpace);
+          pSpace += (p - p2) + 1;
+          p = fts5ConfigSkipWhitespace(p);
+        }
+      }
+      if( p==0 ){
+        *pzErr= sqlite3_mprintf("parse error in tokenize directive");
+        rc = SQLITE_ERROR;
       }
-    }else{
-      pConfig->t.ePattern = sqlite3Fts5TokenizerPattern(
-          pMod->x.xCreate, pConfig->t.pTok
-      );
     }
   }
 
-  if( rc!=SQLITE_OK ){
-    pConfig->t.pTokApi = 0;
-    pConfig->t.pTok = 0;
+  if( rc==SQLITE_OK ){
+    Fts5TokenizerModule *pMod;
+    pMod = fts5LocateTokenizer(pConfig->pGlobal, nArg==0 ? 0 : azArg[0]);
+    if( pMod==0 ){
+      assert( nArg>0 );
+      rc = SQLITE_ERROR;
+      *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
+    }else{
+      int nSpec = zSpec ? strlen(zSpec) + 1 : 0;
+      int nByte = sizeof(Fts5TokenizerInst) + nSpec;
+      Fts5TokenizerInst *pNew = sqlite3Fts5MallocZero(&rc, nByte);
+      if( pNew ){
+        if( zSpec ){
+          pNew->zSpec = (char*)&pNew[1];
+          memcpy(pNew->zSpec, zSpec, nSpec);
+        }
+        rc = pMod->x.xCreate(
+            pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pNew->pTok
+        );
+        pNew->pTokApi = &pMod->x;
+        if( rc!=SQLITE_OK ){
+          if( pzErr ){
+            *pzErr = sqlite3_mprintf("error in tokenizer constructor");
+          }
+        }else if( pConfig->pTokList==0 ){
+          pConfig->ePattern = sqlite3Fts5TokenizerPattern(
+              pMod->x.xCreate, pNew->pTok
+          );
+        }
+      }
+      if( rc==SQLITE_OK ){
+        if( pConfig->pTokList ){
+          pNew->pNext = pConfig->pTokList->pNext;
+          pConfig->pTokList->pNext = pNew;
+        }else{
+          pConfig->pTokList = pNew;
+        }
+      }else{
+        sqlite3_free(pNew);
+      }
+    }
   }
 
+  sqlite3_free(azArg);
+  sqlite3_free(pDel);
   return rc;
 }
 
-/*
-** Attempt to instantiate the tokenizer.
-*/
-int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig){
-  return fts5GetTokenizer(
-      pConfig->pGlobal, pConfig->t.azArg, pConfig->t.nArg, 
-      pConfig, pConfig->pzErrmsg
-  );
-}
-
-
 static void fts5ModuleDestroy(void *pCtx){
   Fts5TokenizerModule *pTok, *pNextTok;
   Fts5Auxiliary *pAux, *pNextAux;
@@ -2987,23 +3041,17 @@ static int fts5IntegrityMethod(
 
   assert( pzErr!=0 && *pzErr==0 );
   UNUSED_PARAM(isQuick);
-  assert( pTab->p.pConfig->pzErrmsg==0 );
-  pTab->p.pConfig->pzErrmsg = pzErr;
   rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0);
-  if( *pzErr==0 && rc!=SQLITE_OK ){
-    if( (rc&0xff)==SQLITE_CORRUPT ){
-      *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s",
-          zSchema, zTabname);
-      rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM;
-    }else{
-      *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
-          " FTS5 table %s.%s: %s",
-          zSchema, zTabname, sqlite3_errstr(rc));
-    }
+  if( (rc&0xff)==SQLITE_CORRUPT ){
+    *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s",
+                zSchema, zTabname);
+     rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM;
+  }else if( rc!=SQLITE_OK ){
+    *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
+                             " FTS5 table %s.%s: %s",
+                zSchema, zTabname, sqlite3_errstr(rc));
   }
-
   sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
-  pTab->p.pConfig->pzErrmsg = 0;
 
   return rc;
 }

ext/fts5/fts5_storage.c

@@ -399,6 +399,69 @@ static int fts5StorageInsertCallback(
   return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
 }
 
+/*
+** If the value passed as the third argument is a tokenizer blob, and the
+** Fts5Config object indicates that the table is a contentless-table, 
+** return non-zero.
+**
+** Or, if the value passed as the third argument is a tokenizer blob but
+** the table is not a contentless table, set *pRc to SQLITE_ERROR and leave
+** an error message in the Fts5Config object. Return 0 in this case.
+**
+** Finally, if the value is not a tokenizer blob, return 0.
+*/
+static int fts5IsTokenizeBlob(
+  int *pRc, 
+  Fts5Config *pConfig, 
+  sqlite3_value *pVal
+){
+  assert( *pRc==SQLITE_OK );
+  if( sqlite3_value_subtype(pVal)==SQLITE_FTS5_TOKENIZE_SUBTYPE 
+   && sqlite3_value_type(pVal)==SQLITE_BLOB 
+  ){
+    if( pConfig->eContent==FTS5_CONTENT_NONE ) return 1;
+
+    *pRc = SQLITE_ERROR;
+    *pConfig->pzErrmsg = sqlite3_mprintf(
+        "table does not support alternative tokenizers"
+    );
+  }
+  return 0;
+}
+
+/*
+** Value pVal is guaranteed to be a tokenize-blob. This function unpacks
+** the blob and returns a pointer to the nul-terminated tokenizer
+** specification. It also sets output parameter (*pzT) to point to the
+** start of the utf-8 text value (not nul-terminated) and (*pnT) to the
+** number of valid bytes in this buffer.
+*/
+static const char *fts5UnpackTokenizeBlob(
+  sqlite3_value *pVal,
+  const char **pzT,
+  int *pnT
+){
+  const u8 *pBlob = sqlite3_value_blob(pVal);
+  int nBlob = sqlite3_value_bytes(pVal);
+  int ii;
+
+  assert( sqlite3_value_subtype(pVal)==SQLITE_FTS5_TOKENIZE_SUBTYPE );
+  assert( sqlite3_value_type(pVal)==SQLITE_BLOB );
+
+  for(ii=0; pBlob[ii]; ii++){
+    if( ii==nBlob ){
+      *pzT = 0;
+      *pnT = 0;
+      return 0;
+    }
+  }
+
+  *pzT = (const char*)&pBlob[ii+1];
+  *pnT = nBlob - ii - 1;
+  return (const char*)pBlob;
+}
+
+
 /*
 ** If a row with rowid iDel is present in the %_content table, add the
 ** delete-markers to the FTS index necessary to delete it. Do not actually
@@ -429,26 +492,34 @@ static int fts5StorageDeleteFromIndex(
   ctx.iCol = -1;
   for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
     if( pConfig->abUnindexed[iCol-1]==0 ){
-      const char *zText;
-      int nText;
+      const char *zText = 0;
+      const char *zTok = 0;
+      int nText = 0;
       assert( pSeek==0 || apVal==0 );
       assert( pSeek!=0 || apVal!=0 );
       if( pSeek ){
         zText = (const char*)sqlite3_column_text(pSeek, iCol);
         nText = sqlite3_column_bytes(pSeek, iCol);
       }else if( ALWAYS(apVal) ){
-        zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
-        nText = sqlite3_value_bytes(apVal[iCol-1]);
+        sqlite3_value *pVal = apVal[iCol-1];
+        if( fts5IsTokenizeBlob(&rc, pConfig, pVal) ){
+          zTok = fts5UnpackTokenizeBlob(pVal, &zText, &nText);
+        }else{
+          zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
+          nText = sqlite3_value_bytes(apVal[iCol-1]);
+        }
       }else{
         continue;
       }
       ctx.szCol = 0;
-      rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
-          zText, nText, (void*)&ctx, fts5StorageInsertCallback
-      );
-      p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
-      if( p->aTotalSize[iCol-1]<0 ){
-        rc = FTS5_CORRUPT;
+      if( rc==SQLITE_OK ){
+        rc = sqlite3Fts5SpecTokenize(pConfig, zTok, FTS5_TOKENIZE_DOCUMENT, 
+            zText, nText, (void*)&ctx, fts5StorageInsertCallback
+        );
+        p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
+        if( p->aTotalSize[iCol-1]<0 ){
+          rc = FTS5_CORRUPT;
+        }
       }
     }
   }
@@ -752,6 +823,42 @@ static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
   return rc;
 }
 
+/*
+** This function is used to extract text from an sqlite3_value to use
+** as an fts5 query string. It also finds the required tokenizer to use
+** for tokenizing query terms.
+**
+** If successful, SQLITE_OK is returned, output variable (*ppTok) is set
+** to point to the required tokenizer instance, (*pzText) points to a
+** nul-terminated buffer containing the query string as utf-8 text, and
+** (*pbDel) is set to true if the caller must sqlite3_free(*pzText) at
+** some point in the future. Or, if an error occurs, an SQLite error
+** code is returned.
+*/
+int sqlite3Fts5UnpackTokenizeBlob(
+  Fts5Config *pConfig,
+  sqlite3_value *pVal,
+  Fts5TokenizerInst **ppTok,
+  char **pzText,
+  int *pbDel                      /* OUT: Set to true if sqlite3_free() req. */
+){
+  int rc = SQLITE_OK;
+  if( fts5IsTokenizeBlob(&rc, pConfig, pVal) ){
+    const char *zTok = 0;
+    const char *zText = 0;
+    int nText = 0;
+    zTok = fts5UnpackTokenizeBlob(pVal, &zText, &nText);
+    rc = sqlite3Fts5ConfigFindTokenizer(pConfig, zTok, ppTok);
+    *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, zText);
+    *pbDel = 1;
+  }else{
+    *pzText = (char*)sqlite3_value_text(pVal);
+    *pbDel = 0;
+    *ppTok = pConfig->pTokList;
+  }
+  return rc;
+}
+
 /*
 ** Insert a new row into the FTS content table.
 */
@@ -775,7 +882,15 @@ int sqlite3Fts5StorageContentInsert(
     int i;                        /* Counter variable */
     rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
     for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
-      rc = sqlite3_bind_value(pInsert, i, apVal[i]);
+      sqlite3_value *pVal = apVal[i];
+      if( fts5IsTokenizeBlob(&rc, pConfig, pVal) ){
+        const char *zT = 0;
+        int nT = 0;
+        fts5UnpackTokenizeBlob(pVal, &zT, &nT);
+        rc = sqlite3_bind_text(pInsert, i, zT, nT, SQLITE_STATIC);
+      }else if( rc==SQLITE_OK ){
+        rc = sqlite3_bind_value(pInsert, i, apVal[i]);
+      }
     }
     if( rc==SQLITE_OK ){
       sqlite3_step(pInsert);
@@ -810,14 +925,23 @@ int sqlite3Fts5StorageIndexInsert(
   for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
     ctx.szCol = 0;
     if( pConfig->abUnindexed[ctx.iCol]==0 ){
-      const char *zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]);
-      int nText = sqlite3_value_bytes(apVal[ctx.iCol+2]);
-      rc = sqlite3Fts5Tokenize(pConfig, 
-          FTS5_TOKENIZE_DOCUMENT,
-          zText, nText,
-          (void*)&ctx,
-          fts5StorageInsertCallback
-      );
+      sqlite3_value *pVal = apVal[ctx.iCol+2];
+      const char *zText = 0;
+      const char *zTok = 0;
+      int nText = 0;
+
+      if( fts5IsTokenizeBlob(&rc, pConfig, pVal) ){
+        zTok = fts5UnpackTokenizeBlob(pVal, &zText, &nText);
+      }else{
+        zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]);
+        nText = sqlite3_value_bytes(apVal[ctx.iCol+2]);
+      }
+      if( rc==SQLITE_OK ){
+        rc = sqlite3Fts5SpecTokenize(pConfig, 
+            zTok, FTS5_TOKENIZE_DOCUMENT, zText, nText, 
+            (void*)&ctx, fts5StorageInsertCallback
+        );
+      }
     }
     sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
     p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;

ext/fts5/fts5_tcl.c

@@ -1126,6 +1126,35 @@ static int SQLITE_TCLAPI f5tRegisterMatchinfo(
   return TCL_OK;
 }
 
+static void f5tFree(void *p) { sqlite3_free(p); }
+
+static void f5tScalarFunc(
+  sqlite3_context *ctx, 
+  int nArg, 
+  sqlite3_value **apArg
+){
+  const char *zText = (const char*)sqlite3_value_text(apArg[0]);
+  int nText = sqlite3_value_bytes(apArg[0]);
+  const char *zTok = (const char*)sqlite3_value_text(apArg[1]);
+  int nTok = sqlite3_value_bytes(apArg[1]);
+  unsigned char *aBuf = 0; 
+  int nBuf = 0;
+
+  assert( nArg==2 );
+
+  if( zTok==0 ) zTok = "";
+  nBuf = nTok + 1 + nText;
+  aBuf = (unsigned char*)sqlite3_malloc(nBuf);
+  if( aBuf==0 ){
+    sqlite3_result_error_nomem(ctx);
+  }else{
+    memcpy(aBuf, zTok, nTok+1);
+    memcpy(&aBuf[nTok+1], zText, nText);
+    sqlite3_result_blob(ctx, aBuf, nBuf, f5tFree);
+    sqlite3_result_subtype(ctx, SQLITE_FTS5_TOKENIZE_SUBTYPE);
+  }
+}
+
 static int SQLITE_TCLAPI f5tRegisterTok(
   void * clientData,
   Tcl_Interp *interp,
@@ -1145,10 +1174,16 @@ static int SQLITE_TCLAPI f5tRegisterTok(
   }
 
   rc = sqlite3Fts5TestRegisterTok(db, pApi);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "fts5tokenize", 2, 
+        SQLITE_UTF8 | SQLITE_RESULT_SUBTYPE, 0, f5tScalarFunc, 0, 0
+    );
+  }
   if( rc!=SQLITE_OK ){
     Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
     return TCL_ERROR;
   }
+  
   return TCL_OK;
 }
 

ext/fts5/fts5_tokenize.c

@@ -1428,16 +1428,6 @@ int sqlite3Fts5TokenizerPattern(
   return FTS5_PATTERN_NONE;
 }
 
-/*
-** Return true if the tokenizer described by p->azArg[] is the trigram
-** tokenizer. This tokenizer needs to be loaded before xBestIndex is
-** called for the first time in order to correctly handle LIKE/GLOB.
-*/
-int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig *p){
-  return (p->nArg>=1 && 0==sqlite3_stricmp(p->azArg[0], "trigram"));
-}
-
-
 /*
 ** Register all built-in tokenizers with FTS5.
 */

ext/fts5/fts5_vocab.c

@@ -64,7 +64,6 @@ struct Fts5VocabCursor {
 
   int nLeTerm;                    /* Size of zLeTerm in bytes */
   char *zLeTerm;                  /* (term <= $zLeTerm) paramater, or NULL */
-  int colUsed;                    /* Copy of sqlite3_index_info.colUsed */
 
   /* These are used by 'col' tables only */
   int iCol;
@@ -91,11 +90,9 @@ struct Fts5VocabCursor {
 /*
 ** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
 */
-#define FTS5_VOCAB_TERM_EQ 0x0100
-#define FTS5_VOCAB_TERM_GE 0x0200
-#define FTS5_VOCAB_TERM_LE 0x0400
-
-#define FTS5_VOCAB_COLUSED_MASK 0xFF
+#define FTS5_VOCAB_TERM_EQ 0x01
+#define FTS5_VOCAB_TERM_GE 0x02
+#define FTS5_VOCAB_TERM_LE 0x04
 
 
 /*
@@ -272,13 +269,11 @@ static int fts5VocabBestIndexMethod(
   int iTermEq = -1;
   int iTermGe = -1;
   int iTermLe = -1;
-  int idxNum = (int)pInfo->colUsed;
+  int idxNum = 0;
   int nArg = 0;
 
   UNUSED_PARAM(pUnused);
 
-  assert( (pInfo->colUsed & FTS5_VOCAB_COLUSED_MASK)==pInfo->colUsed );
-
   for(i=0; i<pInfo->nConstraint; i++){
     struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
     if( p->usable==0 ) continue;
@@ -370,7 +365,7 @@ static int fts5VocabOpenMethod(
       if( rc==SQLITE_OK ){
         pVTab->zErrMsg = sqlite3_mprintf(
             "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
-        );
+            );
         rc = SQLITE_ERROR;
       }
     }else{
@@ -530,19 +525,9 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
 
         switch( pTab->eType ){
           case FTS5_VOCAB_ROW:
-            /* Do not bother counting the number of instances if the "cnt"
-            ** column is not being read (according to colUsed).  */
-            if( eDetail==FTS5_DETAIL_FULL && (pCsr->colUsed & 0x04) ){
-              while( iPos<nPos ){
-                u32 ii;
-                fts5FastGetVarint32(pPos, iPos, ii);
-                if( ii==1 ){ 
-                  /* New column in the position list */
-                  fts5FastGetVarint32(pPos, iPos, ii);
-                }else{
-                  /* An instance - increment pCsr->aCnt[] */
-                  pCsr->aCnt[0]++;
-                }
+            if( eDetail==FTS5_DETAIL_FULL ){
+              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
+                pCsr->aCnt[0]++;
               }
             }
             pCsr->aDoc[0]++;
@@ -640,7 +625,6 @@ static int fts5VocabFilterMethod(
   if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++];
   if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++];
   if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++];
-  pCsr->colUsed = (idxNum & FTS5_VOCAB_COLUSED_MASK);
 
   if( pEq ){
     zTerm = (const char *)sqlite3_value_text(pEq);

ext/fts5/test/fts5_common.tcl

@@ -114,10 +114,6 @@ proc fts5_test_rowcount {cmd} {
   $cmd xRowCount
 }
 
-proc fts5_test_rowid {cmd} {
-  $cmd xRowid
-}
-
 proc test_queryphrase_cb {cnt cmd} {
   upvar $cnt L 
   for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
@@ -171,7 +167,6 @@ proc fts5_aux_test_functions {db} {
     fts5_test_collist
     fts5_test_tokenize
     fts5_test_rowcount
-    fts5_test_rowid
     fts5_test_all
 
     fts5_test_queryphrase

ext/fts5/test/fts5ab.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ab
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ac.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ac
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ad.test

@@ -18,7 +18,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ad
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ae.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ae
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5af.test

@@ -18,7 +18,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5af
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ag.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ag
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ah.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ah
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ai.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ai
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5aj.test

@@ -19,7 +19,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5aj
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ak.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ak
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5al.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5al
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5alter.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5alter
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5auto.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5auto
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5aux.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5aux
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -377,28 +377,4 @@ do_catchsql_test 12.3.3 {
   SELECT fts5_collist(t1, 1) FROM t1('one AND two');
 } {0 1}
 
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 13.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii);
-  INSERT INTO t1 VALUES('a b c'), ('d e f');
-  PRAGMA integrity_check;
-} {ok}
-
-do_catchsql_test 13.2 {
-  SELECT highlight(t1, 0, '[', ']') FROM t1
-} {0 {{a b c} {d e f}}}
-
-do_execsql_test 13.3 {
-  PRAGMA writable_schema = 1;
-  UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)'
-  WHERE name = 't1';
-}
-
-db close
-sqlite3 db test.db
-do_catchsql_test 13.4 {
-  SELECT highlight(t1, 0, '[', ']') FROM t1
-} {1 {SQL logic error}}
-
 finish_test

ext/fts5/test/fts5aux2.test

@@ -1,71 +0,0 @@
-# 2024 June 24
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-# Tests focusing on the auxiliary function APIs.
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5aux
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
-  INSERT INTO x1 VALUES('a b', 'c d');
-  INSERT INTO x1 VALUES('d e', 'a b');
-  INSERT INTO x1 VALUES('a b', 'e f');
-  INSERT INTO x1 VALUES('d e', 'c d');
-}
-
-fts5_aux_test_functions db
-do_execsql_test 1.1 {
-  SELECT fts5_test_all(x1) FROM x1 WHERE rowid=2
-} [list [list {*}{
-  columnsize {2 2}
-  columntext {{d e} {a b}}
-  columntotalsize {8 8}
-  poslist {}
-  tokenize {{d e} {a b}}
-  rowcount 4
-}]]
-
-do_execsql_test 1.2 {
-  SELECT fts5_test_columntext(x1) FROM x1
-} {
-  {{a b} {c d}}
-  {{d e} {a b}}
-  {{a b} {e f}}
-  {{d e} {c d}}
-}
-
-do_execsql_test 1.3 {
-  SELECT fts5_test_rowid(x1) FROM x1
-} {
-  1 2 3 4
-}
-do_execsql_test 1.4 {
-  SELECT fts5_test_phrasecount(x1) FROM x1
-} {
-  0 0 0 0
-}
-do_catchsql_test 1.5 {
-  SELECT fts5_queryphrase(x1, 0) FROM x1
-} {1 SQLITE_RANGE} 
-do_execsql_test 1.6 {
-  SELECT fts5_test_rowcount(x1) FROM x1
-} {4 4 4 4}
-
-
-finish_test

ext/fts5/test/fts5auxdata.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5auxdata
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5bigpl.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5bigpl
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5colset.test

@@ -79,7 +79,7 @@ foreach_detail_mode $::testprefix {
 
   do_catchsql_test 4.1 {
     SELECT * FROM t1 WHERE rowid MATCH 'a'
-  } {1 {no query solution}}
+  } {1 {unable to use function MATCH in the requested context}}
 }
 
 #-------------------------------------------------------------------------

ext/fts5/test/fts5columnsize.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5columnsize
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5config.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5config
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5content.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5content
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5contentless.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5contentless2.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5contentless3.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless3
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5contentless4.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless4
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5contentless5.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless5
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5corrupt.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5corrupt2.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5corrupt3.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt3
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5corrupt4.test

@@ -14,7 +14,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt4
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5corrupt5.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt5
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5corrupt6.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt6
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5corrupt7.test

@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt7
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5dlidx.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5dlidx
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5doclist.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5doclist
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ea.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ea
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5eb.test

@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5eb
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5fault6.test

@@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5fault6
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5faultG.test

@@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5faultG
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5faultH.test

@@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5faultG
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5full.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5full
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5hash.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5hash
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5integrity.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5integrity
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -380,32 +380,5 @@ do_execsql_test 12.3 {
 } {ok}
 
 
-#-------------------------------------------------------------------
-reset_db
-do_execsql_test 13.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii);
-  INSERT INTO t1 VALUES('a b c'), ('d e f');
-  PRAGMA integrity_check;
-} {ok}
-
-db close
-sqlite3 db test.db
-do_catchsql_test 13.2 {
-  PRAGMA integrity_check;
-} {0 ok}
-
-do_execsql_test 13.3 {
-  PRAGMA writable_schema = 1;
-  UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)'
-  WHERE name = 't1';
-}
-
-db close
-sqlite3 db test.db
-breakpoint
-do_catchsql_test 13.4 {
-  PRAGMA integrity_check;
-} {1 {SQL logic error}}
-
 
 finish_test

ext/fts5/test/fts5lastrowid.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5lastrowid
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5matchinfo.test

@@ -517,7 +517,7 @@ fts5_aux_test_functions db
 do_execsql_test 15.3 {
   SELECT fts5_test_all(t1) FROM t1 LIMIT 1;
 } {
-  {columnsize {1 1} columntext {c d} columntotalsize {2 2} poslist {} tokenize {c d} rowcount 2}
+  {columnsize {0 0} columntext {c d} columntotalsize {2 2} poslist {} tokenize {c d} rowcount 2}
 }
 
 finish_test

ext/fts5/test/fts5merge.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5merge
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5misc.test

@@ -535,36 +535,5 @@ do_execsql_test 19.0 {
   COMMIT;
 }
 
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 20.0 {
-  CREATE VIRTUAL TABLE x1 USING fts5(a);
-  INSERT INTO x1(rowid, a) VALUES
-      (1, 'a b c d'),
-      (2, 'x b c d'),
-      (3, 'x y z d'),
-      (4, 'a y c x');
-}
-
-do_execsql_test 20.1 {
-  SELECT rowid FROM x1 WHERE x1 MATCH 'a' AND x1 MATCH 'b';
-} {1}
-
-do_execsql_test 20.2 {
-  SELECT rowid FROM x1 WHERE x1 MATCH 'a' AND x1 MATCH 'y';
-} {4}
-
-do_execsql_test 20.3 {
-  SELECT rowid FROM x1 WHERE x1 MATCH 'a' OR x1 MATCH 'y';
-} {1 4 3}
-
-do_execsql_test 20.4 {
-  SELECT rowid FROM x1 WHERE x1 MATCH 'a' OR (x1 MATCH 'y' AND x1 MATCH 'd');
-} {1 4 3}
-
-do_execsql_test 20.5 {
-  SELECT rowid FROM x1 WHERE x1 MATCH 'z' OR (x1 MATCH 'a' AND x1 MATCH 'd');
-} {3 1}
-
 finish_test
 

ext/fts5/test/fts5near.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5near
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5optimize.test

@@ -14,7 +14,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5optimize
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5optimize2.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5optimize2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5optimize3.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5optimize2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5origintext.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5origintext
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5origintext2.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5origintext2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5origintext3.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5origintext3
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5origintext4.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5origintext4
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5origintext5.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5origintext
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5phrase.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5phrase
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5plan.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5plan
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5porter.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5porter
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5porter2.test

@@ -18,7 +18,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5porter2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5prefix.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5prefix
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5prefix2.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5prefix2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5query.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5query
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5rank.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5rank
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5rebuild.test

@@ -14,7 +14,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5rebuild
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5restart.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5restart
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5rowid.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5rowid
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5savepoint.test

@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5savepoint
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5secure8.test

@@ -42,22 +42,6 @@ do_execsql_test 1.2 {
   PRAGMA integrity_check;
 } {ok}
 
-do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE xyz USING fts5 (
-  	name,
-        content=''
-  );
-
-  INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 1);
-  INSERT INTO xyz (rowid, name) VALUES(1, 'A');
-  INSERT INTO xyz (rowid, name) VALUES(2, 'A');
-  INSERT INTO xyz(xyz, rowid, name) VALUES('delete', 2, 'A');
-}
-
-do_execsql_test 2.1 {
-  pragma quick_check;
-} {ok}
-
 
 
 

ext/fts5/test/fts5securefault.test

@@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5securefault
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 return_if_no_fts5
 
 do_execsql_test 1.0 {

ext/fts5/test/fts5simple.test

@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5simple
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5simple2.test

@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5simple2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5simple3.test

@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5simple3
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5synonym.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5synonym
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5synonym2.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5synonym2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5tokenizer.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5tokenizer
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -300,75 +300,5 @@ set ::flags [list]
 do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {}
 do_test 9.5.2 { set ::flags } {query}
 
-#-------------------------------------------------------------------------
-#
-reset_db
-do_execsql_test 10.1 {
-  CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=unicode61);
-  PRAGMA writable_schema = 1;
-  UPDATE sqlite_schema 
-  SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize="unicode61 error");'
-  WHERE name = 'x1';
-}
-
-db close
-sqlite3 db test.db
-
-do_catchsql_test 10.2 {
-  SELECT * FROM x1('abc');
-} {1 {error in tokenizer constructor}}
-
-do_catchsql_test 10.3 {
-  INSERT INTO x1 VALUES('abc');
-} {1 {error in tokenizer constructor}}
-
-do_execsql_test 10.4 {
-  PRAGMA writable_schema = 1;
-  UPDATE sqlite_schema 
-  SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize="nosuch error");'
-  WHERE name = 'x1';
-}
-
-db close
-sqlite3 db test.db
-
-do_catchsql_test 10.5 {
-  SELECT * FROM x1('abc');
-} {1 {no such tokenizer: nosuch}}
-do_catchsql_test 10.6 {
-  INSERT INTO x1 VALUES('abc');
-} {1 {no such tokenizer: nosuch}}
-
-do_execsql_test 10.7 {
-  DROP TABLE x1;
-  SELECT * FROM sqlite_schema;
-}
-
-reset_db
-do_execsql_test 10.8 {
-  CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=unicode61);
-  INSERT INTO x1 VALUES('a b c'), ('d e f'), ('a b c');
-  CREATE VIRTUAL TABLE x1v USING fts5vocab(x1, row);
-
-  PRAGMA writable_schema = 1;
-  UPDATE sqlite_schema 
-  SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=simplify);'
-  WHERE name = 'x1';
-}
-
-do_execsql_test 10.9 {
-  SELECT * FROM x1v
-} {
-  a 2 2 b 2 2 c 2 2 d 1 1 e 1 1 f 1 1
-}
-
-db close
-sqlite3 db test.db
-
-do_execsql_test 10.10 {
-  SELECT * FROM x1v
-} {
-  a 2 2 b 2 2 c 2 2 d 1 1 e 1 1 f 1 1
-}
 
 finish_test

ext/fts5/test/fts5tokenizer2.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5tokenizer2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5tokenizer3.test

@@ -0,0 +1,158 @@
+# 2024 Apr 16
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focusing on the tokenize-blob functionality.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5tokenizer3
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+set ::constructor_count 0
+proc tcl_create1 {args} { incr ::constructor_count ; return "tcl_tokenize1" }
+proc tcl_create2 {args} { incr ::constructor_count ; return "tcl_tokenize2" }
+sqlite3_fts5_create_tokenizer db tcl1 tcl_create1
+sqlite3_fts5_create_tokenizer db tcl2 tcl_create2
+
+proc tcl_tokenize1 {tflags text} {
+  foreach t [split $text] {
+    sqlite3_fts5_token [string toupper $t] 0 0
+  }
+  return 0
+}
+proc tcl_tokenize2 {tflags text} {
+  foreach t [split $text] {
+    sqlite3_fts5_token [string tolower $t] 0 0
+  }
+  return 0
+}
+
+sqlite3_fts5_register_fts5tokenize db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize = tcl1, content=);
+  CREATE VIRTUAL TABLE v1 USING fts5vocab(x1, instance);
+  INSERT INTO x1 VALUES('Abc Def');
+}
+
+do_execsql_test 1.1 {
+  SELECT hex( fts5tokenize('Ghi Jkl', 'tcl2') );
+} {74636C3200476869204A6B6C}
+
+do_execsql_test 1.2 {
+  INSERT INTO x1 VALUES(fts5tokenize('Ghi Jkl', 'tcl2'));
+}
+
+do_execsql_test 1.3 {
+  SELECT DISTINCT term FROM v1 ORDER BY 1
+} {ABC DEF ghi jkl}
+
+do_execsql_test 1.4 {
+  INSERT INTO x1(x1, rowid, x) VALUES('delete', 2, 'Ghi Jkl');
+  SELECT DISTINCT term FROM v1 ORDER BY 1
+} {ABC DEF ghi jkl}
+
+do_execsql_test 1.5 {
+  INSERT INTO x1(x1, rowid, x) 
+    VALUES('delete', 2, fts5tokenize('Ghi Jkl', 'tcl2'));
+  SELECT DISTINCT term FROM v1 ORDER BY 1
+} {ABC DEF}
+
+do_execsql_test 1.6 {
+  INSERT INTO x1(x1) VALUES('delete-all');
+  INSERT INTO x1 VALUES('Abc Def');
+  INSERT INTO x1 VALUES(fts5tokenize('Ghi Jkl', 'tcl2'));
+} {}
+
+do_execsql_test 1.7 {
+  SELECT rowid FROM x1('Ghi Jkl');
+} {}
+do_execsql_test 1.8 {
+  SELECT rowid FROM x1(fts5tokenize('Abc Def', 'tcl1'));
+} {1}
+do_execsql_test 1.9 {
+  SELECT rowid FROM x1(fts5tokenize('Ghi Jkl', 'tcl2'));
+} {2}
+
+#-------------------------------------------------------------------------
+# Error conditions.
+#
+do_catchsql_test 1.10 {
+  INSERT INTO x1 VALUES(fts5tokenize('Mno Pqr', 'tcl3'));
+} {1 {no such tokenizer: tcl3}}
+do_catchsql_test 1.11 {
+  INSERT INTO x1(x1, rowid, x) 
+    VALUES('delete', 2, fts5tokenize('Mno Pqr', 'tcl3'));
+} {1 {no such tokenizer: tcl3}}
+do_catchsql_test 1.12 {
+  SELECT rowid FROM x1(fts5tokenize('Mno Pqr', 'tcl3'));
+} {1 {no such tokenizer: tcl3}}
+
+do_catchsql_test 1.13 {
+  INSERT INTO x1 VALUES(fts5tokenize('Mno Pqr', 'unicode61 option'));
+} {1 {error in tokenizer constructor}}
+do_catchsql_test 1.14 {
+  INSERT INTO x1(x1, rowid, x) 
+    VALUES('delete', 2, fts5tokenize('Mno Pqr', 'unicode61 option'));
+} {1 {error in tokenizer constructor}}
+do_catchsql_test 1.15 {
+  SELECT rowid FROM x1(fts5tokenize('Mno Pqr', 'unicode61 option'));
+} {1 {error in tokenizer constructor}}
+
+# Check the tokenizer cache has been working.
+#
+do_test 1.16 {
+  set ::constructor_count
+} 2
+proc tcl_create4 {args} { incr ::constructor_count ; return "tcl_tokenize2" }
+sqlite3_fts5_create_tokenizer db tcl4 tcl_create4
+do_execsql_test 1.17 {
+  SELECT rowid FROM x1(fts5tokenize('Mno Pqr', 'tcl4'));
+}
+do_test 1.18 {
+  set ::constructor_count
+} 3
+do_execsql_test 1.19 {
+  SELECT rowid FROM x1(fts5tokenize('Mno Pqr', 'tcl2'));
+}
+do_test 1.20 {
+  set ::constructor_count
+} 3
+
+#-------------------------------------------------------------------------
+# non-contentless tables.
+#
+do_execsql_test 2.0 {
+  CREATE TABLE t1(x);
+  CREATE VIRTUAL TABLE x2 USING fts5(x);
+  CREATE VIRTUAL TABLE x3 USING fts5(x, content=t1);
+}
+
+do_catchsql_test 2.1 {
+  INSERT INTO x2 VALUES( fts5tokenize('hello world', 'tcl2') );
+} {1 {table does not support alternative tokenizers}}
+do_catchsql_test 2.2 {
+  INSERT INTO x3 VALUES( fts5tokenize('hello world', 'tcl2') );
+} {1 {table does not support alternative tokenizers}}
+do_catchsql_test 2.3 {
+  SELECT * FROM x2( fts5tokenize('hello world', 'tcl2') );
+} {1 {table does not support alternative tokenizers}}
+do_catchsql_test 2.4 {
+  SELECT * FROM x3( fts5tokenize('hello world', 'tcl2') );
+} {1 {table does not support alternative tokenizers}}
+
+finish_test
+

ext/fts5/test/fts5trigram2.test

@@ -101,20 +101,9 @@ do_execsql_test 4.0 {
   CREATE VIRTUAL TABLE t4 USING fts5(z, tokenize=trigram);
 } {}
 
+breakpoint
 do_execsql_test 4.1 {
   INSERT INTO t4 VALUES('ABCD');
-  INSERT INTO t4 VALUES('DEFG');
 } {}
 
-db close
-sqlite3 db test.db
-
-do_eqp_test 4.1 {
-  SELECT rowid FROM t4 WHERE z LIKE '%abc%'
-} {VIRTUAL TABLE INDEX 0:L0}
-
-do_execsql_test 4.2 {
-  SELECT rowid FROM t4 WHERE z LIKE '%abc%'
-} {1}
-
 finish_test

ext/fts5/test/fts5ubsan.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ubsan
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5unicode.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5unicode
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5unicode2.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5unicode2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5unicode3.test

@@ -14,7 +14,7 @@
 
 source [file join [file dirname [info script]] fts5_common.tcl]
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5unicode4.test

@@ -14,7 +14,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5unicode4
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return