Prevent tokenize-blobs from being used with non-contentless tables. Fix some other issues with the new code on this branch.

FossilOrigin-Name: 6a640ea4d8938e2b0c73b5aa35d1ce96eea1d1304a26c98dc054e9949d1b7c8c

ext/fts5/fts5.h

@@ -27,6 +27,8 @@
 extern "C" {
 #endif
 
+#define SQLITE_FTS5_TOKENIZE_SUBTYPE ((unsigned int)'T')
+
 /*************************************************************************
 ** CUSTOM AUXILIARY FUNCTIONS
 **

ext/fts5/fts5Int.h

@@ -143,6 +143,18 @@ struct Fts5Colset {
 
 typedef struct Fts5Config Fts5Config;
 
+/*
+** All instantiated tokenizers are stored in a list of the following objects,
+** starting at Fts5Config.pTokList.
+*/
+typedef struct Fts5TokenizerInst Fts5TokenizerInst;
+struct Fts5TokenizerInst {
+  char *zSpec;                    /* Tokenizer specification */
+  Fts5Tokenizer *pTok;
+  fts5_tokenizer *pTokApi;
+  Fts5TokenizerInst *pNext;
+};
+
 /*
 ** An instance of the following structure encodes all information that can
 ** be gleaned from the CREATE VIRTUAL TABLE statement.
@@ -184,6 +196,7 @@ typedef struct Fts5Config Fts5Config;
 */
 struct Fts5Config {
   sqlite3 *db;                    /* Database handle */
+  Fts5Global *pGlobal;            /* Database wide data */
   char *zDb;                      /* Database holding FTS index (e.g. "main") */
   char *zName;                    /* Name of FTS index */
   int nCol;                       /* Number of columns */
@@ -199,8 +212,7 @@ struct Fts5Config {
   int bTokendata;                 /* "tokendata=" option value (dflt==0) */
   int eDetail;                    /* FTS5_DETAIL_XXX value */
   char *zContentExprlist;
-  Fts5Tokenizer *pTok;
-  fts5_tokenizer *pTokApi;
+  Fts5TokenizerInst *pTokList;
   int bLock;                      /* True when table is preparing statement */
   int ePattern;                   /* FTS_PATTERN_XXX constant */
 
@@ -258,6 +270,21 @@ int sqlite3Fts5Tokenize(
   int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
 );
 
+int sqlite3Fts5ConfigFindTokenizer(
+  Fts5Config *pConfig, 
+  const char *z, 
+  Fts5TokenizerInst **ppOut
+);
+
+int sqlite3Fts5SpecTokenize(
+  Fts5Config *pConfig,            /* FTS5 Configuration object */
+  const char *zSpec,              /* Tokenizer specification */
+  int flags,                      /* FTS5_TOKENIZE_* flags */
+  const char *pText, int nText,   /* Text to tokenize */
+  void *pCtx,                     /* Context passed to xToken() */
+  int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
+);
+
 void sqlite3Fts5Dequote(char *z);
 
 /* Load the contents of the %_config table */
@@ -598,11 +625,8 @@ struct Fts5Table {
 };
 
 int sqlite3Fts5GetTokenizer(
-  Fts5Global*, 
-  const char **azArg,
-  int nArg,
   Fts5Config*,
-  char **pzErr
+  const char *zSpec
 );
 
 Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);
@@ -714,6 +738,14 @@ int sqlite3Fts5StorageOptimize(Fts5Storage *p);
 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
 int sqlite3Fts5StorageReset(Fts5Storage *p);
 
+int sqlite3Fts5UnpackTokenizeBlob(
+  Fts5Config *pConfig,
+  sqlite3_value *pVal,
+  Fts5TokenizerInst **ppTok,
+  char **pzText,
+  int *pbDel
+);
+
 /*
 ** End of interface to code in fts5_storage.c.
 **************************************************************************/
@@ -737,6 +769,7 @@ struct Fts5Token {
 /* Parse a MATCH expression. */
 int sqlite3Fts5ExprNew(
   Fts5Config *pConfig, 
+  Fts5TokenizerInst*,
   int bPhraseToAnd,
   int iCol,                       /* Column on LHS of MATCH operator */
   const char *zExpr,

ext/fts5/fts5_config.c

@@ -224,6 +224,38 @@ static int fts5ConfigSetEnum(
   return iVal<0 ? SQLITE_ERROR : SQLITE_OK;
 }
 
+/*
+** Locate a tokenizer instance with a specification matching the second
+** argument. Create a new tokenizer if one can not be found. Return SQLITE_OK
+** if successful, or an SQLite error code otherwise.
+*/
+int sqlite3Fts5ConfigFindTokenizer(
+  Fts5Config *pConfig,            /* Table configuration */
+  const char *z,                  /* Requested tokenizer specification */
+  Fts5TokenizerInst **ppOut       /* OUT: Tokenizer instance */
+){
+  Fts5TokenizerInst *pRet = 0;
+  int rc = SQLITE_OK;
+
+  assert( pConfig->pzErrmsg );
+
+  /* Search for an existing tokenizer that matches this spec */
+  for(pRet=pConfig->pTokList; pRet; pRet=pRet->pNext){
+    if( strcmp(pRet->zSpec, z)==0 ) break;
+  }
+
+  if( pRet==0 ){
+    /* No tokenizer found - create one. */
+    rc = sqlite3Fts5GetTokenizer(pConfig, z);
+    if( rc==SQLITE_OK ){
+      pRet = pConfig->pTokList->pNext;
+    }
+  }
+
+  if( ppOut ) *ppOut = pRet;
+  return rc;
+}
+
 /*
 ** Parse a "special" CREATE VIRTUAL TABLE directive and update
 ** configuration object pConfig as appropriate.
@@ -296,46 +328,15 @@ static int fts5ConfigParseSpecial(
   }
 
   if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
-    const char *p = (const char*)zArg;
-    sqlite3_int64 nArg = strlen(zArg) + 1;
-    char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
-    char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
-    char *pSpace = pDel;
-
-    if( azArg && pSpace ){
-      if( pConfig->pTok ){
-        *pzErr = sqlite3_mprintf("multiple tokenize=... directives");
-        rc = SQLITE_ERROR;
-      }else{
-        for(nArg=0; p && *p; nArg++){
-          const char *p2 = fts5ConfigSkipWhitespace(p);
-          if( *p2=='\'' ){
-            p = fts5ConfigSkipLiteral(p2);
-          }else{
-            p = fts5ConfigSkipBareword(p2);
-          }
-          if( p ){
-            memcpy(pSpace, p2, p-p2);
-            azArg[nArg] = pSpace;
-            sqlite3Fts5Dequote(pSpace);
-            pSpace += (p - p2) + 1;
-            p = fts5ConfigSkipWhitespace(p);
-          }
-        }
-        if( p==0 ){
-          *pzErr = sqlite3_mprintf("parse error in tokenize directive");
-          rc = SQLITE_ERROR;
-        }else{
-          rc = sqlite3Fts5GetTokenizer(pGlobal, 
-              (const char**)azArg, (int)nArg, pConfig,
-              pzErr
-          );
-        }
-      }
+    if( pConfig->pTokList ){
+      *pzErr = sqlite3_mprintf("multiple tokenize=... directives");
+      rc = SQLITE_ERROR;
+    }else{
+      assert( pConfig->pzErrmsg==0 );
+      pConfig->pzErrmsg = pzErr;
+      rc = sqlite3Fts5GetTokenizer(pConfig, zArg);
+      pConfig->pzErrmsg = 0;
     }
-
-    sqlite3_free(azArg);
-    sqlite3_free(pDel);
     return rc;
   }
 
@@ -412,16 +413,6 @@ static int fts5ConfigParseSpecial(
   return SQLITE_ERROR;
 }
 
-/*
-** Allocate an instance of the default tokenizer ("simple") at 
-** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
-** code if an error occurs.
-*/
-static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
-  assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
-  return sqlite3Fts5GetTokenizer(pGlobal, 0, 0, pConfig, 0);
-}
-
 /*
 ** Gobble up the first bareword or quoted word from the input buffer zIn.
 ** Return a pointer to the character immediately following the last in
@@ -555,6 +546,7 @@ int sqlite3Fts5ConfigParse(
   if( pRet==0 ) return SQLITE_NOMEM;
   memset(pRet, 0, sizeof(Fts5Config));
   pRet->db = db;
+  pRet->pGlobal = pGlobal;
   pRet->iCookie = -1;
 
   nByte = nArg * (sizeof(char*) + sizeof(u8));
@@ -643,8 +635,8 @@ int sqlite3Fts5ConfigParse(
   /* If a tokenizer= option was successfully parsed, the tokenizer has
   ** already been allocated. Otherwise, allocate an instance of the default
   ** tokenizer (unicode61) now.  */
-  if( rc==SQLITE_OK && pRet->pTok==0 ){
-    rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
+  if( rc==SQLITE_OK && pRet->pTokList==0 ){
+    rc = sqlite3Fts5GetTokenizer(pRet, 0);
   }
 
   /* If no zContent option was specified, fill in the default values. */
@@ -682,17 +674,30 @@ int sqlite3Fts5ConfigParse(
   return rc;
 }
 
+/*
+** Free all tokenizer instances in the list starting at Fts5Config.pTokList.
+*/
+static void fts5ConfigFreeTokenizers(Fts5Config *pConfig){
+  Fts5TokenizerInst *p = pConfig->pTokList;
+  while( p ){
+    Fts5TokenizerInst *pNext = p->pNext;
+    p->pTokApi->xDelete(p->pTok);
+    sqlite3_free(p);
+    p = pNext;
+  }
+  pConfig->pTokList = 0;
+}
+
 /*
 ** Free the configuration object passed as the only argument.
 */
 void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
   if( pConfig ){
     int i;
-    if( pConfig->pTok ){
-      pConfig->pTokApi->xDelete(pConfig->pTok);
-    }
+    fts5ConfigFreeTokenizers(pConfig);
     sqlite3_free(pConfig->zDb);
     sqlite3_free(pConfig->zName);
+    fts5ConfigFreeTokenizers(pConfig);
     for(i=0; i<pConfig->nCol; i++){
       sqlite3_free(pConfig->azCol[i]);
     }
@@ -765,10 +770,34 @@ int sqlite3Fts5Tokenize(
   void *pCtx,                     /* Context passed to xToken() */
   int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
 ){
-  if( pText==0 ) return SQLITE_OK;
-  return pConfig->pTokApi->xTokenize(
-      pConfig->pTok, pCtx, flags, pText, nText, xToken
-  );
+  if( pText ){
+    Fts5TokenizerInst *p = pConfig->pTokList;
+    return p->pTokApi->xTokenize(p->pTok, pCtx, flags, pText, nText, xToken);
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Like sqlite3Fts5Tokenize(), but using the tokenizer defined by 
+** specification zSpec.
+*/
+int sqlite3Fts5SpecTokenize(
+  Fts5Config *pConfig,            /* FTS5 Configuration object */
+  const char *zSpec,              /* Tokenizer specification */
+  int flags,                      /* FTS5_TOKENIZE_* flags */
+  const char *pText, int nText,   /* Text to tokenize */
+  void *pCtx,                     /* Context passed to xToken() */
+  int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
+){
+  if( pText ){
+    Fts5TokenizerInst *p = pConfig->pTokList;
+    if( zSpec ){
+      int rc = sqlite3Fts5ConfigFindTokenizer(pConfig, zSpec, &p);
+      if( rc!=SQLITE_OK ) return rc;
+    }
+    return p->pTokApi->xTokenize(p->pTok, pCtx, flags, pText, nText, xToken);
+  }
+  return SQLITE_OK;
 }
 
 /*

ext/fts5/fts5_expr.c

@@ -141,6 +141,7 @@ struct Fts5Parse {
   Fts5ExprPhrase **apPhrase;      /* Array of all phrases */
   Fts5ExprNode *pExpr;            /* Result of a successful parse */
   int bPhraseToAnd;               /* Convert "a+b" to "a AND b" */
+  Fts5TokenizerInst *pTok;
 };
 
 /*
@@ -255,6 +256,7 @@ static void fts5ParseFree(void *p){ sqlite3_free(p); }
 
 int sqlite3Fts5ExprNew(
   Fts5Config *pConfig,            /* FTS5 Configuration */
+  Fts5TokenizerInst *pTok,        /* Tokenizer to use, or NULL */
   int bPhraseToAnd,
   int iCol,
   const char *zExpr,              /* Expression text */
@@ -272,6 +274,7 @@ int sqlite3Fts5ExprNew(
   *pzErr = 0;
   memset(&sParse, 0, sizeof(sParse));
   sParse.bPhraseToAnd = bPhraseToAnd;
+  sParse.pTok = pTok ? pTok : pConfig->pTokList;
   pEngine = sqlite3Fts5ParserAlloc(fts5ParseAlloc);
   if( pEngine==0 ){ return SQLITE_NOMEM; }
   sParse.pConfig = pConfig;
@@ -407,7 +410,9 @@ int sqlite3Fts5ExprPattern(
         }
       }
       zExpr[iOut] = '\0';
-      rc = sqlite3Fts5ExprNew(pConfig, bAnd, iCol, zExpr, pp,pConfig->pzErrmsg);
+      rc = sqlite3Fts5ExprNew(
+          pConfig, 0, bAnd, iCol, zExpr, pp,pConfig->pzErrmsg
+      );
     }else{
       *pp = 0;
     }
@@ -1843,11 +1848,12 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
 
   rc = fts5ParseStringFromToken(pToken, &z);
   if( rc==SQLITE_OK ){
+    Fts5TokenizerInst *p = pParse->pTok;
     int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0);
     int n;
     sqlite3Fts5Dequote(z);
     n = (int)strlen(z);
-    rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize);
+    rc = p->pTokApi->xTokenize(p->pTok, &sCtx, flags, z, n, fts5ParseTokenize);
   }
   sqlite3_free(z);
   if( rc || (rc = sCtx.rc) ){
@@ -2777,7 +2783,7 @@ static void fts5ExprFunction(
 
   rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
   if( rc==SQLITE_OK ){
-    rc = sqlite3Fts5ExprNew(pConfig, 0, pConfig->nCol, zExpr, &pExpr, &zErr);
+    rc = sqlite3Fts5ExprNew(pConfig, 0, 0, pConfig->nCol, zExpr, &pExpr, &zErr);
   }
   if( rc==SQLITE_OK ){
     char *zText;

ext/fts5/fts5_main.c

@@ -115,7 +115,6 @@ struct Fts5TokenizerModule {
 struct Fts5FullTable {
   Fts5Table p;                    /* Public class members from fts5Int.h */
   Fts5Storage *pStorage;          /* Document store */
-  Fts5Global *pGlobal;            /* Global (connection wide) data */
   Fts5Cursor *pSortCsr;           /* Sort data from this cursor */
   int iSavepoint;                 /* Successful xSavepoint()+1 */
   
@@ -378,7 +377,6 @@ static int fts5InitVtab(
   }
   if( rc==SQLITE_OK ){
     pTab->p.pConfig = pConfig;
-    pTab->pGlobal = pGlobal;
   }
 
   /* Open the index sub-system */
@@ -693,7 +691,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
 
 static int fts5NewTransaction(Fts5FullTable *pTab){
   Fts5Cursor *pCsr;
-  for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
+  for(pCsr=pTab->p.pConfig->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
     if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK;
   }
   return sqlite3Fts5StorageReset(pTab->pStorage);
@@ -714,7 +712,7 @@ static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
     nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
     pCsr = (Fts5Cursor*)sqlite3_malloc64(nByte);
     if( pCsr ){
-      Fts5Global *pGlobal = pTab->pGlobal;
+      Fts5Global *pGlobal = pConfig->pGlobal;
       memset(pCsr, 0, (size_t)nByte);
       pCsr->aColumnSize = (int*)&pCsr[1];
       pCsr->pNext = pGlobal->pCsr;
@@ -801,7 +799,7 @@ static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
 
     fts5FreeCursorComponents(pCsr);
     /* Remove the cursor from the Fts5Global.pCsr list */
-    for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
+    for(pp=&pTab->p.pConfig->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
     *pp = pCsr->pNext;
 
     sqlite3_free(pCsr);
@@ -854,7 +852,7 @@ static int fts5SorterNext(Fts5Cursor *pCsr){
 */
 static void fts5TripCursors(Fts5FullTable *pTab){
   Fts5Cursor *pCsr;
-  for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
+  for(pCsr=pTab->p.pConfig->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
     if( pCsr->ePlan==FTS5_PLAN_MATCH
      && pCsr->base.pVtab==(sqlite3_vtab*)pTab 
     ){
@@ -1106,7 +1104,7 @@ static int fts5SpecialMatch(
 static Fts5Auxiliary *fts5FindAuxiliary(Fts5FullTable *pTab, const char *zName){
   Fts5Auxiliary *pAux;
 
-  for(pAux=pTab->pGlobal->pAux; pAux; pAux=pAux->pNext){
+  for(pAux=pTab->p.pConfig->pGlobal->pAux; pAux; pAux=pAux->pNext){
     if( sqlite3_stricmp(zName, pAux->zFunc)==0 ) return pAux;
   }
 
@@ -1277,7 +1275,15 @@ static int fts5FilterMethod(
         pRank = apVal[i];
         break;
       case 'M': {
-        const char *zText = (const char*)sqlite3_value_text(apVal[i]);
+        Fts5TokenizerInst *pInst = 0;
+        char *zText = 0;
+        int bDel = 0;
+
+        rc = sqlite3Fts5UnpackTokenizeBlob(
+            pConfig, apVal[i], &pInst, &zText, &bDel
+        );
+        if( rc!=SQLITE_OK ) goto filter_out;
+
         if( zText==0 ) zText = "";
         iCol = 0;
         do{
@@ -1290,17 +1296,19 @@ static int fts5FilterMethod(
           ** indicates that the MATCH expression is not a full text query,
           ** but a request for an internal parameter.  */
           rc = fts5SpecialMatch(pTab, pCsr, &zText[1]);
+          if( bDel ) sqlite3_free(zText);
           goto filter_out;
         }else{
           char **pzErr = &pTab->p.base.zErrMsg;
-          rc = sqlite3Fts5ExprNew(pConfig, 0, iCol, zText, &pExpr, pzErr);
+          rc = sqlite3Fts5ExprNew(pConfig, pInst, 0, iCol, zText, &pExpr,pzErr);
           if( rc==SQLITE_OK ){
             rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr);
             pExpr = 0;
           }
-          if( rc!=SQLITE_OK ) goto filter_out;
         }
 
+        if( bDel ) sqlite3_free(zText);
+        if( rc!=SQLITE_OK ) goto filter_out;
         break;
       }
       case 'L':
@@ -2860,40 +2868,98 @@ static int fts5FindTokenizer(
   return rc;
 }
 
+/*
+** Add a tokenizer with specification zSpec to the list at Fts5Config.pTokList.
+** Return SQLITE_OK if successful, or an SQLite error code otherwise.
+*/
 int sqlite3Fts5GetTokenizer(
-  Fts5Global *pGlobal, 
-  const char **azArg,
-  int nArg,
   Fts5Config *pConfig,
-  char **pzErr
+  const char *zSpec
 ){
-  Fts5TokenizerModule *pMod;
   int rc = SQLITE_OK;
+  char **pzErr = pConfig->pzErrmsg;
+  const char **azArg = 0;
+  char *pDel = 0;
+  sqlite3_int64 nArg = 0;
 
-  pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]);
-  if( pMod==0 ){
-    assert( nArg>0 );
-    rc = SQLITE_ERROR;
-    *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
-  }else{
-    rc = pMod->x.xCreate(
-        pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->pTok
-    );
-    pConfig->pTokApi = &pMod->x;
-    if( rc!=SQLITE_OK ){
-      if( pzErr ) *pzErr = sqlite3_mprintf("error in tokenizer constructor");
-    }else{
-      pConfig->ePattern = sqlite3Fts5TokenizerPattern(
-          pMod->x.xCreate, pConfig->pTok
-      );
+  assert( pzErr || (zSpec==0 && pConfig->pTokList==0) );
+  if( zSpec ){
+    const char *p = (const char*)zSpec;
+    char *pSpace = 0;
+
+    nArg = strlen(zSpec) + 1;
+    pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
+    pSpace = pDel;
+    azArg = (const char**)sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
+
+    if( azArg && pSpace ){
+      for(nArg=0; p && *p; nArg++){
+        const char *p2 = fts5ConfigSkipWhitespace(p);
+        if( *p2=='\'' ){
+          p = fts5ConfigSkipLiteral(p2);
+        }else{
+          p = fts5ConfigSkipBareword(p2);
+        }
+        if( p ){
+          memcpy(pSpace, p2, p-p2);
+          azArg[nArg] = pSpace;
+          sqlite3Fts5Dequote(pSpace);
+          pSpace += (p - p2) + 1;
+          p = fts5ConfigSkipWhitespace(p);
+        }
+      }
+      if( p==0 ){
+        *pzErr= sqlite3_mprintf("parse error in tokenize directive");
+        rc = SQLITE_ERROR;
+      }
     }
   }
 
-  if( rc!=SQLITE_OK ){
-    pConfig->pTokApi = 0;
-    pConfig->pTok = 0;
+  if( rc==SQLITE_OK ){
+    Fts5TokenizerModule *pMod;
+    pMod = fts5LocateTokenizer(pConfig->pGlobal, nArg==0 ? 0 : azArg[0]);
+    if( pMod==0 ){
+      assert( nArg>0 );
+      rc = SQLITE_ERROR;
+      *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
+    }else{
+      int nSpec = zSpec ? strlen(zSpec) + 1 : 0;
+      int nByte = sizeof(Fts5TokenizerInst) + nSpec;
+      Fts5TokenizerInst *pNew = sqlite3Fts5MallocZero(&rc, nByte);
+      if( pNew ){
+        if( zSpec ){
+          pNew->zSpec = (char*)&pNew[1];
+          memcpy(pNew->zSpec, zSpec, nSpec);
+        }
+        rc = pMod->x.xCreate(
+            pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pNew->pTok
+        );
+        pNew->pTokApi = &pMod->x;
+        if( rc!=SQLITE_OK ){
+          if( pzErr ){
+            *pzErr = sqlite3_mprintf("error in tokenizer constructor");
+          }
+        }else if( pConfig->pTokList==0 ){
+          pConfig->ePattern = sqlite3Fts5TokenizerPattern(
+              pMod->x.xCreate, pNew->pTok
+          );
+        }
+      }
+      if( rc==SQLITE_OK ){
+        if( pConfig->pTokList ){
+          pNew->pNext = pConfig->pTokList->pNext;
+          pConfig->pTokList->pNext = pNew;
+        }else{
+          pConfig->pTokList = pNew;
+        }
+      }else{
+        sqlite3_free(pNew);
+      }
+    }
   }
 
+  sqlite3_free(azArg);
+  sqlite3_free(pDel);
   return rc;
 }
 

ext/fts5/fts5_storage.c

@@ -399,6 +399,69 @@ static int fts5StorageInsertCallback(
   return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
 }
 
+/*
+** If the value passed as the third argument is a tokenizer blob, and the
+** Fts5Config object indicates that the table is a contentless-table, 
+** return non-zero.
+**
+** Or, if the value passed as the third argument is a tokenizer blob but
+** the table is not a contentless table, set *pRc to SQLITE_ERROR and leave
+** an error message in the Fts5Config object. Return 0 in this case.
+**
+** Finally, if the value is not a tokenizer blob, return 0.
+*/
+static int fts5IsTokenizeBlob(
+  int *pRc, 
+  Fts5Config *pConfig, 
+  sqlite3_value *pVal
+){
+  assert( *pRc==SQLITE_OK );
+  if( sqlite3_value_subtype(pVal)==SQLITE_FTS5_TOKENIZE_SUBTYPE 
+   && sqlite3_value_type(pVal)==SQLITE_BLOB 
+  ){
+    if( pConfig->eContent==FTS5_CONTENT_NONE ) return 1;
+
+    *pRc = SQLITE_ERROR;
+    *pConfig->pzErrmsg = sqlite3_mprintf(
+        "table does not support alternative tokenizers"
+    );
+  }
+  return 0;
+}
+
+/*
+** Value pVal is guaranteed to be a tokenize-blob. This function unpacks
+** the blob and returns a pointer to the nul-terminated tokenizer
+** specification. It also sets output parameter (*pzT) to point to the
+** start of the utf-8 text value (not nul-terminated) and (*pnT) to the
+** number of valid bytes in this buffer.
+*/
+static const char *fts5UnpackTokenizeBlob(
+  sqlite3_value *pVal,
+  const char **pzT,
+  int *pnT
+){
+  const u8 *pBlob = sqlite3_value_blob(pVal);
+  int nBlob = sqlite3_value_bytes(pVal);
+  int ii;
+
+  assert( sqlite3_value_subtype(pVal)==SQLITE_FTS5_TOKENIZE_SUBTYPE );
+  assert( sqlite3_value_type(pVal)==SQLITE_BLOB );
+
+  for(ii=0; pBlob[ii]; ii++){
+    if( ii==nBlob ){
+      *pzT = 0;
+      *pnT = 0;
+      return 0;
+    }
+  }
+
+  *pzT = (const char*)&pBlob[ii+1];
+  *pnT = nBlob - ii - 1;
+  return (const char*)pBlob;
+}
+
+
 /*
 ** If a row with rowid iDel is present in the %_content table, add the
 ** delete-markers to the FTS index necessary to delete it. Do not actually
@@ -429,26 +492,34 @@ static int fts5StorageDeleteFromIndex(
   ctx.iCol = -1;
   for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
     if( pConfig->abUnindexed[iCol-1]==0 ){
-      const char *zText;
-      int nText;
+      const char *zText = 0;
+      const char *zTok = 0;
+      int nText = 0;
       assert( pSeek==0 || apVal==0 );
       assert( pSeek!=0 || apVal!=0 );
       if( pSeek ){
         zText = (const char*)sqlite3_column_text(pSeek, iCol);
         nText = sqlite3_column_bytes(pSeek, iCol);
       }else if( ALWAYS(apVal) ){
-        zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
-        nText = sqlite3_value_bytes(apVal[iCol-1]);
+        sqlite3_value *pVal = apVal[iCol-1];
+        if( fts5IsTokenizeBlob(&rc, pConfig, pVal) ){
+          zTok = fts5UnpackTokenizeBlob(pVal, &zText, &nText);
+        }else{
+          zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
+          nText = sqlite3_value_bytes(apVal[iCol-1]);
+        }
       }else{
         continue;
       }
       ctx.szCol = 0;
-      rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
-          zText, nText, (void*)&ctx, fts5StorageInsertCallback
-      );
-      p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
-      if( p->aTotalSize[iCol-1]<0 ){
-        rc = FTS5_CORRUPT;
+      if( rc==SQLITE_OK ){
+        rc = sqlite3Fts5SpecTokenize(pConfig, zTok, FTS5_TOKENIZE_DOCUMENT, 
+            zText, nText, (void*)&ctx, fts5StorageInsertCallback
+        );
+        p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
+        if( p->aTotalSize[iCol-1]<0 ){
+          rc = FTS5_CORRUPT;
+        }
       }
     }
   }
@@ -752,6 +823,42 @@ static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
   return rc;
 }
 
+/*
+** This function is used to extract text from an sqlite3_value to use
+** as an fts5 query string. It also finds the required tokenizer to use
+** for tokenizing query terms.
+**
+** If successful, SQLITE_OK is returned, output variable (*ppTok) is set
+** to point to the required tokenizer instance, (*pzText) points to a
+** nul-terminated buffer containing the query string as utf-8 text, and
+** (*pbDel) is set to true if the caller must sqlite3_free(*pzText) at
+** some point in the future. Or, if an error occurs, an SQLite error
+** code is returned.
+*/
+int sqlite3Fts5UnpackTokenizeBlob(
+  Fts5Config *pConfig,
+  sqlite3_value *pVal,
+  Fts5TokenizerInst **ppTok,
+  char **pzText,
+  int *pbDel                      /* OUT: Set to true if sqlite3_free() req. */
+){
+  int rc = SQLITE_OK;
+  if( fts5IsTokenizeBlob(&rc, pConfig, pVal) ){
+    const char *zTok = 0;
+    const char *zText = 0;
+    int nText = 0;
+    zTok = fts5UnpackTokenizeBlob(pVal, &zText, &nText);
+    rc = sqlite3Fts5ConfigFindTokenizer(pConfig, zTok, ppTok);
+    *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, zText);
+    *pbDel = 1;
+  }else{
+    *pzText = (char*)sqlite3_value_text(pVal);
+    *pbDel = 0;
+    *ppTok = pConfig->pTokList;
+  }
+  return rc;
+}
+
 /*
 ** Insert a new row into the FTS content table.
 */
@@ -775,7 +882,15 @@ int sqlite3Fts5StorageContentInsert(
     int i;                        /* Counter variable */
     rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
     for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
-      rc = sqlite3_bind_value(pInsert, i, apVal[i]);
+      sqlite3_value *pVal = apVal[i];
+      if( fts5IsTokenizeBlob(&rc, pConfig, pVal) ){
+        const char *zT = 0;
+        int nT = 0;
+        fts5UnpackTokenizeBlob(pVal, &zT, &nT);
+        rc = sqlite3_bind_text(pInsert, i, zT, nT, SQLITE_STATIC);
+      }else if( rc==SQLITE_OK ){
+        rc = sqlite3_bind_value(pInsert, i, apVal[i]);
+      }
     }
     if( rc==SQLITE_OK ){
       sqlite3_step(pInsert);
@@ -810,14 +925,23 @@ int sqlite3Fts5StorageIndexInsert(
   for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
     ctx.szCol = 0;
     if( pConfig->abUnindexed[ctx.iCol]==0 ){
-      const char *zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]);
-      int nText = sqlite3_value_bytes(apVal[ctx.iCol+2]);
-      rc = sqlite3Fts5Tokenize(pConfig, 
-          FTS5_TOKENIZE_DOCUMENT,
-          zText, nText,
-          (void*)&ctx,
-          fts5StorageInsertCallback
-      );
+      sqlite3_value *pVal = apVal[ctx.iCol+2];
+      const char *zText = 0;
+      const char *zTok = 0;
+      int nText = 0;
+
+      if( fts5IsTokenizeBlob(&rc, pConfig, pVal) ){
+        zTok = fts5UnpackTokenizeBlob(pVal, &zText, &nText);
+      }else{
+        zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]);
+        nText = sqlite3_value_bytes(apVal[ctx.iCol+2]);
+      }
+      if( rc==SQLITE_OK ){
+        rc = sqlite3Fts5SpecTokenize(pConfig, 
+            zTok, FTS5_TOKENIZE_DOCUMENT, zText, nText, 
+            (void*)&ctx, fts5StorageInsertCallback
+        );
+      }
     }
     sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
     p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;

ext/fts5/fts5_tcl.c

@@ -1126,6 +1126,35 @@ static int SQLITE_TCLAPI f5tRegisterMatchinfo(
   return TCL_OK;
 }
 
+static void f5tFree(void *p) { sqlite3_free(p); }
+
+static void f5tScalarFunc(
+  sqlite3_context *ctx, 
+  int nArg, 
+  sqlite3_value **apArg
+){
+  const char *zText = (const char*)sqlite3_value_text(apArg[0]);
+  int nText = sqlite3_value_bytes(apArg[0]);
+  const char *zTok = (const char*)sqlite3_value_text(apArg[1]);
+  int nTok = sqlite3_value_bytes(apArg[1]);
+  unsigned char *aBuf = 0; 
+  int nBuf = 0;
+
+  assert( nArg==2 );
+
+  if( zTok==0 ) zTok = "";
+  nBuf = nTok + 1 + nText;
+  aBuf = (unsigned char*)sqlite3_malloc(nBuf);
+  if( aBuf==0 ){
+    sqlite3_result_error_nomem(ctx);
+  }else{
+    memcpy(aBuf, zTok, nTok+1);
+    memcpy(&aBuf[nTok+1], zText, nText);
+    sqlite3_result_blob(ctx, aBuf, nBuf, f5tFree);
+    sqlite3_result_subtype(ctx, SQLITE_FTS5_TOKENIZE_SUBTYPE);
+  }
+}
+
 static int SQLITE_TCLAPI f5tRegisterTok(
   void * clientData,
   Tcl_Interp *interp,
@@ -1145,10 +1174,16 @@ static int SQLITE_TCLAPI f5tRegisterTok(
   }
 
   rc = sqlite3Fts5TestRegisterTok(db, pApi);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "fts5tokenize", 2, 
+        SQLITE_UTF8 | SQLITE_RESULT_SUBTYPE, 0, f5tScalarFunc, 0, 0
+    );
+  }
   if( rc!=SQLITE_OK ){
     Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
     return TCL_ERROR;
   }
+  
   return TCL_OK;
 }
 

ext/fts5/test/fts5tokenizer3.test

@@ -0,0 +1,158 @@
+# 2024 Apr 16
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focusing on the tokenize-blob functionality.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5tokenizer3
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+set ::constructor_count 0
+proc tcl_create1 {args} { incr ::constructor_count ; return "tcl_tokenize1" }
+proc tcl_create2 {args} { incr ::constructor_count ; return "tcl_tokenize2" }
+sqlite3_fts5_create_tokenizer db tcl1 tcl_create1
+sqlite3_fts5_create_tokenizer db tcl2 tcl_create2
+
+proc tcl_tokenize1 {tflags text} {
+  foreach t [split $text] {
+    sqlite3_fts5_token [string toupper $t] 0 0
+  }
+  return 0
+}
+proc tcl_tokenize2 {tflags text} {
+  foreach t [split $text] {
+    sqlite3_fts5_token [string tolower $t] 0 0
+  }
+  return 0
+}
+
+sqlite3_fts5_register_fts5tokenize db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize = tcl1, content=);
+  CREATE VIRTUAL TABLE v1 USING fts5vocab(x1, instance);
+  INSERT INTO x1 VALUES('Abc Def');
+}
+
+do_execsql_test 1.1 {
+  SELECT hex( fts5tokenize('Ghi Jkl', 'tcl2') );
+} {74636C3200476869204A6B6C}
+
+do_execsql_test 1.2 {
+  INSERT INTO x1 VALUES(fts5tokenize('Ghi Jkl', 'tcl2'));
+}
+
+do_execsql_test 1.3 {
+  SELECT DISTINCT term FROM v1 ORDER BY 1
+} {ABC DEF ghi jkl}
+
+do_execsql_test 1.4 {
+  INSERT INTO x1(x1, rowid, x) VALUES('delete', 2, 'Ghi Jkl');
+  SELECT DISTINCT term FROM v1 ORDER BY 1
+} {ABC DEF ghi jkl}
+
+do_execsql_test 1.5 {
+  INSERT INTO x1(x1, rowid, x) 
+    VALUES('delete', 2, fts5tokenize('Ghi Jkl', 'tcl2'));
+  SELECT DISTINCT term FROM v1 ORDER BY 1
+} {ABC DEF}
+
+do_execsql_test 1.6 {
+  INSERT INTO x1(x1) VALUES('delete-all');
+  INSERT INTO x1 VALUES('Abc Def');
+  INSERT INTO x1 VALUES(fts5tokenize('Ghi Jkl', 'tcl2'));
+} {}
+
+do_execsql_test 1.7 {
+  SELECT rowid FROM x1('Ghi Jkl');
+} {}
+do_execsql_test 1.8 {
+  SELECT rowid FROM x1(fts5tokenize('Abc Def', 'tcl1'));
+} {1}
+do_execsql_test 1.9 {
+  SELECT rowid FROM x1(fts5tokenize('Ghi Jkl', 'tcl2'));
+} {2}
+
+#-------------------------------------------------------------------------
+# Error conditions.
+#
+do_catchsql_test 1.10 {
+  INSERT INTO x1 VALUES(fts5tokenize('Mno Pqr', 'tcl3'));
+} {1 {no such tokenizer: tcl3}}
+do_catchsql_test 1.11 {
+  INSERT INTO x1(x1, rowid, x) 
+    VALUES('delete', 2, fts5tokenize('Mno Pqr', 'tcl3'));
+} {1 {no such tokenizer: tcl3}}
+do_catchsql_test 1.12 {
+  SELECT rowid FROM x1(fts5tokenize('Mno Pqr', 'tcl3'));
+} {1 {no such tokenizer: tcl3}}
+
+do_catchsql_test 1.13 {
+  INSERT INTO x1 VALUES(fts5tokenize('Mno Pqr', 'unicode61 option'));
+} {1 {error in tokenizer constructor}}
+do_catchsql_test 1.14 {
+  INSERT INTO x1(x1, rowid, x) 
+    VALUES('delete', 2, fts5tokenize('Mno Pqr', 'unicode61 option'));
+} {1 {error in tokenizer constructor}}
+do_catchsql_test 1.15 {
+  SELECT rowid FROM x1(fts5tokenize('Mno Pqr', 'unicode61 option'));
+} {1 {error in tokenizer constructor}}
+
+# Check the tokenizer cache has been working.
+#
+do_test 1.16 {
+  set ::constructor_count
+} 2
+proc tcl_create4 {args} { incr ::constructor_count ; return "tcl_tokenize2" }
+sqlite3_fts5_create_tokenizer db tcl4 tcl_create4
+do_execsql_test 1.17 {
+  SELECT rowid FROM x1(fts5tokenize('Mno Pqr', 'tcl4'));
+}
+do_test 1.18 {
+  set ::constructor_count
+} 3
+do_execsql_test 1.19 {
+  SELECT rowid FROM x1(fts5tokenize('Mno Pqr', 'tcl2'));
+}
+do_test 1.20 {
+  set ::constructor_count
+} 3
+
+#-------------------------------------------------------------------------
+# non-contentless tables.
+#
+do_execsql_test 2.0 {
+  CREATE TABLE t1(x);
+  CREATE VIRTUAL TABLE x2 USING fts5(x);
+  CREATE VIRTUAL TABLE x3 USING fts5(x, content=t1);
+}
+
+do_catchsql_test 2.1 {
+  INSERT INTO x2 VALUES( fts5tokenize('hello world', 'tcl2') );
+} {1 {table does not support alternative tokenizers}}
+do_catchsql_test 2.2 {
+  INSERT INTO x3 VALUES( fts5tokenize('hello world', 'tcl2') );
+} {1 {table does not support alternative tokenizers}}
+do_catchsql_test 2.3 {
+  SELECT * FROM x2( fts5tokenize('hello world', 'tcl2') );
+} {1 {table does not support alternative tokenizers}}
+do_catchsql_test 2.4 {
+  SELECT * FROM x3( fts5tokenize('hello world', 'tcl2') );
+} {1 {table does not support alternative tokenizers}}
+
+finish_test
+

manifest

@@ -1,5 +1,5 @@
-C If\sa\sbuild\sfails\sin\stestrunner.tcl,\sdo\snot\sattempt\sto\srun\sthe\sjobs\sthat\ndepend\son\sthat\sbuild.\s\sInstead,\sreport\sthose\sjobs\sas\shaving\sbeen\sskipped.
-D 2024-04-12T18:46:34.309
+C Prevent\stokenize-blobs\sfrom\sbeing\sused\swith\snon-contentless\stables.\sFix\ssome\sother\sissues\swith\sthe\snew\scode\son\sthis\sbranch.
+D 2024-04-17T19:48:41.847
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,17 +92,17 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e
-F ext/fts5/fts5.h 8856e11a5f0269cd346754cea0765efe8089635b80cad3222e8bfdb08cd5348a
-F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880e9834
+F ext/fts5/fts5.h e701ea20480be693f2b50ab314ec4d002bd9b97cd89636427ed1528c690107ae
+F ext/fts5/fts5Int.h 098b3fd928d10035e9b52756affe6315fe337abfbc19e80ea33d1db07d4f5f7a
 F ext/fts5/fts5_aux.c 4584e88878e54828bf7d4d0d83deedd232ec60628b7731be02bad6adb62304b1
 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09
-F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf
-F ext/fts5/fts5_expr.c e91156ebdcc08d837f4f324168f69f3c0d7fdef0e521fd561efb48ef3297b696
+F ext/fts5/fts5_config.c fe565c6a12d6897053a5ab7b0cc6a0691c668103e3c3f1de8dec5491a72316fb
+F ext/fts5/fts5_expr.c f1e9110062a9ff63007431d0af1b1506cca3e5f79e1b2f2dc47795b9e98d4b13
 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
 F ext/fts5/fts5_index.c ee0f4d50bc0c58a7c5ef7d645e7e38e1e59315b8ea9d722ae00c5f949ee65379
-F ext/fts5/fts5_main.c d68bd9533d5a638b7f6fae61c3cb0a15257dcdcccedaf3d0b3c9f55940c85048
-F ext/fts5/fts5_storage.c f9e31b0d155e9b2c92d5d3a09ad7a56b937fbf1c7f962e10f4ca6281349f3934
-F ext/fts5/fts5_tcl.c fdf7e2bb9a9186cfcaf2d2ce11d338309342b7a7593c2812bc54455db53da5d2
+F ext/fts5/fts5_main.c 86b2c807711fc6eef3c1cf3e558093669093bf91a2014bfe5b71be8ad1ea41cb
+F ext/fts5/fts5_storage.c 19fc854c3fad12e3f79ed3608b944a07fb41ffd50af493a1f521cee3a35af192
+F ext/fts5/fts5_tcl.c fd485d0fb56f2c42885e68c74dd53c594a4761af6088617ce120804a6a5aca82
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b
 F ext/fts5/fts5_tokenize.c 83cfcede3898001cab84432a36ce1503e3080cf9b1c682b022ec82e267ea4c13
@@ -229,6 +229,7 @@ F ext/fts5/test/fts5tok1.test 1f7817499f5971450d8c4a652114b3d833393c8134e32422d0
 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
 F ext/fts5/test/fts5tokenizer.test ac3c9112b263a639fb0508ae73a3ee886bf4866d2153771a8e8a20c721305a43
 F ext/fts5/test/fts5tokenizer2.test cb5428c7cfb3b6a74b7adfcde65506e329112003e8dffa7501d01c2d18d02569
+F ext/fts5/test/fts5tokenizer3.test 507d50608b61031f72f8cf3c752ea8db51d3d67ae99ebe6f0d191e58455dc19c
 F ext/fts5/test/fts5trigram.test 6c4e37864f3e7d90673db5563d9736d7e40080ab94d10ebdffa94c1b77941da0
 F ext/fts5/test/fts5trigram2.test 9fe4207f8a4241747aff1005258b564958588d21bfd240d6cd4c2e955d31c156
 F ext/fts5/test/fts5ubsan.test 783d5a8d13ebfa169e634940228db54540780e3ba7a87ad1e4510e61440bf64b
@@ -2184,8 +2185,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5dede50d9e7b6942df9f7b00fbfeaa2103c36c5da01d63d88136fb0ef4b7d26d
-R 4f3eb54a5cce9aa23292ec96983abdb3
-U drh
-Z 1648e8675f24aa01764212367c0fba7b
+P c2f9d1259cc094ad1d3e5e0a50b262a248915743fed3b1a730a1d9f0f845f48b
+R 1ef80b3031ba5f310c6dd6728c192f80
+U dan
+Z 1752c8f42f1640cfd39a63e89bc136e9
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-b40580be719a129ecd1aa3c69d1086c967d063920fdd48617c864e73c059abc1
+6a640ea4d8938e2b0c73b5aa35d1ce96eea1d1304a26c98dc054e9949d1b7c8c