Modify the fts3tokenize table implementation so that it does not use the SQL function fts3_tokenizer. The user may have installed an authorizer callback that prohibits this.

FossilOrigin-Name: 0ba67b64de258883e4c43db09e131bb67083855e

ext/fts3/fts3.c

@@ -3591,9 +3591,6 @@ int sqlite3Fts3Init(sqlite3 *db){
   rc = sqlite3Fts3InitAux(db);
   if( rc!=SQLITE_OK ) return rc;
 
-  rc = sqlite3Fts3InitTok(db);
-  if( rc!=SQLITE_OK ) return rc;
-
   sqlite3Fts3SimpleTokenizerModule(&pSimple);
   sqlite3Fts3PorterTokenizerModule(&pPorter);
 
@@ -3647,9 +3644,13 @@ int sqlite3Fts3Init(sqlite3 *db){
           db, "fts4", &fts3Module, (void *)pHash, 0
       );
     }
+    if( rc==SQLITE_OK ){
+      rc = sqlite3Fts3InitTok(db, (void *)pHash);
+    }
     return rc;
   }
 
+
   /* An error has occurred. Delete the hash table and return the error code. */
   assert( rc!=SQLITE_OK );
   if( pHash ){

ext/fts3/fts3Int.h

@@ -550,7 +550,7 @@ int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
 int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
 
 /* fts3_tokenize_vtab.c */
-int sqlite3Fts3InitTok(sqlite3*);
+int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *);
 
 /* fts3_unicode2.c (functions generated by parsing unicode text files) */
 #ifdef SQLITE_ENABLE_FTS4_UNICODE61

ext/fts3/fts3_tokenize_vtab.c

@@ -76,27 +76,22 @@ struct Fts3tokCursor {
 */
 static int fts3tokQueryTokenizer(
   sqlite3 *db,
+  Fts3Hash *pHash,
   const char *zName,
-  const sqlite3_tokenizer_module **pp
+  const sqlite3_tokenizer_module **pp,
+  char **pzErr
 ){
-  int rc;
-  sqlite3_stmt *pStmt;
-  const char *zSql = "SELECT fts3_tokenizer(?)";
+  sqlite3_tokenizer_module *p;
+  int nName = strlen(zName);
 
-  *pp = 0;
-  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-  if( rc!=SQLITE_OK ){
-    return rc;
+  p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
+  if( !p ){
+    *pzErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
+    return SQLITE_ERROR;
   }
 
-  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
-  if( SQLITE_ROW==sqlite3_step(pStmt) ){
-    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
-      memcpy((void*)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
-    }
-  }
-
-  return sqlite3_finalize(pStmt);
+  *pp = p;
+  return SQLITE_OK;
 }
 
 /*
@@ -163,7 +158,7 @@ static int fts3tokDequoteArray(
 */
 static int fts3tokConnectMethod(
   sqlite3 *db,                    /* Database connection */
-  void *pUnused,                  /* Unused */
+  void *pHash,                    /* Hash table of tokenizers */
   int argc,                       /* Number of elements in argv array */
   const char * const *argv,       /* xCreate/xConnect argument array */
   sqlite3_vtab **ppVtab,          /* OUT: New sqlite3_vtab object */
@@ -175,7 +170,6 @@ static int fts3tokConnectMethod(
   int rc;
   char **azDequote = 0;
   int nDequote;
-  UNUSED_PARAMETER(pUnused);
 
   rc = sqlite3_declare_vtab(db, FTS3_TOK_SCHEMA);
   if( rc!=SQLITE_OK ) return rc;
@@ -190,14 +184,11 @@ static int fts3tokConnectMethod(
     }else{
       zModule = azDequote[0];
     }
-    rc = fts3tokQueryTokenizer(db, zModule, &pMod);
+    rc = fts3tokQueryTokenizer(db, (Fts3Hash *)pHash, zModule, &pMod, pzErr);
   }
 
-  if( rc!=SQLITE_OK ){
-    *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
-  }else if( pMod==0 ){
-    rc = SQLITE_ERROR;
-  }else{
+  assert( (rc==SQLITE_OK)==(pMod!=0) );
+  if( rc==SQLITE_OK ){
     const char * const *azArg = (const char * const *)&azDequote[1];
     rc = pMod->xCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok);
   }
@@ -429,7 +420,7 @@ static int fts3tokRowidMethod(
 ** Register the fts3tok module with database connection db. Return SQLITE_OK
 ** if successful or an error code if sqlite3_create_module() fails.
 */
-int sqlite3Fts3InitTok(sqlite3 *db){
+int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){
   static const sqlite3_module fts3tok_module = {
      0,                           /* iVersion      */
      fts3tokConnectMethod,        /* xCreate       */
@@ -457,7 +448,7 @@ int sqlite3Fts3InitTok(sqlite3 *db){
   };
   int rc;                         /* Return code */
 
-  rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, 0);
+  rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash);
   return rc;
 }
 

manifest

@@ -1,5 +1,5 @@
-C In\sthe\scommand-line\sshell,\soutput\sthe\selements\sof\sthe\sschema\sin\sthe\sorder\nthat\sthey\sappear\sin\sthe\ssqlite_master\stable.\s\sThis\sis\snecessary\sin\ssome\ncases\sto\smake\sthe\sschema\sparsable.
-D 2013-05-06T21:01:06.414
+C Modify\sthe\sfts3tokenize\stable\simplementation\sso\sthat\sit\sdoes\snot\suse\sthe\sSQL\sfunction\sfts3_tokenizer.\sThe\suser\smay\shave\sinstalled\san\sauthorizer\scallback\sthat\sprohibits\sthis.
+D 2013-05-07T12:16:48.845
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -55,9 +55,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 5c3d44d16701cc4bc81ebf0bb9d5bff136d42de0
+F ext/fts3/fts3.c 4bc160e6ff9ab5456b600f389f8941485ea5082f
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h 23ea0a2bb7258d2539376ed60220cce28ba25765
+F ext/fts3/fts3Int.h 0b167bed9e63151635620a4f639bc62ac6012cba
 F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd
 F ext/fts3/fts3_expr.c 193d6fc156d744ab548a2ed06c31869e54dac739
 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914
@@ -67,7 +67,7 @@ F ext/fts3/fts3_porter.c a465b49fcb8249a755792f87516eff182efa42b3
 F ext/fts3/fts3_snippet.c 5fcfcafff46a2a3a63b8e59fcb51987d01c74695
 F ext/fts3/fts3_term.c a521f75132f9a495bdca1bdd45949b3191c52763
 F ext/fts3/fts3_test.c f9a1a1702db1bfad3e2d0064746eeb808f125489
-F ext/fts3/fts3_tokenize_vtab.c a29f126b9e6c6a6f1021a8f7440bf125e68af1f9
+F ext/fts3/fts3_tokenize_vtab.c 03a428b6221dc0b3f4527b6004266c5db71a2026
 F ext/fts3/fts3_tokenizer.c bbdc731bc91338050675c6d1da9ab82147391e16
 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
@@ -514,7 +514,7 @@ F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0
 F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2
 F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2
 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659
-F test/fts3tok1.test 05ff5c57098282bacba7192507702b17f4061295
+F test/fts3tok1.test 4d9e7401679dc71f6b2f76416309b923210bfdbe
 F test/fts3tok_err.test 41e5413139c2ef536ffadfcd1584ee50b1665ec0
 F test/fts4aa.test 95f448fb02c4a976968b08d1b4ce134e720946ae
 F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8
@@ -1061,7 +1061,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
-P b72d365920dbe45e61d8b1cff708a8858c2b3e33
-R e0d7d129e376f94da4504f4153569f21
-U drh
-Z e1d10da5e5ea459f8bce59c30cc68749
+P e5b3cd747bb0b484e38b8611a81925e2cc144435
+R 0fdf3312f3c095b75e4d2deab47476b0
+U dan
+Z 58e39fd3c70070287886f650d61f780d

manifest.uuid

@@ -1 +1 @@
-e5b3cd747bb0b484e38b8611a81925e2cc144435
+0ba67b64de258883e4c43db09e131bb67083855e

test/fts3tok1.test

@@ -106,15 +106,7 @@ do_catchsql_test 2.0 {
   CREATE VIRTUAL TABLE tX USING fts3tokenize(nosuchtokenizer);
 } {1 {unknown tokenizer: nosuchtokenizer}}
 
-proc fts3_tokenizer {args} { return 1 }
-db function fts3_tokenizer -argcount 1 fts3_tokenizer
 do_catchsql_test 2.1 {
-  CREATE VIRTUAL TABLE tX USING fts3tokenize(simple);
-} {1 {vtable constructor failed: tX}}
-
-db close
-sqlite3 db test.db
-do_catchsql_test 2.2 {
   CREATE VIRTUAL TABLE t4 USING fts3tokenize;
   SELECT * FROM t4;
 } {1 {SQL logic error or missing database}}