Add the xTokenize extension API.

FossilOrigin-Name: 8c6b0aff3443fae4b7f0b9adcbf1514992b70653

ext/fts5/fts5.c

@@ -630,8 +630,9 @@ static int fts5ApiTokenize(
   void *pUserData,
   int (*xToken)(void*, const char*, int, int, int, int)
 ){
-  assert( 0 );
-  return SQLITE_OK;
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
+  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
+  return sqlite3Fts5Tokenize(pTab->pConfig, pText, nText, pUserData, xToken);
 }
 
 static int fts5ApiPhraseCount(Fts5Context *pCtx){

ext/fts5/fts5.h

@@ -68,6 +68,9 @@ typedef void (*fts5_extension_function)(
 **
 ** xPoslist:
 **   Iterate through instances of phrase iPhrase in the current row. 
+**
+** xTokenize:
+**   Tokenize text using the tokenizer belonging to the FTS5 table.
 */
 struct Fts5ExtensionApi {
   int iVersion;                   /* Currently always set to 1 */

ext/fts5/fts5_aux.c

@@ -23,6 +23,24 @@ static void fts5SnippetFunction(
   assert( 0 );
 }
 
+static int fts5TestCallback(
+  void *pContext,                 /* Pointer to Fts5Buffer object */
+  const char *pToken,             /* Buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStart,                     /* Start offset of token */
+  int iEnd,                       /* End offset of token */
+  int iPos                        /* Position offset of token */
+){
+  int rc = SQLITE_OK;
+  Fts5Buffer *pBuf = (Fts5Buffer*)pContext;
+  if( pBuf->n!=0 ){
+    sqlite3Fts5BufferAppendString(&rc, pBuf, " ");
+  }
+  sqlite3Fts5BufferAppendListElem(&rc, pBuf, pToken, nToken);
+  return rc;
+}
+
+
 static void fts5TestFunction(
   const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
   Fts5Context *pFts,              /* First arg to pass to pApi functions */
@@ -167,6 +185,33 @@ static void fts5TestFunction(
     sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", iRowid);
   }
 
+  if( zReq==0 ){
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, " tokenize ");
+  }
+  if( 0==zReq || 0==sqlite3_stricmp(zReq, "tokenize") ){
+    Fts5Buffer buf;
+    memset(&buf, 0, sizeof(buf));
+    for(i=0; rc==SQLITE_OK && i<nCol; i++){
+      const char *z;
+      int n;
+      rc = pApi->xColumnText(pFts, i, &z, &n);
+      if( rc==SQLITE_OK ){
+        Fts5Buffer buf1;
+        memset(&buf1, 0, sizeof(Fts5Buffer));
+        rc = pApi->xTokenize(pFts, z, n, (void*)&buf1, fts5TestCallback);
+        if( i!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &buf, " ");
+        sqlite3Fts5BufferAppendListElem(&rc, &buf, (const char*)buf1.p, buf1.n);
+        sqlite3_free(buf1.p);
+      }
+    }
+    if( zReq==0 ){
+      sqlite3Fts5BufferAppendListElem(&rc, &s, (const char*)buf.p, buf.n);
+    }else{
+      sqlite3Fts5BufferAppendString(&rc, &s, (const char*)buf.p);
+    }
+    sqlite3_free(buf.p);
+  }
+
   if( rc==SQLITE_OK ){
     sqlite3_result_text(pCtx, (const char*)s.p, -1, SQLITE_TRANSIENT);
   }else{

manifest

@@ -1,5 +1,5 @@
-C Fix\sthe\sxColumnSize()\sextension\sAPI.
-D 2014-07-21T11:44:47.659
+C Add\sthe\sxTokenize\sextension\sAPI.
+D 2014-07-21T14:22:38.753
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -103,10 +103,10 @@ F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
-F ext/fts5/fts5.c 5bf93402f9bafa55181dfa70c3a785a41af31025
-F ext/fts5/fts5.h 84060c2fe91aa03a783cc993f313d8a5b22cbe11
+F ext/fts5/fts5.c fda4dd1d2c3e1c6bcb0bf673428a23e4752c21f5
+F ext/fts5/fts5.h c77b6a4a56d80f70fc4f0444030c88724397ed10
 F ext/fts5/fts5Int.h 12d03496152b716e63a5380e396b776fbefa2065
-F ext/fts5/fts5_aux.c 9f0487715cd9933f2268620b41fb47f78a389297
+F ext/fts5/fts5_aux.c 14961135231dd50e6c17894e649c3bbc8c042829
 F ext/fts5/fts5_buffer.c 00361d4a70040ebd2c32bc349ab708ff613a1749
 F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef
 F ext/fts5/fts5_expr.c 288b3e016253eab69ea8cefbff346a4697b44291
@@ -599,7 +599,7 @@ F test/fts5aa.test 0f5d29bf0a86b9dff0906c9e166d624c591d3437
 F test/fts5ab.test dc04ed48cf93ca957d174406e6c192f2ff4f3397
 F test/fts5ac.test 9be418d037763f4cc5d86f4239db41fc86bb4f85
 F test/fts5ad.test 2ed38bbc865678cb2905247120d02ebba7f20e07
-F test/fts5ae.test b856782549ae0c56628e0980444fc38cf20b41a0
+F test/fts5ae.test fe9db78201bbb87c6f82b72a14b946d0f7fc3026
 F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
 F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
@@ -1195,7 +1195,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 1e9053abdaf5e128d44504ee00dfd909dc25f378
-R 962590916dedf53a395813418f70a86f
+P 19504c4108472d2ad1281221642b8bd06eb69f4e
+R 5c4f852f4d425543d226dd267d759872
 U dan
-Z 393e4d18f784466676fdeef71a737c83
+Z 17df0b0e1c67547ce560ffb2b8c93af3

manifest.uuid

@@ -1 +1 @@
-19504c4108472d2ad1281221642b8bd06eb69f4e
+8c6b0aff3443fae4b7f0b9adcbf1514992b70653

test/fts5ae.test

@@ -175,7 +175,22 @@ do_execsql_test 5.4 {
   1 {2 3}
 }
 
+#-------------------------------------------------------------------------
+# Test the xTokenize() API
+#
+reset_db
+do_execsql_test 6.1 {
+  CREATE VIRTUAL TABLE t6 USING fts5(x, y);
+  INSERT INTO t6 VALUES('There are more', 'things in heaven and earth');
+  INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.');
+}
 
+do_execsql_test 6.2 {
+  SELECT rowid, fts5_test(t6, 'tokenize') FROM t6 WHERE t6 MATCH 't*'
+} {
+  2 {{horatio than are} {dreamt of in your philosophy}}
+  1 {{there are more} {things in heaven and earth}}
+}
 
 finish_test