paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change the fts5 tokenizer API to allow more than one token to occupy a single position within a document.

Browse Source 
FossilOrigin-Name: 90b85b42f2b2dd3e939b129b7df2b822a05e243d
This commit is contained in:
dan
2015-08-28 19:56:47 +00:00
parent 2798f0b54b
commit 57e0add3f9
12 changed files with 99 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ext/fts5/fts5_config.c
+5 -2
View File
@@ -645,12 +645,15 @@ int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){
*/
int sqlite3Fts5Tokenize(
Fts5Config *pConfig, /* FTS5 Configuration object */
int flags, /* FTS5_TOKENIZE_* flags */
const char *pText, int nText, /* Text to tokenize */
void *pCtx, /* Context passed to xToken() */
int (*xToken)(void*, const char*, int, int, int) /* Callback */
int (*xToken)(void*, const char*, int, int, int, int) /* Callback */
){
if( pText==0 ) return SQLITE_OK;
return pConfig->pTokApi->xTokenize(pConfig->pTok, pCtx, pText, nText, xToken);
return pConfig->pTokApi->xTokenize(
pConfig->pTok, pCtx, flags, pText, nText, xToken
);
}
/*