From 57e0add3f9227fb3325a73d41029168a9f08bfd2 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 28 Aug 2015 19:56:47 +0000 Subject: [PATCH 01/18] Change the fts5 tokenizer API to allow more than one token to occupy a single position within a document. FossilOrigin-Name: 90b85b42f2b2dd3e939b129b7df2b822a05e243d --- ext/fts5/fts5.h | 13 ++++++++++--- ext/fts5/fts5Int.h | 3 ++- ext/fts5/fts5_aux.c | 7 +++++-- ext/fts5/fts5_config.c | 7 +++++-- ext/fts5/fts5_expr.c | 8 ++++++-- ext/fts5/fts5_main.c | 17 ++++++++++------ ext/fts5/fts5_storage.c | 20 +++++++++++++------ ext/fts5/fts5_tcl.c | 11 +++++++---- ext/fts5/fts5_tokenize.c | 26 +++++++++++++++---------- ext/fts5/test/fts5matchinfo.test | 10 ++++------ manifest | 33 +++++++++++++++++--------------- manifest.uuid | 2 +- 12 files changed, 99 insertions(+), 58 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index c123d6444c..2e145d1068 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -217,7 +217,7 @@ struct Fts5ExtensionApi { int (*xTokenize)(Fts5Context*, const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ - int (*xToken)(void*, const char*, int, int, int) /* Callback */ + int (*xToken)(void*, const char*, int, int, int, int) /* Callback */ ); int (*xPhraseCount)(Fts5Context*); @@ -309,17 +309,24 @@ struct fts5_tokenizer { void (*xDelete)(Fts5Tokenizer*); int (*xTokenize)(Fts5Tokenizer*, void *pCtx, + int flags, const char *pText, int nText, int (*xToken)( void *pCtx, /* Copy of 2nd argument to xTokenize() */ const char *pToken, /* Pointer to buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Byte offset of token within input text */ - int iEnd /* Byte offset of end of token within input text */ + int iEnd, /* Byte offset of end of token within input text */ + int iPos /* Number of tokens before this one in input text */ ) ); }; +#define FTS5_TOKENIZE_QUERY 0x0001 +#define FTS5_TOKENIZE_PREFIX 0x0002 +#define FTS5_TOKENIZE_DOCUMENT 0x0004 +#define FTS5_TOKENIZE_AUX 0x0008 + /* ** END OF CUSTOM TOKENIZERS *************************************************************************/ @@ -329,7 +336,7 @@ struct fts5_tokenizer { */ typedef struct fts5_api fts5_api; struct fts5_api { - int iVersion; /* Currently always set to 1 */ + int iVersion; /* Currently always set to 2 */ /* Create a new tokenizer */ int (*xCreateTokenizer)( diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 5298429437..0357903c7b 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -166,9 +166,10 @@ int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig); int sqlite3Fts5Tokenize( Fts5Config *pConfig, /* FTS5 Configuration object */ + int flags, /* FTS5_TOKENIZE_* flags */ const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ - int (*xToken)(void*, const char*, int, int, int) /* Callback */ + int (*xToken)(void*, const char*, int, int, int, int) /* Callback */ ); void sqlite3Fts5Dequote(char *z); diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 818dfcf297..0e608721f7 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -151,11 +151,14 @@ static int fts5HighlightCb( const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStartOff, /* Start offset of token */ - int iEndOff /* End offset of token */ + int iEndOff, /* End offset of token */ + int iPos ){ HighlightContext *p = (HighlightContext*)pContext; int rc = SQLITE_OK; - int iPos = p->iPos++; + + if( iPosiPos ) return SQLITE_OK; + p->iPos = iPos+1; if( p->iRangeEnd>0 ){ if( iPosiRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 7e991fc21d..7987b8af76 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -645,12 +645,15 @@ int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){ */ int sqlite3Fts5Tokenize( Fts5Config *pConfig, /* FTS5 Configuration object */ + int flags, /* FTS5_TOKENIZE_* flags */ const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ - int (*xToken)(void*, const char*, int, int, int) /* Callback */ + int (*xToken)(void*, const char*, int, int, int, int) /* Callback */ ){ if( pText==0 ) return SQLITE_OK; - return pConfig->pTokApi->xTokenize(pConfig->pTok, pCtx, pText, nText, xToken); + return pConfig->pTokApi->xTokenize( + pConfig->pTok, pCtx, flags, pText, nText, xToken + ); } /* diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index a713bb7c5a..46b4091e37 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1341,7 +1341,8 @@ static int fts5ParseTokenize( const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ - int iEnd /* End offset of token */ + int iEnd, /* End offset of token */ + int iPos ){ int rc = SQLITE_OK; const int SZALLOC = 8; @@ -1417,8 +1418,11 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( rc = fts5ParseStringFromToken(pToken, &z); if( rc==SQLITE_OK ){ + int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0); + int n; sqlite3Fts5Dequote(z); - rc = sqlite3Fts5Tokenize(pConfig, z, strlen(z), &sCtx, fts5ParseTokenize); + n = strlen(z); + rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize); } sqlite3_free(z); if( rc ){ diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 1f42716514..5f0a90eadf 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1498,11 +1498,13 @@ static int fts5ApiTokenize( Fts5Context *pCtx, const char *pText, int nText, void *pUserData, - int (*xToken)(void*, const char*, int, int, int) + int (*xToken)(void*, const char*, int, int, int, int) ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); - return sqlite3Fts5Tokenize(pTab->pConfig, pText, nText, pUserData, xToken); + return sqlite3Fts5Tokenize( + pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken + ); } static int fts5ApiPhraseCount(Fts5Context *pCtx){ @@ -1658,10 +1660,11 @@ static int fts5ColumnSizeCb( const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ - int iEnd /* End offset of token */ + int iEnd, /* End offset of token */ + int iPos ){ int *pCnt = (int*)pContext; - *pCnt = *pCnt + 1; + *pCnt = iPos+1; return SQLITE_OK; } @@ -1691,7 +1694,9 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ pCsr->aColumnSize[i] = 0; rc = fts5ApiColumnText(pCtx, i, &z, &n); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5Tokenize(pConfig, z, n, p, fts5ColumnSizeCb); + rc = sqlite3Fts5Tokenize( + pConfig, FTS5_TOKENIZE_AUX, z, n, p, fts5ColumnSizeCb + ); } } } @@ -2344,7 +2349,7 @@ int sqlite3_fts5_init( void *p = (void*)pGlobal; memset(pGlobal, 0, sizeof(Fts5Global)); pGlobal->db = db; - pGlobal->api.iVersion = 1; + pGlobal->api.iVersion = 2; pGlobal->api.xCreateFunction = fts5CreateAux; pGlobal->api.xCreateTokenizer = fts5CreateTokenizer; pGlobal->api.xFindTokenizer = fts5FindTokenizer; diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index f09b7d9158..6b1c69c008 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -362,11 +362,13 @@ static int fts5StorageInsertCallback( const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ - int iEnd /* End offset of token */ + int iEnd, /* End offset of token */ + int iPos ){ Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext; Fts5Index *pIdx = pCtx->pStorage->pIndex; - int iPos = pCtx->szCol++; + assert( iPos+1>=pCtx->szCol ); + pCtx->szCol = iPos+1; return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, iPos, pToken, nToken); } @@ -394,6 +396,7 @@ static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){ if( pConfig->abUnindexed[iCol-1] ) continue; ctx.szCol = 0; rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_column_text(pSeek, iCol), sqlite3_column_bytes(pSeek, iCol), (void*)&ctx, @@ -565,6 +568,7 @@ int sqlite3Fts5StorageSpecialDelete( if( pConfig->abUnindexed[iCol] ) continue; ctx.szCol = 0; rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_value_text(apVal[iCol]), sqlite3_value_bytes(apVal[iCol]), (void*)&ctx, @@ -654,6 +658,7 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_column_text(pScan, ctx.iCol+1), sqlite3_column_bytes(pScan, ctx.iCol+1), (void*)&ctx, @@ -771,6 +776,7 @@ int sqlite3Fts5StorageInsert( ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_value_text(apVal[ctx.iCol+2]), sqlite3_value_bytes(apVal[ctx.iCol+2]), (void*)&ctx, @@ -841,10 +847,12 @@ static int fts5StorageIntegrityCallback( const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ - int iEnd /* End offset of token */ + int iEnd, /* End offset of token */ + int iPos ){ Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; - int iPos = pCtx->szCol++; + assert( iPos+1>=pCtx->szCol ); + pCtx->szCol = iPos+1; pCtx->cksum ^= sqlite3Fts5IndexCksum( pCtx->pConfig, pCtx->iRowid, pCtx->iCol, iPos, pToken, nToken ); @@ -886,8 +894,8 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ if( pConfig->abUnindexed[i] ) continue; ctx.iCol = i; ctx.szCol = 0; - rc = sqlite3Fts5Tokenize( - pConfig, + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_column_text(pScan, i+1), sqlite3_column_bytes(pScan, i+1), (void*)&ctx, diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 82f3e0390d..fecbd41d32 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -142,7 +142,7 @@ struct F5tAuxData { static int xTokenizeCb( void *pCtx, const char *zToken, int nToken, - int iStart, int iEnd + int iStart, int iEnd, int iPos ){ F5tFunction *p = (F5tFunction*)pCtx; Tcl_Obj *pEval = Tcl_DuplicateObj(p->pScript); @@ -585,7 +585,7 @@ struct F5tTokenizeCtx { static int xTokenizeCb2( void *pCtx, const char *zToken, int nToken, - int iStart, int iEnd + int iStart, int iEnd, int iPos ){ F5tTokenizeCtx *p = (F5tTokenizeCtx*)pCtx; if( p->bSubst ){ @@ -666,7 +666,9 @@ static int f5tTokenize( ctx.bSubst = (objc==5); ctx.pRet = pRet; ctx.zInput = zText; - rc = tokenizer.xTokenize(pTok, (void*)&ctx, zText, nText, xTokenizeCb2); + rc = tokenizer.xTokenize( + pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText, nText, xTokenizeCb2 + ); tokenizer.xDelete(pTok); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", 0); @@ -748,8 +750,9 @@ static void f5tTokenizerDelete(Fts5Tokenizer *p){ static int f5tTokenizerTokenize( Fts5Tokenizer *p, void *pCtx, + int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int, int, int) + int (*xToken)(void*, const char*, int, int, int, int) ){ F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; void *pOldCtx; diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index 426e35551b..d2477a050b 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -116,13 +116,15 @@ static void asciiFold(char *aOut, const char *aIn, int nByte){ static int fts5AsciiTokenize( Fts5Tokenizer *pTokenizer, void *pCtx, + int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd) + int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos) ){ AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer; int rc = SQLITE_OK; int ie; int is = 0; + int iPos = 0; char aFold[64]; int nFold = sizeof(aFold); @@ -158,7 +160,7 @@ static int fts5AsciiTokenize( asciiFold(pFold, &pText[is], nByte); /* Invoke the token callback */ - rc = xToken(pCtx, pFold, nByte, is, ie); + rc = xToken(pCtx, pFold, nByte, is, ie, iPos++); is = ie+1; } @@ -385,12 +387,14 @@ static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){ static int fts5UnicodeTokenize( Fts5Tokenizer *pTokenizer, void *pCtx, + int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd) + int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos) ){ Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer; int rc = SQLITE_OK; unsigned char *a = p->aTokenChar; + int iPos = 0; unsigned char *zTerm = (unsigned char*)&pText[nText]; unsigned char *zCsr = (unsigned char *)pText; @@ -475,7 +479,7 @@ static int fts5UnicodeTokenize( } /* Invoke the token callback */ - rc = xToken(pCtx, aFold, zOut-aFold, is, ie); + rc = xToken(pCtx, aFold, zOut-aFold, is, ie, iPos++); } tokenize_done: @@ -553,7 +557,7 @@ static int fts5PorterCreate( typedef struct PorterContext PorterContext; struct PorterContext { void *pCtx; - int (*xToken)(void*, const char*, int, int, int); + int (*xToken)(void*, const char*, int, int, int, int); char *aBuf; }; @@ -1121,7 +1125,8 @@ static int fts5PorterCb( const char *pToken, int nToken, int iStart, - int iEnd + int iEnd, + int iPos ){ PorterContext *p = (PorterContext*)pCtx; @@ -1175,10 +1180,10 @@ static int fts5PorterCb( nBuf--; } - return p->xToken(p->pCtx, aBuf, nBuf, iStart, iEnd); + return p->xToken(p->pCtx, aBuf, nBuf, iStart, iEnd, iPos); pass_through: - return p->xToken(p->pCtx, pToken, nToken, iStart, iEnd); + return p->xToken(p->pCtx, pToken, nToken, iStart, iEnd, iPos); } /* @@ -1187,8 +1192,9 @@ static int fts5PorterCb( static int fts5PorterTokenize( Fts5Tokenizer *pTokenizer, void *pCtx, + int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd) + int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos) ){ PorterTokenizer *p = (PorterTokenizer*)pTokenizer; PorterContext sCtx; @@ -1196,7 +1202,7 @@ static int fts5PorterTokenize( sCtx.pCtx = pCtx; sCtx.aBuf = p->aBuf; return p->tokenizer.xTokenize( - p->pTokenizer, (void*)&sCtx, pText, nText, fts5PorterCb + p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb ); } diff --git a/ext/fts5/test/fts5matchinfo.test b/ext/fts5/test/fts5matchinfo.test index 359702eff6..21f9b003e7 100644 --- a/ext/fts5/test/fts5matchinfo.test +++ b/ext/fts5/test/fts5matchinfo.test @@ -355,10 +355,10 @@ do_execsql_test 10.1 { #--------------------------------------------------------------------------- # Test the 'y' matchinfo flag # -set sqlite_fts3_enable_parentheses 1 reset_db +sqlite3_fts5_register_matchinfo db do_execsql_test 11.0 { - CREATE VIRTUAL TABLE tt USING fts3(x, y); + CREATE VIRTUAL TABLE tt USING fts5(x, y); INSERT INTO tt VALUES('c d a c d d', 'e a g b d a'); -- 1 INSERT INTO tt VALUES('c c g a e b', 'c g d g e c'); -- 2 INSERT INTO tt VALUES('b e f d e g', 'b a c b c g'); -- 3 @@ -432,19 +432,18 @@ foreach {tn expr res} { SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr } $r2 } -set sqlite_fts3_enable_parentheses 0 #--------------------------------------------------------------------------- # Test the 'b' matchinfo flag # -set sqlite_fts3_enable_parentheses 1 reset_db +sqlite3_fts5_register_matchinfo db db func mit mit do_test 12.0 { set cols [list] for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" } - execsql "CREATE VIRTUAL TABLE tt USING fts3([join $cols ,])" + execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,])" } {} do_execsql_test 12.1 { @@ -452,6 +451,5 @@ do_execsql_test 12.1 { SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc'; } [list [list [expr 1<<4] [expr 1<<(45-32)]]] -set sqlite_fts3_enable_parentheses 0 finish_test diff --git a/manifest b/manifest index f4ea3f942d..824f32d4c0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings\sin\srbu\scode. -D 2015-08-28T16:41:45.530 +C Change\sthe\sfts5\stokenizer\sAPI\sto\sallow\smore\sthan\sone\stoken\sto\soccupy\sa\ssingle\sposition\swithin\sa\sdocument. +D 2015-08-28T19:56:47.300 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -105,19 +105,19 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 -F ext/fts5/fts5.h 1950ec0544de667a24c1d8af9b2fde5db7db3bc9 -F ext/fts5/fts5Int.h 45f2ceb3c030f70e2cc4c199e9f700c2f2367f77 -F ext/fts5/fts5_aux.c 044cb176a815f4388308738437f6e130aa384fb0 +F ext/fts5/fts5.h b9dfb487ada3caab4400210609b8309b71a4fb4d +F ext/fts5/fts5Int.h b0cfe44ec9451f766b77c4e5f771e7919c6dc8d5 +F ext/fts5/fts5_aux.c 7d0e275ee94ad7afdd4208d6b071b4319e8f9ca0 F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 -F ext/fts5/fts5_config.c fdfa63ae8e527ecfaa50f94063c610429cc887cf -F ext/fts5/fts5_expr.c d075d36c84975a1cfcf070442d28e28027b61c25 +F ext/fts5/fts5_config.c ab81c8ccff6c0fb79f21c369e18e8e0dec365ec5 +F ext/fts5/fts5_expr.c f53917b6e68dee62e4c525466edacacf82eb7cbc F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 -F ext/fts5/fts5_main.c fc47ad734dfb55765b7542a345cee981170e7caa -F ext/fts5/fts5_storage.c 22ec9b5d35a39e2b5b65daf4ba7cd47fbb2d0df5 -F ext/fts5/fts5_tcl.c 96a3b9e982c4a64a242eefd752fa6669cd405a67 +F ext/fts5/fts5_main.c 7afdb84ac40b0e5bbb920a07a5cd5e062963816c +F ext/fts5/fts5_storage.c 9c263323479a4aa554738e421813cd05615d379c +F ext/fts5/fts5_tcl.c 41e2d6b455547a157085fd35fd59d4fd890dc7d3 F ext/fts5/fts5_test_mi.c 80a9e86fb4c5b6b58f8fefac05e9b96d1a6574e1 -F ext/fts5/fts5_tokenize.c 2836f6728bd74c7efac7487f5d9c27ca3e1b509c +F ext/fts5/fts5_tokenize.c 07a894410bc074685ddc0a9d89b5e7bf57ea4482 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc @@ -160,7 +160,7 @@ F ext/fts5/test/fts5fault6.test 234dc6355f8d3f8b5be2763f30699d770247c215 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6 F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145 -F ext/fts5/test/fts5matchinfo.test ee6e7b130096c708c12049fa9c1ceb628954c4f9 +F ext/fts5/test/fts5matchinfo.test 2163b0013e824bba65499da9e34ea4da41349cc2 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367 F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc F ext/fts5/test/fts5optimize.test 42741e7c085ee0a1276140a752d4407d97c2c9f5 @@ -1380,7 +1380,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a84cf4f5d326270a61faf4ff867260f2dd1e68a6 -R 25627e3633787096f1c6a6ac573b9337 +P 0fdc36fe35ae2fc8e9688fe6c53437f4d47502d9 +R 694c0e23ba08ed9bcc32d2c502ed8f13 +T *branch * fts5-incompatible +T *sym-fts5-incompatible * +T -sym-trunk * U dan -Z 8fac2bb28cf0b676d514df6920f608e8 +Z 745a50831400d199b74f44c2476ec260 diff --git a/manifest.uuid b/manifest.uuid index 453c3b2f4d..91a79bb98f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0fdc36fe35ae2fc8e9688fe6c53437f4d47502d9 \ No newline at end of file +90b85b42f2b2dd3e939b129b7df2b822a05e243d \ No newline at end of file From ee0c0a8de3cdcb336b062677eedd92e8f3d44cdb Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 29 Aug 2015 15:44:27 +0000 Subject: [PATCH 02/18] Another change to the fts5 tokenizer API. FossilOrigin-Name: fc71868496f45f9c7a79ed2bf2d164a7c4718ce1 --- ext/fts5/fts5.h | 11 ++++++++--- ext/fts5/fts5Int.h | 2 +- ext/fts5/fts5_aux.c | 9 +++++---- ext/fts5/fts5_config.c | 2 +- ext/fts5/fts5_expr.c | 6 ++++-- ext/fts5/fts5_main.c | 10 ++++++---- ext/fts5/fts5_storage.c | 22 ++++++++++++---------- ext/fts5/fts5_tcl.c | 14 ++++++++------ ext/fts5/fts5_tokenize.c | 22 ++++++++++------------ manifest | 31 ++++++++++++++----------------- manifest.uuid | 2 +- 11 files changed, 70 insertions(+), 61 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 2e145d1068..c9eb91d4cc 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -217,7 +217,7 @@ struct Fts5ExtensionApi { int (*xTokenize)(Fts5Context*, const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ - int (*xToken)(void*, const char*, int, int, int, int) /* Callback */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ ); int (*xPhraseCount)(Fts5Context*); @@ -313,20 +313,25 @@ struct fts5_tokenizer { const char *pText, int nText, int (*xToken)( void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Pointer to buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Byte offset of token within input text */ - int iEnd, /* Byte offset of end of token within input text */ - int iPos /* Number of tokens before this one in input text */ + int iEnd /* Byte offset of end of token within input text */ ) ); }; +/* Flags that may be passed as the third argument to xTokenize() */ #define FTS5_TOKENIZE_QUERY 0x0001 #define FTS5_TOKENIZE_PREFIX 0x0002 #define FTS5_TOKENIZE_DOCUMENT 0x0004 #define FTS5_TOKENIZE_AUX 0x0008 +/* Flags that may be passed by the tokenizer implementation back to FTS5 +** as the third argument to the supplied xToken callback. */ +#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ + /* ** END OF CUSTOM TOKENIZERS *************************************************************************/ diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 0357903c7b..c61f8c4052 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -169,7 +169,7 @@ int sqlite3Fts5Tokenize( int flags, /* FTS5_TOKENIZE_* flags */ const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ - int (*xToken)(void*, const char*, int, int, int, int) /* Callback */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ ); void sqlite3Fts5Dequote(char *z); diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 0e608721f7..2e33d5132f 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -148,17 +148,18 @@ static void fts5HighlightAppend( */ static int fts5HighlightCb( void *pContext, /* Pointer to HighlightContext object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStartOff, /* Start offset of token */ - int iEndOff, /* End offset of token */ - int iPos + int iEndOff /* End offset of token */ ){ HighlightContext *p = (HighlightContext*)pContext; int rc = SQLITE_OK; + int iPos; - if( iPosiPos ) return SQLITE_OK; - p->iPos = iPos+1; + if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK; + iPos = p->iPos++; if( p->iRangeEnd>0 ){ if( iPosiRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 7987b8af76..74faf6dd30 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -648,7 +648,7 @@ int sqlite3Fts5Tokenize( int flags, /* FTS5_TOKENIZE_* flags */ const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ - int (*xToken)(void*, const char*, int, int, int, int) /* Callback */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ ){ if( pText==0 ) return SQLITE_OK; return pConfig->pTokApi->xTokenize( diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 46b4091e37..9b52bb08c6 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1338,11 +1338,11 @@ struct TokenCtx { */ static int fts5ParseTokenize( void *pContext, /* Pointer to Fts5InsertCtx object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ - int iEnd, /* End offset of token */ - int iPos + int iEnd /* End offset of token */ ){ int rc = SQLITE_OK; const int SZALLOC = 8; @@ -1350,6 +1350,8 @@ static int fts5ParseTokenize( Fts5ExprPhrase *pPhrase = pCtx->pPhrase; Fts5ExprTerm *pTerm; + if( tflags & FTS5_TOKEN_COLOCATED ) return rc; + if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){ Fts5ExprPhrase *pNew; int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 5f0a90eadf..4704fb8906 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1498,7 +1498,7 @@ static int fts5ApiTokenize( Fts5Context *pCtx, const char *pText, int nText, void *pUserData, - int (*xToken)(void*, const char*, int, int, int, int) + int (*xToken)(void*, int, const char*, int, int, int) ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); @@ -1657,14 +1657,16 @@ static int fts5ApiColumnText( static int fts5ColumnSizeCb( void *pContext, /* Pointer to int */ + int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ - int iEnd, /* End offset of token */ - int iPos + int iEnd /* End offset of token */ ){ int *pCnt = (int*)pContext; - *pCnt = iPos+1; + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + (*pCnt)++; + } return SQLITE_OK; } diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 6b1c69c008..d0572e60e6 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -359,17 +359,18 @@ struct Fts5InsertCtx { */ static int fts5StorageInsertCallback( void *pContext, /* Pointer to Fts5InsertCtx object */ + int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ - int iEnd, /* End offset of token */ - int iPos + int iEnd /* End offset of token */ ){ Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext; Fts5Index *pIdx = pCtx->pStorage->pIndex; - assert( iPos+1>=pCtx->szCol ); - pCtx->szCol = iPos+1; - return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, iPos, pToken, nToken); + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + pCtx->szCol++; + } + return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); } /* @@ -844,17 +845,18 @@ struct Fts5IntegrityCtx { */ static int fts5StorageIntegrityCallback( void *pContext, /* Pointer to Fts5InsertCtx object */ + int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ - int iEnd, /* End offset of token */ - int iPos + int iEnd /* End offset of token */ ){ Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; - assert( iPos+1>=pCtx->szCol ); - pCtx->szCol = iPos+1; + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + pCtx->szCol++; + } pCtx->cksum ^= sqlite3Fts5IndexCksum( - pCtx->pConfig, pCtx->iRowid, pCtx->iCol, iPos, pToken, nToken + pCtx->pConfig, pCtx->iRowid, pCtx->iCol, pCtx->szCol-1, pToken, nToken ); return SQLITE_OK; } diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index fecbd41d32..528ca958f2 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -141,8 +141,9 @@ struct F5tAuxData { static int xTokenizeCb( void *pCtx, + int tflags, const char *zToken, int nToken, - int iStart, int iEnd, int iPos + int iStart, int iEnd ){ F5tFunction *p = (F5tFunction*)pCtx; Tcl_Obj *pEval = Tcl_DuplicateObj(p->pScript); @@ -584,8 +585,9 @@ struct F5tTokenizeCtx { static int xTokenizeCb2( void *pCtx, + int tflags, const char *zToken, int nToken, - int iStart, int iEnd, int iPos + int iStart, int iEnd ){ F5tTokenizeCtx *p = (F5tTokenizeCtx*)pCtx; if( p->bSubst ){ @@ -694,7 +696,7 @@ typedef struct F5tTokenizerModule F5tTokenizerInstance; struct F5tTokenizerContext { void *pCtx; - int (*xToken)(void*, const char*, int, int, int); + int (*xToken)(void*, int, const char*, int, int, int); }; struct F5tTokenizerModule { @@ -752,11 +754,11 @@ static int f5tTokenizerTokenize( void *pCtx, int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int, int, int, int) + int (*xToken)(void*, int, const char*, int, int, int) ){ F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; void *pOldCtx; - int (*xOldToken)(void*, const char*, int, int, int); + int (*xOldToken)(void*, int, const char*, int, int, int); Tcl_Obj *pEval; int rc; @@ -813,7 +815,7 @@ static int f5tTokenizerReturn( return TCL_ERROR; } - rc = p->xToken(p->pCtx, zToken, nToken, iStart, iEnd); + rc = p->xToken(p->pCtx, 0, zToken, nToken, iStart, iEnd); Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index d2477a050b..6b1129c24d 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -118,13 +118,12 @@ static int fts5AsciiTokenize( void *pCtx, int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos) + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) ){ AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer; int rc = SQLITE_OK; int ie; int is = 0; - int iPos = 0; char aFold[64]; int nFold = sizeof(aFold); @@ -160,7 +159,7 @@ static int fts5AsciiTokenize( asciiFold(pFold, &pText[is], nByte); /* Invoke the token callback */ - rc = xToken(pCtx, pFold, nByte, is, ie, iPos++); + rc = xToken(pCtx, 0, pFold, nByte, is, ie); is = ie+1; } @@ -389,12 +388,11 @@ static int fts5UnicodeTokenize( void *pCtx, int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos) + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) ){ Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer; int rc = SQLITE_OK; unsigned char *a = p->aTokenChar; - int iPos = 0; unsigned char *zTerm = (unsigned char*)&pText[nText]; unsigned char *zCsr = (unsigned char *)pText; @@ -479,7 +477,7 @@ static int fts5UnicodeTokenize( } /* Invoke the token callback */ - rc = xToken(pCtx, aFold, zOut-aFold, is, ie, iPos++); + rc = xToken(pCtx, 0, aFold, zOut-aFold, is, ie); } tokenize_done: @@ -557,7 +555,7 @@ static int fts5PorterCreate( typedef struct PorterContext PorterContext; struct PorterContext { void *pCtx; - int (*xToken)(void*, const char*, int, int, int, int); + int (*xToken)(void*, int, const char*, int, int, int); char *aBuf; }; @@ -1122,11 +1120,11 @@ static void fts5PorterStep1A(char *aBuf, int *pnBuf){ static int fts5PorterCb( void *pCtx, + int tflags, const char *pToken, int nToken, int iStart, - int iEnd, - int iPos + int iEnd ){ PorterContext *p = (PorterContext*)pCtx; @@ -1180,10 +1178,10 @@ static int fts5PorterCb( nBuf--; } - return p->xToken(p->pCtx, aBuf, nBuf, iStart, iEnd, iPos); + return p->xToken(p->pCtx, tflags, aBuf, nBuf, iStart, iEnd); pass_through: - return p->xToken(p->pCtx, pToken, nToken, iStart, iEnd, iPos); + return p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd); } /* @@ -1194,7 +1192,7 @@ static int fts5PorterTokenize( void *pCtx, int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos) + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) ){ PorterTokenizer *p = (PorterTokenizer*)pTokenizer; PorterContext sCtx; diff --git a/manifest b/manifest index 824f32d4c0..964f295f10 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sfts5\stokenizer\sAPI\sto\sallow\smore\sthan\sone\stoken\sto\soccupy\sa\ssingle\sposition\swithin\sa\sdocument. -D 2015-08-28T19:56:47.300 +C Another\schange\sto\sthe\sfts5\stokenizer\sAPI. +D 2015-08-29T15:44:27.938 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -105,19 +105,19 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 -F ext/fts5/fts5.h b9dfb487ada3caab4400210609b8309b71a4fb4d -F ext/fts5/fts5Int.h b0cfe44ec9451f766b77c4e5f771e7919c6dc8d5 -F ext/fts5/fts5_aux.c 7d0e275ee94ad7afdd4208d6b071b4319e8f9ca0 +F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42 +F ext/fts5/fts5Int.h 9fd31e682acae32806f77e7c3b543c4294274c92 +F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 -F ext/fts5/fts5_config.c ab81c8ccff6c0fb79f21c369e18e8e0dec365ec5 -F ext/fts5/fts5_expr.c f53917b6e68dee62e4c525466edacacf82eb7cbc +F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 +F ext/fts5/fts5_expr.c 7ea46f676491989069d31ae1f75c9439b0858711 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 -F ext/fts5/fts5_main.c 7afdb84ac40b0e5bbb920a07a5cd5e062963816c -F ext/fts5/fts5_storage.c 9c263323479a4aa554738e421813cd05615d379c -F ext/fts5/fts5_tcl.c 41e2d6b455547a157085fd35fd59d4fd890dc7d3 +F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42 +F ext/fts5/fts5_storage.c 9820e7b53ea12baf3c818485efd66346b73030c3 +F ext/fts5/fts5_tcl.c 058f8da51964458e9859edfc1ee13b1863edaeae F ext/fts5/fts5_test_mi.c 80a9e86fb4c5b6b58f8fefac05e9b96d1a6574e1 -F ext/fts5/fts5_tokenize.c 07a894410bc074685ddc0a9d89b5e7bf57ea4482 +F ext/fts5/fts5_tokenize.c 710541513ecf3fe6d9365326fc85aee6efe97229 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc @@ -1380,10 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0fdc36fe35ae2fc8e9688fe6c53437f4d47502d9 -R 694c0e23ba08ed9bcc32d2c502ed8f13 -T *branch * fts5-incompatible -T *sym-fts5-incompatible * -T -sym-trunk * +P 90b85b42f2b2dd3e939b129b7df2b822a05e243d +R f343432805e01f14633e088d58d566cf U dan -Z 745a50831400d199b74f44c2476ec260 +Z e4288542e4294b868813263b0597051d diff --git a/manifest.uuid b/manifest.uuid index 91a79bb98f..c7283c3a07 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90b85b42f2b2dd3e939b129b7df2b822a05e243d \ No newline at end of file +fc71868496f45f9c7a79ed2bf2d164a7c4718ce1 \ No newline at end of file From 0cb01f5c2c3d6e7cca3c9fcbc99751a8bdbf421e Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 29 Aug 2015 18:46:12 +0000 Subject: [PATCH 03/18] Add a test for an fts5 tokenizer that supports synonyms by adding multiple entries to the fts index. FossilOrigin-Name: 98d07d16cab92f1e7001afbe370df3ec6343fc1f --- ext/fts5/fts5_storage.c | 4 +- ext/fts5/fts5_tcl.c | 80 ++++++++++--- ext/fts5/test/fts5synonym.test | 198 +++++++++++++++++++++++++++++++++ manifest | 15 +-- manifest.uuid | 2 +- 5 files changed, 271 insertions(+), 28 deletions(-) create mode 100644 ext/fts5/test/fts5synonym.test diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index d0572e60e6..6ff81d428d 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -367,7 +367,7 @@ static int fts5StorageInsertCallback( ){ Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext; Fts5Index *pIdx = pCtx->pStorage->pIndex; - if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ pCtx->szCol++; } return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); @@ -852,7 +852,7 @@ static int fts5StorageIntegrityCallback( int iEnd /* End offset of token */ ){ Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; - if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ pCtx->szCol++; } pCtx->cksum ^= sqlite3Fts5IndexCksum( diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 528ca958f2..21ce5f7bde 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -692,7 +692,7 @@ static int f5tTokenize( typedef struct F5tTokenizerContext F5tTokenizerContext; typedef struct F5tTokenizerCb F5tTokenizerCb; typedef struct F5tTokenizerModule F5tTokenizerModule; -typedef struct F5tTokenizerModule F5tTokenizerInstance; +typedef struct F5tTokenizerInstance F5tTokenizerInstance; struct F5tTokenizerContext { void *pCtx; @@ -705,6 +705,12 @@ struct F5tTokenizerModule { F5tTokenizerContext *pContext; }; +struct F5tTokenizerInstance { + Tcl_Interp *interp; + Tcl_Obj *pScript; + F5tTokenizerContext *pContext; +}; + static int f5tTokenizerCreate( void *pCtx, const char **azArg, @@ -761,18 +767,44 @@ static int f5tTokenizerTokenize( int (*xOldToken)(void*, int, const char*, int, int, int); Tcl_Obj *pEval; int rc; + const char *zFlags; pOldCtx = pInst->pContext->pCtx; xOldToken = pInst->pContext->xToken; + pInst->pContext->pCtx = pCtx; + pInst->pContext->xToken = xToken; + + assert( + flags==FTS5_TOKENIZE_DOCUMENT + || flags==FTS5_TOKENIZE_AUX + || flags==FTS5_TOKENIZE_QUERY + || flags==(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) + ); pEval = Tcl_DuplicateObj(pInst->pScript); Tcl_IncrRefCount(pEval); - rc = Tcl_ListObjAppendElement( - pInst->interp, pEval, Tcl_NewStringObj(pText, nText) - ); - if( rc==TCL_OK ){ - rc = Tcl_EvalObjEx(pInst->interp, pEval, TCL_GLOBAL_ONLY); + switch( flags ){ + case FTS5_TOKENIZE_DOCUMENT: + zFlags = "document"; + break; + case FTS5_TOKENIZE_AUX: + zFlags = "aux"; + break; + case FTS5_TOKENIZE_QUERY: + zFlags = "query"; + break; + case (FTS5_TOKENIZE_PREFIX | FTS5_TOKENIZE_QUERY): + zFlags = "prefixquery"; + break; + default: + assert( 0 ); + zFlags = "invalid"; + break; } + + Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(zFlags, -1)); + Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(pText,nText)); + rc = Tcl_EvalObjEx(pInst->interp, pEval, TCL_GLOBAL_ONLY); Tcl_DecrRefCount(pEval); pInst->pContext->pCtx = pOldCtx; @@ -781,7 +813,7 @@ static int f5tTokenizerTokenize( } /* -** sqlite3_fts5_token TEXT START END POS +** sqlite3_fts5_token ?-colocated? TEXT START END */ static int f5tTokenizerReturn( void * clientData, @@ -793,14 +825,29 @@ static int f5tTokenizerReturn( int iStart; int iEnd; int nToken; + int tflags = 0; char *zToken; int rc; - assert( p ); - if( objc!=4 ){ - Tcl_WrongNumArgs(interp, 1, objv, "TEXT START END"); + if( objc==5 ){ + int nArg; + char *zArg = Tcl_GetStringFromObj(objv[1], &nArg); + if( nArg<=10 && nArg>=2 && memcmp("-colocated", zArg, nArg)==0 ){ + tflags |= FTS5_TOKEN_COLOCATED; + }else{ + goto usage; + } + }else if( objc!=4 ){ + goto usage; + } + + zToken = Tcl_GetStringFromObj(objv[objc-3], &nToken); + if( Tcl_GetIntFromObj(interp, objv[objc-2], &iStart) + || Tcl_GetIntFromObj(interp, objv[objc-1], &iEnd) + ){ return TCL_ERROR; } + if( p->xToken==0 ){ Tcl_AppendResult(interp, "sqlite3_fts5_token may only be used by tokenizer callback", 0 @@ -808,16 +855,13 @@ static int f5tTokenizerReturn( return TCL_ERROR; } - zToken = Tcl_GetStringFromObj(objv[1], &nToken); - if( Tcl_GetIntFromObj(interp, objv[2], &iStart) - || Tcl_GetIntFromObj(interp, objv[3], &iEnd) - ){ - return TCL_ERROR; - } - - rc = p->xToken(p->pCtx, 0, zToken, nToken, iStart, iEnd); + rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd); Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; + + usage: + Tcl_WrongNumArgs(interp, 1, objv, "?-colocated? TEXT START END"); + return TCL_ERROR; } static void f5tDelTokenizer(void *pCtx){ diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test new file mode 100644 index 0000000000..812f7c5882 --- /dev/null +++ b/ext/fts5/test/fts5synonym.test @@ -0,0 +1,198 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on custom tokenizers that support synonyms. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5synonym + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +proc gobble_whitespace {textvar} { + upvar $textvar t + regexp {([ ]*)(.*)} $t -> space t + return [string length $space] +} + +proc gobble_text {textvar wordvar} { + upvar $textvar t + upvar $wordvar w + regexp {([^ ]*)(.*)} $t -> w t + return [string length $w] +} + +proc do_tokenize_split {text} { + set token "" + set ret [list] + set iOff [gobble_whitespace text] + while {[set nToken [gobble_text text word]]} { + lappend ret $word $iOff [expr $iOff+$nToken] + incr iOff $nToken + incr iOff [gobble_whitespace text] + } + + set ret +} + +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [do_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + } +} + +proc tcl_create {args} { + return "tcl_tokenize" +} + +sqlite3_fts5_create_tokenizer db tcl tcl_create + +#------------------------------------------------------------------------- +# Warm body test for the code in fts5_tcl.c. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl); + INSERT INTO ft VALUES('abc def ghi'); + INSERT INTO ft VALUES('jkl mno pqr'); + SELECT rowid, x FROM ft WHERE ft MATCH 'def'; + SELECT x, rowid FROM ft WHERE ft MATCH 'pqr'; +} {1 {abc def ghi} {jkl mno pqr} 2} + +#------------------------------------------------------------------------- +# Test a tokenizer that supports synonyms by adding extra entries to the +# FTS index. +# +foreach S { + {zero 0} + {one 1} + {two 2} + {three 3 iii} + {four 4} + {five 5} + {six 6} + {seven 7} + {eight 8} + {nine 9} +} { + foreach s $S { + set o [list] + foreach x $S {if {$x!=$s} {lappend o $x}} + set ::syn($s) $o + } +} + +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [do_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags=="document" && [info exists ::syn($w)]} { + foreach s $::syn($w) { + sqlite3_fts5_token -colo $s $iStart $iEnd + } + } + } +} +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl); + INSERT INTO ft VALUES('one two three'); + INSERT INTO ft VALUES('four five six'); + INSERT INTO ft VALUES('eight nine ten'); +} {} + +foreach {tn expr res} { + 1 "3" 1 + 2 "eight OR 8 OR 5" {2 3} + 3 "10" {} + 4 "1*" {1} +} { + do_execsql_test 2.1.$tn { + SELECT rowid FROM ft WHERE ft MATCH $expr + } $res +} + +#------------------------------------------------------------------------- +# Test some broken tokenizers: +# +# 3.1.*: A tokenizer that declares the very first token to be colocated. +# +# 3.2.*: A tokenizer that reports two identical tokens at the same position. +# This is allowed. +# +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_tokenize {tflags text} { + set bColo 1 + foreach {w iStart iEnd} [do_tokenize_split $text] { + if {$bColo} { + sqlite3_fts5_token -colo $w $iStart $iEnd + set bColo 0 + } { + sqlite3_fts5_token $w $iStart $iEnd + } + } +} +do_execsql_test 3.1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl); + INSERT INTO ft VALUES('one two three'); + CREATE VIRTUAL TABLE vv USING fts5vocab(ft, row); + SELECT * FROM vv; +} { + one 1 1 three 1 1 two 1 1 +} + +do_execsql_test 3.1.1 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} {} + +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [do_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + } +} + +do_execsql_test 3.1.2 { + SELECT rowid FROM ft WHERE ft MATCH 'one two three' +} {1} + +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [do_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + sqlite3_fts5_token -colo $w $iStart $iEnd + } +} +do_execsql_test 3.2.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl); + INSERT INTO ft VALUES('one one two three'); + CREATE VIRTUAL TABLE vv USING fts5vocab(ft, row); + SELECT * FROM vv; +} { + one 1 4 three 1 2 two 1 2 +} +do_execsql_test 3.2.1 { + SELECT rowid FROM ft WHERE ft MATCH 'one two three'; + SELECT rowid FROM ft WHERE ft MATCH 'one + one + two + three'; +} {1 1} +do_execsql_test 3.2.2 { + SELECT rowid FROM ft WHERE ft MATCH 'one two two three'; + SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three'; +} {1} + +finish_test + diff --git a/manifest b/manifest index 964f295f10..fd2b4870e3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Another\schange\sto\sthe\sfts5\stokenizer\sAPI. -D 2015-08-29T15:44:27.938 +C Add\sa\stest\sfor\san\sfts5\stokenizer\sthat\ssupports\ssynonyms\sby\sadding\smultiple\sentries\sto\sthe\sfts\sindex. +D 2015-08-29T18:46:12.456 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,8 +114,8 @@ F ext/fts5/fts5_expr.c 7ea46f676491989069d31ae1f75c9439b0858711 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42 -F ext/fts5/fts5_storage.c 9820e7b53ea12baf3c818485efd66346b73030c3 -F ext/fts5/fts5_tcl.c 058f8da51964458e9859edfc1ee13b1863edaeae +F ext/fts5/fts5_storage.c c888defbb961d64c12299b3d1725a24a770b047e +F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 F ext/fts5/fts5_test_mi.c 80a9e86fb4c5b6b58f8fefac05e9b96d1a6574e1 F ext/fts5/fts5_tokenize.c 710541513ecf3fe6d9365326fc85aee6efe97229 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c @@ -172,6 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 +F ext/fts5/test/fts5synonym.test 6f1cfa5022bdae999f018075254e9fc51da2e618 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1380,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 90b85b42f2b2dd3e939b129b7df2b822a05e243d -R f343432805e01f14633e088d58d566cf +P fc71868496f45f9c7a79ed2bf2d164a7c4718ce1 +R 8d19f3ef91012c48b3ad32684321db98 U dan -Z e4288542e4294b868813263b0597051d +Z 14116ac3039ffa0cba068c002654bc88 diff --git a/manifest.uuid b/manifest.uuid index c7283c3a07..3c4a7727be 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc71868496f45f9c7a79ed2bf2d164a7c4718ce1 \ No newline at end of file +98d07d16cab92f1e7001afbe370df3ec6343fc1f \ No newline at end of file From cb8b58caf79f42bd5d067f8673d339ff6c0c64cc Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 31 Aug 2015 20:06:06 +0000 Subject: [PATCH 04/18] Begin changes to allow synonym support by adding multiple terms to a query (an alternative to adding multiple terms to the FTS index). FossilOrigin-Name: ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3 --- ext/fts5/fts5Int.h | 2 +- ext/fts5/fts5_expr.c | 284 ++++++++++++++++++++++++++------- ext/fts5/test/fts5aa.test | 2 +- ext/fts5/test/fts5ea.test | 6 + ext/fts5/test/fts5eb.test | 24 +-- ext/fts5/test/fts5synonym.test | 61 ++++--- manifest | 22 +-- manifest.uuid | 2 +- 8 files changed, 297 insertions(+), 106 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index c61f8c4052..e8d997b060 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -236,7 +236,7 @@ struct Fts5PoslistReader { int i; /* Current offset in a[] */ /* Output variables */ - int bEof; /* Set to true at EOF */ + u8 bEof; /* Set to true at EOF */ i64 iPos; /* (iCol<<32) + iPos */ }; int sqlite3Fts5PoslistReaderInit( diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 9b52bb08c6..bba9932daf 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -73,6 +73,7 @@ struct Fts5ExprTerm { int bPrefix; /* True for a prefix term */ char *zTerm; /* nul-terminated term */ Fts5IndexIter *pIter; /* Iterator for this term */ + Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */ }; /* @@ -181,6 +182,10 @@ static int fts5ExprGetToken( default: { const char *z2; + if( sqlite3Fts5IsBareword(z[0])==0 ){ + sqlite3Fts5ParseError(pParse, "fts5: syntax error near \"%.1s\"", z); + return FTS5_EOF; + } tok = FTS5_STRING; for(z2=&z[1]; sqlite3Fts5IsBareword(*z2); z2++); pToken->n = (z2 - z); @@ -350,6 +355,60 @@ static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){ return 0; } +/* +** Argument pTerm must be a synonym iterator. Return the current rowid +** that it points to. +*/ +static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){ + i64 iRet; + int bRetValid = 0; + Fts5ExprTerm *p; + + assert( pTerm->pSynonym ); + assert( bDesc==0 || bDesc==1 ); + for(p=pTerm; p; p=p->pSynonym){ + if( 0==sqlite3Fts5IterEof(p->pIter) ){ + i64 iRowid = sqlite3Fts5IterRowid(p->pIter); + if( bRetValid==0 || (bDesc!=(iRowidpSynonym ); + for(p=pTerm; p; p=p->pSynonym){ + Fts5IndexIter *pIter = p->pIter; + if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ + i64 dummy; + int rc = sqlite3Fts5IterPoslist(pIter, (const u8**)pa, pn, &dummy); + return rc; + } + } + + assert( 0 ); + return SQLITE_ERROR; +} + + /* ** All individual term iterators in pPhrase are guaranteed to be valid and ** pointing to the same rowid when this function is called. This function @@ -362,7 +421,7 @@ static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){ ** not a match. */ static int fts5ExprPhraseIsMatch( - Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pNode, /* Node pPhrase belongs to */ Fts5ExprColset *pColset, /* Restrict matches to these columns */ Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */ int *pbMatch /* OUT: Set to true if really a match */ @@ -391,10 +450,15 @@ static int fts5ExprPhraseIsMatch( /* Initialize a term iterator for each term in the phrase */ for(i=0; inTerm; i++){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; i64 dummy; int n; const u8 *a; - rc = sqlite3Fts5IterPoslist(pPhrase->aTerm[i].pIter, &a, &n, &dummy); + if( pTerm->pSynonym ){ + rc = fts5ExprSynonymPoslist(pTerm, pNode->iRowid, (u8**)&a, &n); + }else{ + rc = sqlite3Fts5IterPoslist(pTerm->pIter, &a, &n, &dummy); + } if( rc || sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]) ){ goto ismatch_out; } @@ -598,17 +662,43 @@ static int fts5ExprNearAdvanceFirst( int bFromValid, i64 iFrom ){ - Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter; + Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0]; int rc; - assert( Fts5NodeIsString(pNode) ); - if( bFromValid ){ - rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + if( pTerm->pSynonym ){ + int bEof = 1; + Fts5ExprTerm *p; + + /* Find the firstest rowid any synonym points to. */ + i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc); + + /* Advance each iterator that currently points to iRowid */ + for(p=pTerm; p; p=p->pSynonym){ + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + bEof = 0; + if( sqlite3Fts5IterRowid(p->pIter)==iRowid ){ + rc = sqlite3Fts5IterNext(p->pIter); + if( rc!=SQLITE_OK ) break; + } + } + } + + /* Set the EOF flag if either all synonym iterators are at EOF or an + ** error has occurred. */ + pNode->bEof = (rc || bEof); }else{ - rc = sqlite3Fts5IterNext(pIter); + Fts5IndexIter *pIter = pTerm->pIter; + + assert( Fts5NodeIsString(pNode) ); + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(pIter); + } + + pNode->bEof = (rc || sqlite3Fts5IterEof(pIter)); } - pNode->bEof = (rc || sqlite3Fts5IterEof(pIter)); return rc; } @@ -719,7 +809,7 @@ static int fts5ExprNearTest( Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; if( pPhrase->nTerm>1 || pNear->pColset ){ int bMatch = 0; - rc = fts5ExprPhraseIsMatch(pExpr, pNear->pColset, pPhrase, &bMatch); + rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); if( bMatch==0 ) break; }else{ rc = sqlite3Fts5IterPoslistBuffer( @@ -755,6 +845,7 @@ static int fts5ExprTokenTest( assert( pNode->eType==FTS5_TERM ); assert( pNear->nPhrase==1 && pPhrase->nTerm==1 ); + assert( pPhrase->aTerm[0].pSynonym==0 ); rc = sqlite3Fts5IterPoslist(pIter, &pPos, &nPos, &pNode->iRowid); @@ -801,69 +892,107 @@ static int fts5ExprNearNextMatch( i64 iLast; /* Lastest rowid any iterator points to */ int i, j; /* Phrase and token index, respectively */ int bMatch; /* True if all terms are at the same rowid */ + const int bDesc = pExpr->bDesc; - assert( pNear->nPhrase>1 || pNear->apPhrase[0]->nTerm>1 ); + /* Check that this node should not be FTS5_TERM */ + assert( pNear->nPhrase>1 + || pNear->apPhrase[0]->nTerm>1 + || pNear->apPhrase[0]->aTerm[0].pSynonym + ); /* Initialize iLast, the "lastest" rowid any iterator points to. If the ** iterator skips through rowids in the default ascending order, this means ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it ** means the minimum rowid. */ - iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter); + if( pLeft->aTerm[0].pSynonym ){ + iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc); + }else{ + iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter); + } do { bMatch = 1; for(i=0; inPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; for(j=0; jnTerm; j++){ - Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; - i64 iRowid = sqlite3Fts5IterRowid(pIter); - if( iRowid!=iLast ) bMatch = 0; - if( fts5ExprAdvanceto(pIter, pExpr->bDesc, &iLast,&rc,&pNode->bEof) ){ - return rc; + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; + if( pTerm->pSynonym ){ + Fts5ExprTerm *p; + int bEof = 1; + i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc); + if( iRowid==iLast ) continue; + for(p=pTerm; p; p=p->pSynonym){ + Fts5IndexIter *pIter = p->pIter; + int dummy; + if( 0==sqlite3Fts5IterEof(pIter) + && 0==fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &dummy)==0 + ){ + bEof = 0; + } + } + if( bEof || rc ){ + pNode->bEof = 1; + return rc; + } + }else{ + Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; + i64 iRowid = sqlite3Fts5IterRowid(pIter); + if( iRowid==iLast ) continue; + bMatch = 0; + if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ + return rc; + } } } } }while( bMatch==0 ); - pNode->bNomatch = (0==fts5ExprNearTest(&rc, pExpr, pNode)); pNode->iRowid = iLast; + pNode->bNomatch = (0==fts5ExprNearTest(&rc, pExpr, pNode)); return rc; } /* ** Initialize all term iterators in the pNear object. If any term is found -** to match no documents at all, set *pbEof to true and return immediately, -** without initializing any further iterators. +** to match no documents at all, return immediately without initializing any +** further iterators. */ static int fts5ExprNearInitAll( Fts5Expr *pExpr, Fts5ExprNode *pNode ){ Fts5ExprNearset *pNear = pNode->pNear; - Fts5ExprTerm *pTerm; - Fts5ExprPhrase *pPhrase; int i, j; int rc = SQLITE_OK; for(i=0; rc==SQLITE_OK && inPhrase; i++){ - pPhrase = pNear->apPhrase[i]; + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; for(j=0; jnTerm; j++){ - pTerm = &pPhrase->aTerm[j]; - if( pTerm->pIter ){ - sqlite3Fts5IterClose(pTerm->pIter); - pTerm->pIter = 0; + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; + Fts5ExprTerm *p; + int bEof = 1; + + for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){ + if( p->pIter ){ + sqlite3Fts5IterClose(p->pIter); + p->pIter = 0; + } + rc = sqlite3Fts5IndexQuery( + pExpr->pIndex, p->zTerm, strlen(p->zTerm), + (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | + (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), + &p->pIter + ); + assert( rc==SQLITE_OK || p->pIter==0 ); + if( p->pIter && 0==sqlite3Fts5IterEof(pTerm->pIter) ){ + bEof = 0; + } } - rc = sqlite3Fts5IndexQuery( - pExpr->pIndex, pTerm->zTerm, strlen(pTerm->zTerm), - (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | - (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), - &pTerm->pIter - ); - assert( rc==SQLITE_OK || pTerm->pIter==0 ); - if( pTerm->pIter==0 || sqlite3Fts5IterEof(pTerm->pIter) ){ + + if( bEof ){ pNode->bEof = 1; - break; + return rc; } } } @@ -1266,10 +1395,16 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){ if( pPhrase ){ int i; for(i=0; inTerm; i++){ + Fts5ExprTerm *pSyn; + Fts5ExprTerm *pNext; Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; sqlite3_free(pTerm->zTerm); - if( pTerm->pIter ){ - sqlite3Fts5IterClose(pTerm->pIter); + sqlite3Fts5IterClose(pTerm->pIter); + + for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){ + pNext = pSyn->pSynonym; + sqlite3Fts5IterClose(pSyn->pIter); + sqlite3_free(pSyn); } } if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist); @@ -1348,27 +1483,37 @@ static int fts5ParseTokenize( const int SZALLOC = 8; TokenCtx *pCtx = (TokenCtx*)pContext; Fts5ExprPhrase *pPhrase = pCtx->pPhrase; - Fts5ExprTerm *pTerm; - if( tflags & FTS5_TOKEN_COLOCATED ) return rc; + assert( pPhrase==0 || pPhrase->nTerm>0 ); + if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){ + Fts5ExprTerm *pSyn; + int nByte = sizeof(Fts5ExprTerm) + nToken+1; + pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte); + if( pSyn==0 ) return SQLITE_NOMEM; + memset(pSyn, 0, nByte); + pSyn->zTerm = (char*)&pSyn[1]; + memcpy(pSyn->zTerm, pToken, nToken); + pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; + pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; + }else{ + Fts5ExprTerm *pTerm; + if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){ + Fts5ExprPhrase *pNew; + int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); - if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){ - Fts5ExprPhrase *pNew; - int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); + pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, + sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew + ); + if( pNew==0 ) return SQLITE_NOMEM; + if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); + pCtx->pPhrase = pPhrase = pNew; + pNew->nTerm = nNew - SZALLOC; + } - pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, - sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew - ); - if( pNew==0 ) return SQLITE_NOMEM; - if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); - pCtx->pPhrase = pPhrase = pNew; - pNew->nTerm = nNew - SZALLOC; + pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; + memset(pTerm, 0, sizeof(Fts5ExprTerm)); + pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); } - - pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; - memset(pTerm, 0, sizeof(Fts5ExprTerm)); - pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); - return rc; } @@ -1636,7 +1781,10 @@ Fts5ExprNode *sqlite3Fts5ParseNode( for(iPhrase=0; iPhrasenPhrase; iPhrase++){ pNear->apPhrase[iPhrase]->pNode = pRet; } - if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){ + if( pNear->nPhrase==1 + && pNear->apPhrase[0]->nTerm==1 + && pNear->apPhrase[0]->aTerm[0].pSynonym==0 + ){ pRet->eType = FTS5_TERM; } }else{ @@ -1656,16 +1804,28 @@ Fts5ExprNode *sqlite3Fts5ParseNode( } static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ - char *zQuoted = sqlite3_malloc(strlen(pTerm->zTerm) * 2 + 3 + 2); + int nByte = 0; + Fts5ExprTerm *p; + char *zQuoted; + + /* Determine the maximum amount of space required. */ + for(p=pTerm; p; p=p->pSynonym){ + nByte += strlen(pTerm->zTerm) * 2 + 3 + 2; + } + zQuoted = sqlite3_malloc(nByte); + if( zQuoted ){ int i = 0; - char *zIn = pTerm->zTerm; - zQuoted[i++] = '"'; - while( *zIn ){ - if( *zIn=='"' ) zQuoted[i++] = '"'; - zQuoted[i++] = *zIn++; + for(p=pTerm; p; p=p->pSynonym){ + char *zIn = p->zTerm; + zQuoted[i++] = '"'; + while( *zIn ){ + if( *zIn=='"' ) zQuoted[i++] = '"'; + zQuoted[i++] = *zIn++; + } + zQuoted[i++] = '"'; + if( p->pSynonym ) zQuoted[i++] = '|'; } - zQuoted[i++] = '"'; if( pTerm->bPrefix ){ zQuoted[i++] = ' '; zQuoted[i++] = '*'; diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index daa535cd9b..22aa1b0f35 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -343,7 +343,7 @@ do_execsql_test 13.5 { } {1} do_execsql_test 13.6 { - SELECT rowid FROM t1 WHERE t1 MATCH '.'; + SELECT rowid FROM t1 WHERE t1 MATCH '""'; } {} #------------------------------------------------------------------------- diff --git a/ext/fts5/test/fts5ea.test b/ext/fts5/test/fts5ea.test index ad05412ba9..3ccbd7d7a2 100644 --- a/ext/fts5/test/fts5ea.test +++ b/ext/fts5/test/fts5ea.test @@ -87,6 +87,12 @@ do_execsql_test 4.0 { SELECT fts5_expr('a AND """"', 'x', 'tokenize="unicode61 tokenchars ''""''"'); } {{"a" AND """"}} +#------------------------------------------------------------------------- +# Experiment with a tokenizer that considers " to be a token character. +# +do_catchsql_test 5.0 { + SELECT fts5_expr('abc | def'); +} {1 {fts5: syntax error near "|"}} diff --git a/ext/fts5/test/fts5eb.test b/ext/fts5/test/fts5eb.test index 352e1b4a17..55c4b15cf3 100644 --- a/ext/fts5/test/fts5eb.test +++ b/ext/fts5/test/fts5eb.test @@ -30,18 +30,18 @@ proc do_syntax_test {tn expr res} { } foreach {tn expr res} { - 1 {abc} {"abc"} - 2 {abc .} {"abc"} - 3 {.} {} - 4 {abc OR .} {"abc"} - 5 {abc NOT .} {"abc"} - 6 {abc AND .} {"abc"} - 7 {. OR abc} {"abc"} - 8 {. NOT abc} {"abc"} - 9 {. AND abc} {"abc"} - 10 {abc + . + def} {"abc" + "def"} - 11 {abc . def} {"abc" AND "def"} - 12 {r+e OR w} {"r" + "e" OR "w"} + 1 {abc} {"abc"} + 2 {abc ""} {"abc"} + 3 {""} {} + 4 {abc OR ""} {"abc"} + 5 {abc NOT ""} {"abc"} + 6 {abc AND ""} {"abc"} + 7 {"" OR abc} {"abc"} + 8 {"" NOT abc} {"abc"} + 9 {"" AND abc} {"abc"} + 10 {abc + "" + def} {"abc" + "def"} + 11 {abc "" def} {"abc" AND "def"} + 12 {r+e OR w} {"r" + "e" OR "w"} } { do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res] } diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 812f7c5882..a607b7e29c 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -21,6 +21,24 @@ ifcapable !fts5 { return } +foreach S { + {zero 0} + {one 1} + {two 2} + {three 3 iii} + {four 4} + {five 5} + {six 6} + {seven 7} + {eight 8} + {nine 9} +} { + foreach s $S { + set o [list] + foreach x $S {if {$x!=$s} {lappend o $x}} + set ::syn($s) $o + } +} proc gobble_whitespace {textvar} { upvar $textvar t @@ -75,24 +93,6 @@ do_execsql_test 1.0 { # Test a tokenizer that supports synonyms by adding extra entries to the # FTS index. # -foreach S { - {zero 0} - {one 1} - {two 2} - {three 3 iii} - {four 4} - {five 5} - {six 6} - {seven 7} - {eight 8} - {nine 9} -} { - foreach s $S { - set o [list] - foreach x $S {if {$x!=$s} {lappend o $x}} - set ::syn($s) $o - } -} proc tcl_tokenize {tflags text} { foreach {w iStart iEnd} [do_tokenize_split $text] { @@ -194,5 +194,30 @@ do_execsql_test 3.2.2 { SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three'; } {1} +#------------------------------------------------------------------------- +# Check that expressions with synonyms can be parsed. +# +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [do_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags=="query" && [info exists ::syn($w)]} { + foreach s $::syn($w) { + sqlite3_fts5_token -colo $s $iStart $iEnd + } + } + } +} + +foreach {tn expr res} { + 1 {abc} {"abc"} + 2 {one} {"one"|"1"} + 3 {3} {"3"|"iii"|"three"} + 4 {3*} {"3"|"iii"|"three" *} +} { + do_execsql_test 4.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res] +} + finish_test diff --git a/manifest b/manifest index fd2b4870e3..4d5891206e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\sfor\san\sfts5\stokenizer\sthat\ssupports\ssynonyms\sby\sadding\smultiple\sentries\sto\sthe\sfts\sindex. -D 2015-08-29T18:46:12.456 +C Begin\schanges\sto\sallow\ssynonym\ssupport\sby\sadding\smultiple\sterms\sto\sa\squery\s(an\salternative\sto\sadding\smultiple\sterms\sto\sthe\sFTS\sindex). +D 2015-08-31T20:06:06.235 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -106,11 +106,11 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42 -F ext/fts5/fts5Int.h 9fd31e682acae32806f77e7c3b543c4294274c92 +F ext/fts5/fts5Int.h 9c538f0fcc1c3bc2fa12f7199d1326bd2362ce9c F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 7ea46f676491989069d31ae1f75c9439b0858711 +F ext/fts5/fts5_expr.c 1458c3e33c1ec3ad99284f9692edfc49c44afd7c F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42 @@ -124,7 +124,7 @@ F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl 3338968de1880ca12b0451ae8f9b8b12d14e0ba7 -F ext/fts5/test/fts5aa.test c6e680a0d1b6c2616a382f1006d5d91eca697bd0 +F ext/fts5/test/fts5aa.test caa44c528f7270aa4e325c4f2c28d355c1e8c307 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c F ext/fts5/test/fts5ad.test b2edee8b7de0c21d2c88f8a18c195034aad6952d @@ -149,8 +149,8 @@ F ext/fts5/test/fts5corrupt2.test 1a830ccd6dbe1b601c7e3f5bbc1cf77bd8c8803b F ext/fts5/test/fts5corrupt3.test 1ccf575f5126e79f9fec7979fd02a1f40a076be3 F ext/fts5/test/fts5dlidx.test 59b80bbe34169a082c575d9c26f0a7019a7b79c1 F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b -F ext/fts5/test/fts5ea.test 451bb37310ee6df8ef72e4354fda5621b3b51448 -F ext/fts5/test/fts5eb.test 46f49497edc25ef3b2bff9fb6d75b6d201e2b39e +F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0 +F ext/fts5/test/fts5eb.test 3e5869af2008cbc4ad03a175a0b6f6e58134cd43 F ext/fts5/test/fts5fault1.test 7a562367cb4a735b57b410dbdb62dcc8d971faec F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test 6f1cfa5022bdae999f018075254e9fc51da2e618 +F ext/fts5/test/fts5synonym.test 3343a05f7b1d3fc6fa4559bd3bd0ee979928c30d F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fc71868496f45f9c7a79ed2bf2d164a7c4718ce1 -R 8d19f3ef91012c48b3ad32684321db98 +P 98d07d16cab92f1e7001afbe370df3ec6343fc1f +R e2a3883c4d0440bf5b53c5a71ba46260 U dan -Z 14116ac3039ffa0cba068c002654bc88 +Z 7c01c61f2a729597f3365ce7db7a6ce6 diff --git a/manifest.uuid b/manifest.uuid index 3c4a7727be..b0ebea4bef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -98d07d16cab92f1e7001afbe370df3ec6343fc1f \ No newline at end of file +ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3 \ No newline at end of file From 3faa07ee0474436575a6cb81d56e80f6c47c3bd9 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 1 Sep 2015 18:08:36 +0000 Subject: [PATCH 05/18] Add tests for fts5 synonyms implemented by adding extra terms to queries. And fixes for the same. FossilOrigin-Name: dbcb73802b88f76be17f09f3eb83ffac542de633 --- ext/fts5/fts5Int.h | 2 + ext/fts5/fts5_expr.c | 92 +++++++++++++++++++++++------ ext/fts5/test/fts5synonym.test | 102 +++++++++++++++++++++++++++------ manifest | 16 +++--- manifest.uuid | 2 +- 5 files changed, 172 insertions(+), 42 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index e8d997b060..1ad61617d6 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -235,6 +235,8 @@ struct Fts5PoslistReader { int n; /* Size of buffer at a[] in bytes */ int i; /* Current offset in a[] */ + u8 bFlag; /* For client use (any custom purpose) */ + /* Output variables */ u8 bEof; /* Set to true at EOF */ i64 iPos; /* (iCol<<32) + iPos */ diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index bba9932daf..24d63c1cab 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -22,6 +22,8 @@ */ #define FTS5_EOF 0 +#define FTS5_LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) + typedef struct Fts5ExprTerm Fts5ExprTerm; /* @@ -386,26 +388,66 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){ static int fts5ExprSynonymPoslist( Fts5ExprTerm *pTerm, i64 iRowid, + int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */ u8 **pa, int *pn ){ Fts5PoslistWriter writer = {0}; Fts5PoslistReader aStatic[4]; Fts5PoslistReader *aIter = aStatic; int nIter = 0; + int rc = SQLITE_OK; Fts5ExprTerm *p; assert( pTerm->pSynonym ); for(p=pTerm; p; p=p->pSynonym){ Fts5IndexIter *pIter = p->pIter; if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ + const u8 *a; + int n; i64 dummy; - int rc = sqlite3Fts5IterPoslist(pIter, (const u8**)pa, pn, &dummy); - return rc; + rc = sqlite3Fts5IterPoslist(pIter, &a, &n, &dummy); + if( rc!=SQLITE_OK ) return rc; + if( sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter])==0 ){ + nIter++; + } } } - assert( 0 ); - return SQLITE_ERROR; + assert( *pbDel==0 ); + if( nIter==1 ){ + *pa = (u8*)aIter[0].a; + *pn = aIter[0].n; + }else{ + Fts5PoslistWriter writer = {0}; + Fts5Buffer buf = {0,0,0}; + i64 iPrev = -1; + while( 1 ){ + int i; + i64 iMin = FTS5_LARGEST_INT64; + for(i=0; inTerm); /* Initialize a term iterator for each term in the phrase */ for(i=0; inTerm; i++){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; i64 dummy; - int n; - const u8 *a; + int n = 0; + int bFlag = 0; + const u8 *a = 0; if( pTerm->pSynonym ){ - rc = fts5ExprSynonymPoslist(pTerm, pNode->iRowid, (u8**)&a, &n); + rc = fts5ExprSynonymPoslist(pTerm, pNode->iRowid, &bFlag, (u8**)&a, &n); }else{ rc = sqlite3Fts5IterPoslist(pTerm->pIter, &a, &n, &dummy); } - if( rc || sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]) ){ - goto ismatch_out; - } + if( rc!=SQLITE_OK ) goto ismatch_out; + sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]); + aIter[i].bFlag = bFlag; + if( aIter[i].bEof ) goto ismatch_out; } while( 1 ){ @@ -495,6 +540,9 @@ static int fts5ExprPhraseIsMatch( ismatch_out: *pbMatch = (pPhrase->poslist.n>0); + for(i=0; inTerm; i++){ + if( aIter[i].bFlag ) sqlite3_free((u8*)aIter[i].a); + } if( aIter!=aStatic ) sqlite3_free(aIter); return rc; } @@ -672,13 +720,25 @@ static int fts5ExprNearAdvanceFirst( /* Find the firstest rowid any synonym points to. */ i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc); - /* Advance each iterator that currently points to iRowid */ + /* Advance each iterator that currently points to iRowid. Or, if iFrom + ** is valid - each iterator that points to a rowid before iFrom. */ for(p=pTerm; p; p=p->pSynonym){ if( sqlite3Fts5IterEof(p->pIter)==0 ){ - bEof = 0; - if( sqlite3Fts5IterRowid(p->pIter)==iRowid ){ - rc = sqlite3Fts5IterNext(p->pIter); + i64 ii = sqlite3Fts5IterRowid(p->pIter); + if( ii==iRowid + || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) + ){ + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(p->pIter); + } if( rc!=SQLITE_OK ) break; + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + bEof = 0; + } + }else{ + bEof = 0; } } } @@ -807,7 +867,7 @@ static int fts5ExprNearTest( ** phrase is not a match, break out of the loop early. */ for(i=0; rc==SQLITE_OK && inPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; - if( pPhrase->nTerm>1 || pNear->pColset ){ + if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ int bMatch = 0; rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); if( bMatch==0 ) break; @@ -985,7 +1045,7 @@ static int fts5ExprNearInitAll( &p->pIter ); assert( rc==SQLITE_OK || p->pIter==0 ); - if( p->pIter && 0==sqlite3Fts5IterEof(pTerm->pIter) ){ + if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){ bEof = 0; } } diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index a607b7e29c..5aa29e003a 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -23,15 +23,15 @@ ifcapable !fts5 { foreach S { {zero 0} - {one 1} - {two 2} + {one 1 i} + {two 2 ii} {three 3 iii} - {four 4} - {five 5} - {six 6} - {seven 7} - {eight 8} - {nine 9} + {four 4 iv} + {five 5 v} + {six 6 vi} + {seven 7 vii} + {eight 8 viii} + {nine 9 ix} } { foreach s $S { set o [list] @@ -186,16 +186,23 @@ do_execsql_test 3.2.0 { one 1 4 three 1 2 two 1 2 } do_execsql_test 3.2.1 { - SELECT rowid FROM ft WHERE ft MATCH 'one two three'; - SELECT rowid FROM ft WHERE ft MATCH 'one + one + two + three'; -} {1 1} -do_execsql_test 3.2.2 { - SELECT rowid FROM ft WHERE ft MATCH 'one two two three'; - SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three'; + SELECT rowid FROM ft WHERE ft MATCH 'one'; } {1} +do_execsql_test 3.2.2 { + SELECT rowid FROM ft WHERE ft MATCH 'one two three'; +} {1} +do_execsql_test 3.2.3 { + SELECT rowid FROM ft WHERE ft MATCH 'one + one + two + three'; +} {1} +do_execsql_test 3.2.4 { + SELECT rowid FROM ft WHERE ft MATCH 'one two two three'; +} {1} +do_execsql_test 3.2.5 { + SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three'; +} {} #------------------------------------------------------------------------- -# Check that expressions with synonyms can be parsed. +# Check that expressions with synonyms can be parsed and executed. # reset_db sqlite3_fts5_create_tokenizer db tcl tcl_create @@ -212,11 +219,72 @@ proc tcl_tokenize {tflags text} { foreach {tn expr res} { 1 {abc} {"abc"} - 2 {one} {"one"|"1"} + 2 {one} {"one"|"i"|"1"} 3 {3} {"3"|"iii"|"three"} 4 {3*} {"3"|"iii"|"three" *} } { - do_execsql_test 4.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res] + do_execsql_test 4.1.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res] +} + +do_execsql_test 4.2.1 { + CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tcl); + INSERT INTO xx VALUES('one two'); + INSERT INTO xx VALUES('three four'); +} + +do_execsql_test 4.2.2 { + SELECT rowid FROM xx WHERE xx MATCH '2' +} {1} + +do_execsql_test 4.2.3 { + SELECT rowid FROM xx WHERE xx MATCH '3' +} {2} + +do_test 5.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tcl) + } + foreach {rowid a b} { + 1 {four v 4 i three} {1 3 five five 4 one} + 2 {5 1 3 4 i} {2 2 v two 4} + 3 {5 i 5 2 four 4 1} {iii ii five two 1} + 4 {ii four 4 one 5 three five} {one 5 1 iii 4 3} + 5 {three i v i four 4 1} {ii five five five iii} + 6 {4 2 ii two 2 iii} {three 1 four 4 iv 1 iv} + 7 {ii ii two three 2 5} {iii i ii iii iii one one} + 8 {2 ii i two 3 three 2} {two iv v iii 3 five} + 9 {i 2 iv 3 five four v} {iii 4 three i three ii 1} + } { + execsql { INSERT INTO t1(rowid, a, b) VALUES($rowid, $a, $b) } + } +} {} + +foreach {tn q res} { + 1 {one} { + 1 {four v 4 [i] three} {[1] 3 five five 4 [one]} + 2 {5 [1] 3 4 [i]} {2 2 v two 4} + 3 {5 [i] 5 2 four 4 [1]} {iii ii five two [1]} + 4 {ii four 4 [one] 5 three five} {[one] 5 [1] iii 4 3} + 5 {three [i] v [i] four 4 [1]} {ii five five five iii} + 6 {4 2 ii two 2 iii} {three [1] four 4 iv [1] iv} + 7 {ii ii two three 2 5} {iii [i] ii iii iii [one] [one]} + 8 {2 ii [i] two 3 three 2} {two iv v iii 3 five} + 9 {[i] 2 iv 3 five four v} {iii 4 three [i] three ii [1]} + } + 2 {five four} { + 1 {[four] [v] [4] i three} {1 3 [five] [five] [4] one} + 2 {[5] 1 3 [4] i} {2 2 [v] two [4]} + 3 {[5] i [5] 2 [four] [4] 1} {iii ii [five] two 1} + 4 {ii [four] [4] one [5] three [five]} {one [5] 1 iii [4] 3} + 5 {three i [v] i [four] [4] 1} {ii [five] [five] [five] iii} + 8 {2 ii i two 3 three 2} {two [iv] [v] iii 3 [five]} + 9 {i 2 [iv] 3 [five] [four] [v]} {iii [4] three i three ii 1} + } +} { + do_execsql_test 5.1.$tn { + SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']') + FROM t1 WHERE t1 MATCH $q + } $res } finish_test diff --git a/manifest b/manifest index 4d5891206e..4a0414c4b0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begin\schanges\sto\sallow\ssynonym\ssupport\sby\sadding\smultiple\sterms\sto\sa\squery\s(an\salternative\sto\sadding\smultiple\sterms\sto\sthe\sFTS\sindex). -D 2015-08-31T20:06:06.235 +C Add\stests\sfor\sfts5\ssynonyms\simplemented\sby\sadding\sextra\sterms\sto\squeries.\sAnd\sfixes\sfor\sthe\ssame. +D 2015-09-01T18:08:36.324 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -106,11 +106,11 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42 -F ext/fts5/fts5Int.h 9c538f0fcc1c3bc2fa12f7199d1326bd2362ce9c +F ext/fts5/fts5Int.h d46f89aeb357fbcf5b268d71b0d2c5000cd27bd9 F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 1458c3e33c1ec3ad99284f9692edfc49c44afd7c +F ext/fts5/fts5_expr.c 99560f28339b635ba0e0b13f80586c0be58fc680 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test 3343a05f7b1d3fc6fa4559bd3bd0ee979928c30d +F ext/fts5/test/fts5synonym.test a2b0fb9a584417a9c02554aa465ed9084653cdde F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 98d07d16cab92f1e7001afbe370df3ec6343fc1f -R e2a3883c4d0440bf5b53c5a71ba46260 +P ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3 +R 7ab2a9e80bd7422fd7466024fe59c669 U dan -Z 7c01c61f2a729597f3365ce7db7a6ce6 +Z 679d87b3074cd417fe8a5b64a65c64bd diff --git a/manifest.uuid b/manifest.uuid index b0ebea4bef..2ddf214da5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3 \ No newline at end of file +dbcb73802b88f76be17f09f3eb83ffac542de633 \ No newline at end of file From d917ad019c446dedc74abf76cd49da512ed0fcbb Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 1 Sep 2015 18:44:33 +0000 Subject: [PATCH 06/18] Fix a problem that occurs when more than 4 synonyms for a term appear within a single row. FossilOrigin-Name: cd359550bdc2bf7be4c294b60130c9fc3f583902 --- ext/fts5/fts5_expr.c | 16 ++++++- ext/fts5/test/fts5synonym.test | 88 ++++++++++++++++++++++++++++++++++ manifest | 14 +++--- manifest.uuid | 2 +- 4 files changed, 111 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 24d63c1cab..a44c4e4427 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -395,6 +395,7 @@ static int fts5ExprSynonymPoslist( Fts5PoslistReader aStatic[4]; Fts5PoslistReader *aIter = aStatic; int nIter = 0; + int nAlloc = 4; int rc = SQLITE_OK; Fts5ExprTerm *p; @@ -406,7 +407,18 @@ static int fts5ExprSynonymPoslist( int n; i64 dummy; rc = sqlite3Fts5IterPoslist(pIter, &a, &n, &dummy); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ) goto synonym_poslist_out; + if( nIter==nAlloc ){ + int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; + Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + goto synonym_poslist_out; + } + memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter); + nAlloc = nAlloc*2; + aIter = aNew; + } if( sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter])==0 ){ nIter++; } @@ -447,6 +459,8 @@ static int fts5ExprSynonymPoslist( } } + synonym_poslist_out: + if( aIter!=aStatic ) sqlite3_free(aIter); return rc; } diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 5aa29e003a..ddd685c155 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -280,6 +280,17 @@ foreach {tn q res} { 8 {2 ii i two 3 three 2} {two [iv] [v] iii 3 [five]} 9 {i 2 [iv] 3 [five] [four] [v]} {iii [4] three i three ii 1} } + 3 {one OR two OR iii OR 4 OR v} { + 1 {[four] [v] [4] [i] [three]} {[1] [3] [five] [five] [4] [one]} + 2 {[5] [1] [3] [4] [i]} {[2] [2] [v] [two] [4]} + 3 {[5] [i] [5] [2] [four] [4] [1]} {[iii] [ii] [five] [two] [1]} + 4 {[ii] [four] [4] [one] [5] [three] [five]} {[one] [5] [1] [iii] [4] [3]} + 5 {[three] [i] [v] [i] [four] [4] [1]} {[ii] [five] [five] [five] [iii]} + 6 {[4] [2] [ii] [two] [2] [iii]} {[three] [1] [four] [4] [iv] [1] [iv]} + 7 {[ii] [ii] [two] [three] [2] [5]} {[iii] [i] [ii] [iii] [iii] [one] [one]} + 8 {[2] [ii] [i] [two] [3] [three] [2]} {[two] [iv] [v] [iii] [3] [five]} + 9 {[i] [2] [iv] [3] [five] [four] [v]} {[iii] [4] [three] [i] [three] [ii] [1]} + } } { do_execsql_test 5.1.$tn { SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']') @@ -287,5 +298,82 @@ foreach {tn q res} { } $res } +#------------------------------------------------------------------------- +# Test terms with more than 4 synonyms. +# +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [do_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags=="query" && [string length $w]==1} { + for {set i 2} {$i<=10} {incr i} { + sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd + } + } + } +} + +do_test 6.0 { + execsql { + CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl) + } + foreach {rowid a b} { + 1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq aaaa} + 2 {ww oooooo bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq} + 3 {zzzz llll gggggg cccc uu} {hhhhhh aaaa ppppp rr ee jjjj} + 4 {r f i rrrrrr ww hhh} {aa yyy t x aaaaa ii} + 5 {fffff mm vvvv ooo ffffff kkkk tttt} {cccccc bb e zzz d n} + 6 {iii dddd hh qqqq ddd ooo} {ttt d c b aaaaaa qqqq} + 7 {jjjj rrrr v zzzzz u tt t} {ppppp pp dddd mm hhh uuu} + 8 {gggg rrrrrr kkkk vvvv gggg jjjjjj b} {dddddd jj r w cccc wwwwww ss} + 9 {kkkkk qqq oooo e tttttt mmm} {e ss qqqqqq hhhh llllll gg} + } { + execsql { INSERT INTO t2(rowid, a, b) VALUES($rowid, $a, $b) } + } +} {} + +foreach {tn q res} { + 1 {a} { + 1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq [aaaa]} + 3 {zzzz llll gggggg cccc uu} {hhhhhh [aaaa] ppppp rr ee jjjj} + 4 {r f i rrrrrr ww hhh} {[aa] yyy t x [aaaaa] ii} + 6 {iii dddd hh qqqq ddd ooo} {ttt d c b [aaaaaa] qqqq} + } + + 2 {a AND q} { + 1 {yyyy vvvvv [qq] oo yyyyyy vvvv eee} {ffff uu r [qq] [aaaa]} + 6 {iii dddd hh [qqqq] ddd ooo} {ttt d c b [aaaaaa] [qqqq]} + } + + 3 {o OR (q AND a)} { + 1 {yyyy vvvvv [qq] [oo] yyyyyy vvvv eee} {ffff uu r [qq] [aaaa]} + 2 {ww [oooooo] bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq} + 5 {fffff mm vvvv [ooo] ffffff kkkk tttt} {cccccc bb e zzz d n} + 6 {iii dddd hh [qqqq] ddd [ooo]} {ttt d c b [aaaaaa] [qqqq]} + 9 {kkkkk qqq [oooo] e tttttt mmm} {e ss qqqqqq hhhh llllll gg} + } +} { + do_execsql_test 6.1.$tn { + SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']') + FROM t2 WHERE t2 MATCH $q + } $res +} + +do_execsql_test 6.2.1 { + INSERT INTO t2(rowid, a, b) VALUES(13, + 'x xx xxx xxxx xxxxx xxxxxx xxxxxxx', 'y yy yyy yyyy yyyyy yyyyyy yyyyyyy' + ); + SELECT rowid, highlight(t2, 0, '<', '>'), highlight(t2, 1, '(', ')') + FROM t2 WHERE t2 MATCH 'x OR y' +} { + 1 { vvvvv qq oo vvvv eee} {ffff uu r qq aaaa} + 2 {ww oooooo bbbbb ssssss mm} {ffffff (yy) iiii rr s ccc qqqqq} + 4 {r f i rrrrrr ww hhh} {aa (yyy) t (x) aaaaa ii} + 13 { } + {(y) (yy) (yyy) (yyyy) (yyyyy) (yyyyyy) (yyyyyyy)} +} + + finish_test diff --git a/manifest b/manifest index 4a0414c4b0..55a78b93ad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\sfts5\ssynonyms\simplemented\sby\sadding\sextra\sterms\sto\squeries.\sAnd\sfixes\sfor\sthe\ssame. -D 2015-09-01T18:08:36.324 +C Fix\sa\sproblem\sthat\soccurs\swhen\smore\sthan\s4\ssynonyms\sfor\sa\sterm\sappear\swithin\sa\ssingle\srow. +D 2015-09-01T18:44:33.194 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h d46f89aeb357fbcf5b268d71b0d2c5000cd27bd9 F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 99560f28339b635ba0e0b13f80586c0be58fc680 +F ext/fts5/fts5_expr.c 44caa0ccd7e9a392864fda0c14e9b9829a395a84 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test a2b0fb9a584417a9c02554aa465ed9084653cdde +F ext/fts5/test/fts5synonym.test 4321e59c29186d9187cf8ab576e70530176eae49 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3 -R 7ab2a9e80bd7422fd7466024fe59c669 +P dbcb73802b88f76be17f09f3eb83ffac542de633 +R 07c5ba5d7a2806d5caa7dc123b3f32ad U dan -Z 679d87b3074cd417fe8a5b64a65c64bd +Z fc32f7df10846658b7b8e62d59624854 diff --git a/manifest.uuid b/manifest.uuid index 2ddf214da5..6c61644fa9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dbcb73802b88f76be17f09f3eb83ffac542de633 \ No newline at end of file +cd359550bdc2bf7be4c294b60130c9fc3f583902 \ No newline at end of file From df998c3d4f05e59411db9abca726ae2cc81d2d5c Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 2 Sep 2015 08:22:41 +0000 Subject: [PATCH 07/18] Fix a problem with fts5 synonyms and the xQueryPhrase() auxiliary function API. FossilOrigin-Name: cf3e45e76d23e10ee06296c3561a341591597a04 --- ext/fts5/fts5Int.h | 2 +- ext/fts5/fts5_expr.c | 156 +++++++++++++++++---------------- ext/fts5/fts5_main.c | 2 +- ext/fts5/test/fts5synonym.test | 26 ++++++ manifest | 18 ++-- manifest.uuid | 2 +- 6 files changed, 118 insertions(+), 88 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 1ad61617d6..1da017fb01 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -599,7 +599,7 @@ int sqlite3Fts5ExprPhraseCount(Fts5Expr*); int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); -int sqlite3Fts5ExprPhraseExpr(Fts5Config*, Fts5Expr*, int, Fts5Expr**); +int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**); /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index a44c4e4427..d4fd6c3634 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -251,79 +251,6 @@ int sqlite3Fts5ExprNew( return sParse.rc; } -/* -** Create a new FTS5 expression by cloning phrase iPhrase of the -** expression passed as the second argument. -*/ -int sqlite3Fts5ExprPhraseExpr( - Fts5Config *pConfig, - Fts5Expr *pExpr, - int iPhrase, - Fts5Expr **ppNew -){ - int rc = SQLITE_OK; /* Return code */ - Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ - Fts5ExprPhrase *pCopy; /* Copy of pOrig */ - Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ - - pOrig = pExpr->apExprPhrase[iPhrase]; - pCopy = (Fts5ExprPhrase*)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm - ); - if( pCopy ){ - int i; /* Used to iterate through phrase terms */ - Fts5ExprPhrase **apPhrase; - Fts5ExprNode *pNode; - Fts5ExprNearset *pNear; - - pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); - apPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprPhrase*) - ); - pNode = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprNode)); - pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*) - ); - - for(i=0; inTerm; i++){ - pCopy->aTerm[i].zTerm = sqlite3Fts5Strndup(&rc, pOrig->aTerm[i].zTerm,-1); - pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; - } - - if( rc==SQLITE_OK ){ - /* All the allocations succeeded. Put the expression object together. */ - pNew->pIndex = pExpr->pIndex; - pNew->pRoot = pNode; - pNew->nPhrase = 1; - pNew->apExprPhrase = apPhrase; - pNew->apExprPhrase[0] = pCopy; - - pNode->eType = (pOrig->nTerm==1 ? FTS5_TERM : FTS5_STRING); - pNode->pNear = pNear; - - pNear->nPhrase = 1; - pNear->apPhrase[0] = pCopy; - - pCopy->nTerm = pOrig->nTerm; - pCopy->pNode = pNode; - }else{ - /* At least one allocation failed. Free them all. */ - for(i=0; inTerm; i++){ - sqlite3_free(pCopy->aTerm[i].zTerm); - } - sqlite3_free(pCopy); - sqlite3_free(pNear); - sqlite3_free(pNode); - sqlite3_free(apPhrase); - sqlite3_free(pNew); - pNew = 0; - } - } - - *ppNew = pNew; - return rc; -} - /* ** Free the expression node object passed as the only argument. */ @@ -1550,8 +1477,8 @@ static int fts5ParseTokenize( int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ - int iStart, /* Start offset of token */ - int iEnd /* End offset of token */ + int iUnused1, /* Start offset of token */ + int iUnused2 /* End offset of token */ ){ int rc = SQLITE_OK; const int SZALLOC = 8; @@ -1577,7 +1504,7 @@ static int fts5ParseTokenize( pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew - ); + ); if( pNew==0 ) return SQLITE_NOMEM; if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); pCtx->pPhrase = pPhrase = pNew; @@ -1675,6 +1602,83 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( return sCtx.pPhrase; } +/* +** Create a new FTS5 expression by cloning phrase iPhrase of the +** expression passed as the second argument. +*/ +int sqlite3Fts5ExprClonePhrase( + Fts5Config *pConfig, + Fts5Expr *pExpr, + int iPhrase, + Fts5Expr **ppNew +){ + int rc = SQLITE_OK; /* Return code */ + Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ + Fts5ExprPhrase *pCopy; /* Copy of pOrig */ + int i; /* Used to iterate through phrase terms */ + + Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ + Fts5ExprPhrase **apPhrase; /* pNew->apPhrase */ + Fts5ExprNode *pNode; /* pNew->pRoot */ + Fts5ExprNearset *pNear; /* pNew->pRoot->pNear */ + + TokenCtx sCtx = {0}; /* Context object for fts5ParseTokenize */ + + + pOrig = pExpr->apExprPhrase[iPhrase]; + + pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + if( rc==SQLITE_OK ){ + pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprPhrase*)); + } + if( rc==SQLITE_OK ){ + pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprNode)); + } + if( rc==SQLITE_OK ){ + pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); + } + + for(i=0; rc==SQLITE_OK && inTerm; i++){ + int tflags = 0; + Fts5ExprTerm *p; + for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ + const char *zTerm = p->zTerm; + rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, strlen(zTerm), 0, 0); + tflags = FTS5_TOKEN_COLOCATED; + } + if( rc==SQLITE_OK ){ + sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + } + } + + if( rc==SQLITE_OK ){ + /* All the allocations succeeded. Put the expression object together. */ + pNew->pIndex = pExpr->pIndex; + pNew->nPhrase = 1; + pNew->apExprPhrase[0] = sCtx.pPhrase; + pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; + pNew->pRoot->pNear->nPhrase = 1; + sCtx.pPhrase->pNode = pNew->pRoot; + + if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){ + pNew->pRoot->eType = FTS5_TERM; + }else{ + pNew->pRoot->eType = FTS5_STRING; + } + }else{ + sqlite3Fts5ExprFree(pNew); + fts5ExprPhraseFree(sCtx.pPhrase); + pNew = 0; + } + + *ppNew = pNew; + return rc; +} + + /* ** Token pTok has appeared in a MATCH expression where the NEAR operator ** is expected. If token pTok does not contain "NEAR", store an error diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 4704fb8906..f337005d00 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1860,7 +1860,7 @@ static int fts5ApiQueryPhrase( pNew->iFirstRowid = SMALLEST_INT64; pNew->iLastRowid = LARGEST_INT64; pNew->base.pVtab = (sqlite3_vtab*)pTab; - rc = sqlite3Fts5ExprPhraseExpr(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr); + rc = sqlite3Fts5ExprClonePhrase(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr); } if( rc==SQLITE_OK ){ diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index ddd685c155..7bc3ffaf90 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -298,6 +298,32 @@ foreach {tn q res} { } $res } +# Test that the xQueryPhrase() API works with synonyms. +# +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit +sqlite3_fts5_register_matchinfo db + +foreach {tn q res} { + 1 {one} { + 1 {1 11 7 2 12 6} 2 {2 11 7 0 12 6} + 3 {2 11 7 1 12 6} 4 {1 11 7 2 12 6} + 5 {3 11 7 0 12 6} 6 {0 11 7 2 12 6} + 7 {0 11 7 3 12 6} 8 {1 11 7 0 12 6} + 9 {1 11 7 2 12 6} + } +} { + do_execsql_test 5.2.$tn { + SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH $q + } $res +} + + #------------------------------------------------------------------------- # Test terms with more than 4 synonyms. # diff --git a/manifest b/manifest index 55a78b93ad..c5af116d50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sthat\soccurs\swhen\smore\sthan\s4\ssynonyms\sfor\sa\sterm\sappear\swithin\sa\ssingle\srow. -D 2015-09-01T18:44:33.194 +C Fix\sa\sproblem\swith\sfts5\ssynonyms\sand\sthe\sxQueryPhrase()\sauxiliary\sfunction\sAPI. +D 2015-09-02T08:22:41.779 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -106,14 +106,14 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42 -F ext/fts5/fts5Int.h d46f89aeb357fbcf5b268d71b0d2c5000cd27bd9 +F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 44caa0ccd7e9a392864fda0c14e9b9829a395a84 +F ext/fts5/fts5_expr.c d92beea335f96f9256cc19203b21fd45826655c9 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 -F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42 +F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e F ext/fts5/fts5_storage.c c888defbb961d64c12299b3d1725a24a770b047e F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 F ext/fts5/fts5_test_mi.c 80a9e86fb4c5b6b58f8fefac05e9b96d1a6574e1 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test 4321e59c29186d9187cf8ab576e70530176eae49 +F ext/fts5/test/fts5synonym.test fd66afccec36bf719ba66a5b6579efe007607b05 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P dbcb73802b88f76be17f09f3eb83ffac542de633 -R 07c5ba5d7a2806d5caa7dc123b3f32ad +P cd359550bdc2bf7be4c294b60130c9fc3f583902 +R f0fefb84cb3010de72f87418a092a81a U dan -Z fc32f7df10846658b7b8e62d59624854 +Z b6230653499a35c239ca936f1dd1428a diff --git a/manifest.uuid b/manifest.uuid index 6c61644fa9..7c523ff1d0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cd359550bdc2bf7be4c294b60130c9fc3f583902 \ No newline at end of file +cf3e45e76d23e10ee06296c3561a341591597a04 \ No newline at end of file From 583611df65d799dec0b86cec7627106b9ddf0c81 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 2 Sep 2015 14:17:38 +0000 Subject: [PATCH 08/18] Fix a problem handling OOM conditions within fts5 queries that feature synonyms. FossilOrigin-Name: 11fa980897c6c7be218bbd9c4cd8253272d2c300 --- ext/fts5/fts5_expr.c | 44 +++++++++++++++-------- ext/fts5/fts5_test_mi.c | 13 ++++--- ext/fts5/test/fts5_common.tcl | 33 +++++++++++++++++ ext/fts5/test/fts5fault6.test | 66 ++++++++++++++++++++++++++++++++++ ext/fts5/test/fts5synonym.test | 40 ++++----------------- manifest | 20 +++++------ manifest.uuid | 2 +- 7 files changed, 155 insertions(+), 63 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index d4fd6c3634..cc24179561 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1467,6 +1467,7 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset( typedef struct TokenCtx TokenCtx; struct TokenCtx { Fts5ExprPhrase *pPhrase; + int rc; }; /* @@ -1485,17 +1486,23 @@ static int fts5ParseTokenize( TokenCtx *pCtx = (TokenCtx*)pContext; Fts5ExprPhrase *pPhrase = pCtx->pPhrase; + /* If an error has already occurred, this is a no-op */ + if( pCtx->rc!=SQLITE_OK ) return pCtx->rc; + assert( pPhrase==0 || pPhrase->nTerm>0 ); if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){ Fts5ExprTerm *pSyn; int nByte = sizeof(Fts5ExprTerm) + nToken+1; pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte); - if( pSyn==0 ) return SQLITE_NOMEM; - memset(pSyn, 0, nByte); - pSyn->zTerm = (char*)&pSyn[1]; - memcpy(pSyn->zTerm, pToken, nToken); - pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; - pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; + if( pSyn==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pSyn, 0, nByte); + pSyn->zTerm = (char*)&pSyn[1]; + memcpy(pSyn->zTerm, pToken, nToken); + pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; + pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; + } }else{ Fts5ExprTerm *pTerm; if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){ @@ -1505,16 +1512,23 @@ static int fts5ParseTokenize( pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew ); - if( pNew==0 ) return SQLITE_NOMEM; - if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); - pCtx->pPhrase = pPhrase = pNew; - pNew->nTerm = nNew - SZALLOC; + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); + pCtx->pPhrase = pPhrase = pNew; + pNew->nTerm = nNew - SZALLOC; + } } - pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; - memset(pTerm, 0, sizeof(Fts5ExprTerm)); - pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); + if( rc==SQLITE_OK ){ + pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; + memset(pTerm, 0, sizeof(Fts5ExprTerm)); + pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); + } } + + pCtx->rc = rc; return rc; } @@ -1573,7 +1587,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize); } sqlite3_free(z); - if( rc ){ + if( rc || (rc = sCtx.rc) ){ pParse->rc = rc; fts5ExprPhraseFree(sCtx.pPhrase); sCtx.pPhrase = 0; @@ -1622,7 +1636,7 @@ int sqlite3Fts5ExprClonePhrase( Fts5ExprNode *pNode; /* pNew->pRoot */ Fts5ExprNearset *pNear; /* pNew->pRoot->pNear */ - TokenCtx sCtx = {0}; /* Context object for fts5ParseTokenize */ + TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */ pOrig = pExpr->apExprPhrase[iPhrase]; diff --git a/ext/fts5/fts5_test_mi.c b/ext/fts5/fts5_test_mi.c index 8ebf5f5077..355f23330d 100644 --- a/ext/fts5/fts5_test_mi.c +++ b/ext/fts5/fts5_test_mi.c @@ -352,7 +352,7 @@ static void fts5MatchinfoFunc( ){ const char *zArg; Fts5MatchinfoCtx *p; - int rc; + int rc = SQLITE_OK; if( nVal>0 ){ zArg = (const char*)sqlite3_value_text(apVal[0]); @@ -363,11 +363,16 @@ static void fts5MatchinfoFunc( p = (Fts5MatchinfoCtx*)pApi->xGetAuxdata(pFts, 0); if( p==0 || sqlite3_stricmp(zArg, p->zArg) ){ p = fts5MatchinfoNew(pApi, pFts, pCtx, zArg); - pApi->xSetAuxdata(pFts, p, sqlite3_free); - if( p==0 ) return; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = pApi->xSetAuxdata(pFts, p, sqlite3_free); + } } - rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb); + if( rc==SQLITE_OK ){ + rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb); + } if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pCtx, rc); }else{ diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 59058a8462..5eba1296ad 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -295,3 +295,36 @@ proc NOT {a b} { return $a } +#------------------------------------------------------------------------- +# This command is similar to [split], except that it also provides the +# start and end offsets of each token. For example: +# +# [fts5_tokenize_split "abc d ef"] -> {abc 0 3 d 4 5 ef 6 8} +# + +proc gobble_whitespace {textvar} { + upvar $textvar t + regexp {([ ]*)(.*)} $t -> space t + return [string length $space] +} + +proc gobble_text {textvar wordvar} { + upvar $textvar t + upvar $wordvar w + regexp {([^ ]*)(.*)} $t -> w t + return [string length $w] +} + +proc fts5_tokenize_split {text} { + set token "" + set ret [list] + set iOff [gobble_whitespace text] + while {[set nToken [gobble_text text word]]} { + lappend ret $word $iOff [expr $iOff+$nToken] + incr iOff $nToken + incr iOff [gobble_whitespace text] + } + + set ret +} + diff --git a/ext/fts5/test/fts5fault6.test b/ext/fts5/test/fts5fault6.test index b9657be1cc..b92df3ce79 100644 --- a/ext/fts5/test/fts5fault6.test +++ b/ext/fts5/test/fts5fault6.test @@ -148,5 +148,71 @@ do_faultsim_test 4.1 -faults oom-t* -prep { faultsim_test_result {0 {}} } +#------------------------------------------------------------------------- +# OOM while running a query that includes synonyms and matchinfo(). +# +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags=="query" && [string length $w]==1} { + for {set i 2} {$i < 7} {incr i} { + sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd + } + } + } +} +proc tcl_create {args} { return "tcl_tokenize" } +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +db func mit mit +sqlite3_fts5_register_matchinfo db +do_test 5.0 { + execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl) } + foreach {rowid text} { + 1 {aaaa cc b aaaaa cc aa} + 2 {aa aa bb a bbb} + 3 {bb aaaaa aaaaa b aaaa aaaaa} + 4 {aa a b aaaa aa} + 5 {aa b ccc aaaaa cc} + 6 {aa aaaaa bbbb cc aaa} + 7 {aaaaa aa aa ccccc bb} + 8 {ccc bbbbb ccccc bbb c} + 9 {cccccc bbbb a aaa cccc c} + } { + execsql { INSERT INTO t1(rowid, a) VALUES($rowid, $text) } + } +} {} + +set res [list {*}{ + 1 {3 24 8 2 12 6} + 5 {2 24 8 2 12 6} + 6 {3 24 8 1 12 6} + 7 {3 24 8 1 12 6} + 9 {2 24 8 3 12 6} +}] +do_execsql_test 5.1 { + SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c' +} $res + +faultsim_save_and_close +do_faultsim_test 5.2 -faults oom* -prep { + faultsim_restore_and_reopen + sqlite3_fts5_create_tokenizer db tcl tcl_create + sqlite3_fts5_register_matchinfo db + db func mit mit +} -body { + db eval { + SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c' + } +} -test { + faultsim_test_result [list 0 $::res] +} + finish_test diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 7bc3ffaf90..6acc5617f0 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -40,34 +40,8 @@ foreach S { } } -proc gobble_whitespace {textvar} { - upvar $textvar t - regexp {([ ]*)(.*)} $t -> space t - return [string length $space] -} - -proc gobble_text {textvar wordvar} { - upvar $textvar t - upvar $wordvar w - regexp {([^ ]*)(.*)} $t -> w t - return [string length $w] -} - -proc do_tokenize_split {text} { - set token "" - set ret [list] - set iOff [gobble_whitespace text] - while {[set nToken [gobble_text text word]]} { - lappend ret $word $iOff [expr $iOff+$nToken] - incr iOff $nToken - incr iOff [gobble_whitespace text] - } - - set ret -} - proc tcl_tokenize {tflags text} { - foreach {w iStart iEnd} [do_tokenize_split $text] { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { sqlite3_fts5_token $w $iStart $iEnd } } @@ -95,7 +69,7 @@ do_execsql_test 1.0 { # proc tcl_tokenize {tflags text} { - foreach {w iStart iEnd} [do_tokenize_split $text] { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { sqlite3_fts5_token $w $iStart $iEnd if {$tflags=="document" && [info exists ::syn($w)]} { foreach s $::syn($w) { @@ -137,7 +111,7 @@ reset_db sqlite3_fts5_create_tokenizer db tcl tcl_create proc tcl_tokenize {tflags text} { set bColo 1 - foreach {w iStart iEnd} [do_tokenize_split $text] { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { if {$bColo} { sqlite3_fts5_token -colo $w $iStart $iEnd set bColo 0 @@ -160,7 +134,7 @@ do_execsql_test 3.1.1 { } {} proc tcl_tokenize {tflags text} { - foreach {w iStart iEnd} [do_tokenize_split $text] { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { sqlite3_fts5_token $w $iStart $iEnd } } @@ -172,7 +146,7 @@ do_execsql_test 3.1.2 { reset_db sqlite3_fts5_create_tokenizer db tcl tcl_create proc tcl_tokenize {tflags text} { - foreach {w iStart iEnd} [do_tokenize_split $text] { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { sqlite3_fts5_token $w $iStart $iEnd sqlite3_fts5_token -colo $w $iStart $iEnd } @@ -207,7 +181,7 @@ do_execsql_test 3.2.5 { reset_db sqlite3_fts5_create_tokenizer db tcl tcl_create proc tcl_tokenize {tflags text} { - foreach {w iStart iEnd} [do_tokenize_split $text] { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { sqlite3_fts5_token $w $iStart $iEnd if {$tflags=="query" && [info exists ::syn($w)]} { foreach s $::syn($w) { @@ -330,7 +304,7 @@ foreach {tn q res} { reset_db sqlite3_fts5_create_tokenizer db tcl tcl_create proc tcl_tokenize {tflags text} { - foreach {w iStart iEnd} [do_tokenize_split $text] { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { sqlite3_fts5_token $w $iStart $iEnd if {$tflags=="query" && [string length $w]==1} { for {set i 2} {$i<=10} {incr i} { diff --git a/manifest b/manifest index c5af116d50..01f188aef6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sfts5\ssynonyms\sand\sthe\sxQueryPhrase()\sauxiliary\sfunction\sAPI. -D 2015-09-02T08:22:41.779 +C Fix\sa\sproblem\shandling\sOOM\sconditions\swithin\sfts5\squeries\sthat\sfeature\ssynonyms. +D 2015-09-02T14:17:38.670 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,20 +110,20 @@ F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c d92beea335f96f9256cc19203b21fd45826655c9 +F ext/fts5/fts5_expr.c 5ae9bae1a36583a1e85f742f972c65bffd7e9d3b F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e F ext/fts5/fts5_storage.c c888defbb961d64c12299b3d1725a24a770b047e F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 -F ext/fts5/fts5_test_mi.c 80a9e86fb4c5b6b58f8fefac05e9b96d1a6574e1 +F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf F ext/fts5/fts5_tokenize.c 710541513ecf3fe6d9365326fc85aee6efe97229 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl 3338968de1880ca12b0451ae8f9b8b12d14e0ba7 +F ext/fts5/test/fts5_common.tcl b6e6a40ef5d069c8e86ca4fbad491e1195485dbc F ext/fts5/test/fts5aa.test caa44c528f7270aa4e325c4f2c28d355c1e8c307 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c @@ -156,7 +156,7 @@ F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7 F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875 -F ext/fts5/test/fts5fault6.test 234dc6355f8d3f8b5be2763f30699d770247c215 +F ext/fts5/test/fts5fault6.test 233a29d4a086cd09cfae262bbb15c709a438d010 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6 F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test fd66afccec36bf719ba66a5b6579efe007607b05 +F ext/fts5/test/fts5synonym.test dd3c9016345e1bbd7384ec25bbcd3de1f1700475 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cd359550bdc2bf7be4c294b60130c9fc3f583902 -R f0fefb84cb3010de72f87418a092a81a +P cf3e45e76d23e10ee06296c3561a341591597a04 +R 0e89c166acdebcc1b36d66580652787d U dan -Z b6230653499a35c239ca936f1dd1428a +Z 84847a00fa2f61f76712fa9a6aa07879 diff --git a/manifest.uuid b/manifest.uuid index 7c523ff1d0..985c544ce9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cf3e45e76d23e10ee06296c3561a341591597a04 \ No newline at end of file +11fa980897c6c7be218bbd9c4cd8253272d2c300 \ No newline at end of file From bea34fc53af825712a983dd3f1a717fcb9e6c03d Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 2 Sep 2015 17:34:22 +0000 Subject: [PATCH 09/18] Fix a problem with fts5 synonyms and phrase queries. Also fix an OOM handling bug in fts5. FossilOrigin-Name: a4c35fa2c94fe34b376670244fe72303c99868c1 --- ext/fts5/fts5_expr.c | 61 +++++++++++++++++++++++----------- ext/fts5/test/fts5fault6.test | 58 ++++++++++++++++++++++++++++++-- ext/fts5/test/fts5synonym.test | 19 +++++++++++ main.mk | 46 +++++++++++++------------ manifest | 18 +++++----- manifest.uuid | 2 +- 6 files changed, 151 insertions(+), 53 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index cc24179561..8796ba5e2d 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -288,8 +288,8 @@ static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){ ** Argument pTerm must be a synonym iterator. Return the current rowid ** that it points to. */ -static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){ - i64 iRet; +static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ + i64 iRet = 0; int bRetValid = 0; Fts5ExprTerm *p; @@ -305,7 +305,7 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){ } } - assert( bRetValid ); + if( pbEof && bRetValid==0 ) *pbEof = 1; return iRet; } @@ -346,9 +346,9 @@ static int fts5ExprSynonymPoslist( nAlloc = nAlloc*2; aIter = aNew; } - if( sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter])==0 ){ - nIter++; - } + sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter]); + assert( aIter[nIter].bEof==0 ); + nIter++; } } @@ -659,7 +659,7 @@ static int fts5ExprNearAdvanceFirst( Fts5ExprTerm *p; /* Find the firstest rowid any synonym points to. */ - i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc); + i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0); /* Advance each iterator that currently points to iRowid. Or, if iFrom ** is valid - each iterator that points to a rowid before iFrom. */ @@ -738,6 +738,35 @@ static int fts5ExprAdvanceto( return 0; } +static int fts5ExprSynonymAdvanceto( + Fts5ExprTerm *pTerm, /* Term iterator to advance */ + int bDesc, /* True if iterator is "rowid DESC" */ + i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ + int *pRc, /* OUT: Error code */ + int *pbEof /* OUT: Set to true if EOF */ +){ + int rc = SQLITE_OK; + i64 iLast = *piLast; + Fts5ExprTerm *p; + + for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){ + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + i64 iRowid = sqlite3Fts5IterRowid(p->pIter); + if( (bDesc==0 && iLast>iRowid) || (bDesc && iLastpIter, iLast); + } + } + } + + if( rc!=SQLITE_OK ){ + *pbEof = 1; + }else{ + *piLast = fts5ExprSynonymRowid(pTerm, bDesc, pbEof); + } + + return rc; +} + /* ** IN/OUT parameter (*pa) points to a position list n bytes in size. If ** the position list contains entries for column iCol, then (*pa) is set @@ -906,7 +935,7 @@ static int fts5ExprNearNextMatch( ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it ** means the minimum rowid. */ if( pLeft->aTerm[0].pSynonym ){ - iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc); + iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0); }else{ iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter); } @@ -920,21 +949,13 @@ static int fts5ExprNearNextMatch( if( pTerm->pSynonym ){ Fts5ExprTerm *p; int bEof = 1; - i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc); + i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0); if( iRowid==iLast ) continue; - for(p=pTerm; p; p=p->pSynonym){ - Fts5IndexIter *pIter = p->pIter; - int dummy; - if( 0==sqlite3Fts5IterEof(pIter) - && 0==fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &dummy)==0 - ){ - bEof = 0; - } - } - if( bEof || rc ){ - pNode->bEof = 1; + bMatch = 0; + if( fts5ExprSynonymAdvanceto(pTerm,bDesc,&iLast,&rc,&pNode->bEof) ){ return rc; } + }else{ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; i64 iRowid = sqlite3Fts5IterRowid(pIter); diff --git a/ext/fts5/test/fts5fault6.test b/ext/fts5/test/fts5fault6.test index b92df3ce79..b739e87418 100644 --- a/ext/fts5/test/fts5fault6.test +++ b/ext/fts5/test/fts5fault6.test @@ -22,6 +22,8 @@ ifcapable !fts5 { return } +if 1 { + #------------------------------------------------------------------------- # OOM while rebuilding an FTS5 table. # @@ -148,8 +150,14 @@ do_faultsim_test 4.1 -faults oom-t* -prep { faultsim_test_result {0 {}} } +} + #------------------------------------------------------------------------- -# OOM while running a query that includes synonyms and matchinfo(). +# +# 5.2.* OOM while running a query that includes synonyms and matchinfo(). +# +# 5.3.* OOM while running a query that returns a row containing instances +# of more than 4 synonyms for a single term. # proc mit {blob} { set scan(littleEndian) i* @@ -174,6 +182,7 @@ db func mit mit sqlite3_fts5_register_matchinfo db do_test 5.0 { execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl) } + execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) } foreach {rowid text} { 1 {aaaa cc b aaaaa cc aa} 2 {aa aa bb a bbb} @@ -184,6 +193,37 @@ do_test 5.0 { 7 {aaaaa aa aa ccccc bb} 8 {ccc bbbbb ccccc bbb c} 9 {cccccc bbbb a aaa cccc c} + + 20 {ddd f ddd eeeee fff ffff eeee ddd fff eeeee dddddd eeee} + 21 {fffff eee dddd fffff dd ee ee eeeee eee eeeeee ee dd e} + 22 {fffff d eeee dddd fffff dddddd ffff ddddd eeeee ee eee dddd ddddd} + 23 {ddddd fff ddd eeeee ffff eeee ddd ff ff ffffff eeeeee dddd ffffff} + 24 {eee dd ee dddd dddd eeeeee e eee fff ffff} + 25 {ddddd ffffff dddddd fff ddd ddddd ddd f eeee fff dddd f} + 26 {f ffff fff fff eeeeee dddd d dddddd ddddd eee ff eeeee} + 27 {eee fff dddddd eeeee eeeee dddd ddddd ffff f eeeee eee dddddd ddddd d} + 28 {dd ddddd d ddd d fff d dddd ee dddd ee ddd dddddd dddddd} + 29 {eeee dddd ee dddd eeee dddd dd fffff f ddd eeeee ddd ee} + 30 {ff ffffff eeeeee eeeee eee ffffff ff ffff f fffff eeeee} + 31 {fffff eeeeee dddd eeee eeee eeeeee eee fffff d ddddd ffffff ffff dddddd} + 32 {dddddd fffff ee eeeeee eeee ee fff dddd fff eeee ffffff eeeeee ffffff} + 33 {ddddd eeee dd ffff dddddd fff eeee ddddd ffff eeee ddd} + 34 {ee dddd ddddd dddddd eeee eeeeee f dd ee dddddd ffffff} + 35 {ee dddd dd eeeeee ddddd eee d eeeeee dddddd eee dddd fffff} + 36 {eee ffffff ffffff e fffff eeeee ff dddddd dddddd fff} + 37 {eeeee fffff dddddd dddd ffffff fff f dd ee dd dd eeeee} + 38 {eeeeee ee d ff eeeeee eeeeee eee eeeee ee ffffff dddd eeee dddddd ee} + 39 {eeeeee ddd fffff e dddd ee eee eee ffffff ee f d dddd} + 40 {ffffff dddddd eee ee ffffff eee eeee ddddd ee eeeeee f} + 41 {ddd ddd fff fffff ee fffff f fff ddddd fffff} + 42 {dddd ee ff d f ffffff fff ffffff ff dd dddddd f eeee} + 43 {d dd fff fffff d f fff e dddd ee ee} + 44 {ff ffff eee ddd d dd ffff dddd d eeee d eeeeee} + 45 {eeee f eeeee ee e ffff f ddd e fff} + 46 {ffff d ffff eeee ffff eeeee f ffff ddddd eee} + 47 {dd dd dddddd ddddd fffff dddddd ddd ddddd eeeeee ffff eeee eee ee} + 48 {ffff ffff e dddd ffffff dd dd dddd f fffff} + 49 {ffffff d dddddd ffff eeeee f ffff ffff d dd fffff eeeee} } { execsql { INSERT INTO t1(rowid, a) VALUES($rowid, $text) } } @@ -196,9 +236,12 @@ set res [list {*}{ 7 {3 24 8 1 12 6} 9 {2 24 8 3 12 6} }] -do_execsql_test 5.1 { +do_execsql_test 5.1.1 { SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c' } $res +do_execsql_test 5.1.2 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'd e f' +} 29 faultsim_save_and_close do_faultsim_test 5.2 -faults oom* -prep { @@ -214,5 +257,16 @@ do_faultsim_test 5.2 -faults oom* -prep { faultsim_test_result [list 0 $::res] } +do_faultsim_test 5.3 -faults oom* -prep { + faultsim_restore_and_reopen + sqlite3_fts5_create_tokenizer db tcl tcl_create +} -body { + db eval { + SELECT count(*) FROM t1 WHERE t1 MATCH 'd AND e AND f' + } +} -test { + faultsim_test_result {0 29} +} + finish_test diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 6acc5617f0..33d20a014e 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -93,6 +93,7 @@ foreach {tn expr res} { 2 "eight OR 8 OR 5" {2 3} 3 "10" {} 4 "1*" {1} + 5 "1 + 2" {1} } { do_execsql_test 2.1.$tn { SELECT rowid FROM ft WHERE ft MATCH $expr @@ -233,6 +234,7 @@ do_test 5.0 { } } {} + foreach {tn q res} { 1 {one} { 1 {four v 4 [i] three} {[1] 3 five five 4 [one]} @@ -265,6 +267,23 @@ foreach {tn q res} { 8 {[2] [ii] [i] [two] [3] [three] [2]} {[two] [iv] [v] [iii] [3] [five]} 9 {[i] [2] [iv] [3] [five] [four] [v]} {[iii] [4] [three] [i] [three] [ii] [1]} } + + 4 {5 + 1} { + 2 {[5 1] 3 4 i} {2 2 v two 4} + 3 {[5 i] 5 2 four 4 1} {iii ii five two 1} + 4 {ii four 4 one 5 three five} {one [5 1] iii 4 3} + 5 {three i [v i] four 4 1} {ii five five five iii} + } + + 5 {one + two + three} { + 7 {ii ii two three 2 5} {iii [i ii iii] iii one one} + 8 {2 ii [i two 3] three 2} {two iv v iii 3 five} + } + + 6 {"v v"} { + 1 {four v 4 i three} {1 3 [five five] 4 one} + 5 {three i v i four 4 1} {ii [five five five] iii} + } } { do_execsql_test 5.1.$tn { SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']') diff --git a/main.mk b/main.mk index 9988fe2775..828a824eee 100644 --- a/main.mk +++ b/main.mk @@ -47,6 +47,7 @@ TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 TCCX += -I$(TOP)/ext/async -I$(TOP)/ext/userauth +TCCX += -I$(TOP)/ext/fts5 # Object files for the SQLite library. # @@ -230,6 +231,29 @@ SRC += \ $(TOP)/ext/rbu/sqlite3rbu.h +# FTS5 things +# +FTS5_HDR = \ + $(TOP)/ext/fts5/fts5.h \ + $(TOP)/ext/fts5/fts5Int.h \ + fts5parse.h + +FTS5_SRC = \ + $(TOP)/ext/fts5/fts5_aux.c \ + $(TOP)/ext/fts5/fts5_buffer.c \ + $(TOP)/ext/fts5/fts5_main.c \ + $(TOP)/ext/fts5/fts5_config.c \ + $(TOP)/ext/fts5/fts5_expr.c \ + $(TOP)/ext/fts5/fts5_hash.c \ + $(TOP)/ext/fts5/fts5_index.c \ + fts5parse.c \ + $(TOP)/ext/fts5/fts5_storage.c \ + $(TOP)/ext/fts5/fts5_tokenize.c \ + $(TOP)/ext/fts5/fts5_unicode2.c \ + $(TOP)/ext/fts5/fts5_varint.c \ + $(TOP)/ext/fts5/fts5_vocab.c \ + + # Generated source code files # SRC += \ @@ -636,25 +660,6 @@ fts3_write.o: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR) rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c -# FTS5 things -# -FTS5_SRC = \ - $(TOP)/ext/fts5/fts5.h \ - $(TOP)/ext/fts5/fts5Int.h \ - $(TOP)/ext/fts5/fts5_aux.c \ - $(TOP)/ext/fts5/fts5_buffer.c \ - $(TOP)/ext/fts5/fts5_main.c \ - $(TOP)/ext/fts5/fts5_config.c \ - $(TOP)/ext/fts5/fts5_expr.c \ - $(TOP)/ext/fts5/fts5_hash.c \ - $(TOP)/ext/fts5/fts5_index.c \ - fts5parse.c fts5parse.h \ - $(TOP)/ext/fts5/fts5_storage.c \ - $(TOP)/ext/fts5/fts5_tokenize.c \ - $(TOP)/ext/fts5/fts5_unicode2.c \ - $(TOP)/ext/fts5/fts5_varint.c \ - $(TOP)/ext/fts5/fts5_vocab.c \ - fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon cp $(TOP)/ext/fts5/fts5parse.y . rm -f fts5parse.h @@ -662,11 +667,10 @@ fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon fts5parse.h: fts5parse.c -fts5.c: $(FTS5_SRC) +fts5.c: $(FTS5_SRC) $(FTS5_HDR) tclsh $(TOP)/ext/fts5/tool/mkfts5c.tcl cp $(TOP)/ext/fts5/fts5.h . - userauth.o: $(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c diff --git a/manifest b/manifest index 01f188aef6..30f63e1524 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\shandling\sOOM\sconditions\swithin\sfts5\squeries\sthat\sfeature\ssynonyms. -D 2015-09-02T14:17:38.670 +C Fix\sa\sproblem\swith\sfts5\ssynonyms\sand\sphrase\squeries.\sAlso\sfix\san\sOOM\shandling\sbug\sin\sfts5. +D 2015-09-02T17:34:22.663 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 5ae9bae1a36583a1e85f742f972c65bffd7e9d3b +F ext/fts5/fts5_expr.c 408384b2f540dcf72a363e1cde4c3e9896e8d41d F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e @@ -156,7 +156,7 @@ F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7 F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875 -F ext/fts5/test/fts5fault6.test 233a29d4a086cd09cfae262bbb15c709a438d010 +F ext/fts5/test/fts5fault6.test 93e9d43ab1e05ae5efa30cfde17649b3bb6a82c3 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6 F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test dd3c9016345e1bbd7384ec25bbcd3de1f1700475 +F ext/fts5/test/fts5synonym.test 88efc3de51a2c46aff984d74f06c833e4def0493 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -259,7 +259,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 04840e8277ab5159af16172eafd214dae7cffff5 +F main.mk 8a8468564b2e387116614fcda2a6d4f5f754b955 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cf3e45e76d23e10ee06296c3561a341591597a04 -R 0e89c166acdebcc1b36d66580652787d +P 11fa980897c6c7be218bbd9c4cd8253272d2c300 +R 6df2cc151a0891be96a1f7bf13aeaf04 U dan -Z 84847a00fa2f61f76712fa9a6aa07879 +Z 72b6e5381768fc37d01c74eff21e6a40 diff --git a/manifest.uuid b/manifest.uuid index 985c544ce9..2675a32e17 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -11fa980897c6c7be218bbd9c4cd8253272d2c300 \ No newline at end of file +a4c35fa2c94fe34b376670244fe72303c99868c1 \ No newline at end of file From 50ea76e748403b4f7af07137dc593f100e6a68f4 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 2 Sep 2015 18:56:01 +0000 Subject: [PATCH 10/18] Fix an issue with fts5 synonyms and NEAR(...) queries. FossilOrigin-Name: f2e590700d8eb30da8ac3a92f6ca92a2bbe9c1fe --- ext/fts5/fts5_expr.c | 16 ++++++++-------- ext/fts5/test/fts5synonym.test | 16 +++++++++++++++- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 8796ba5e2d..7cdb5e7632 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -742,12 +742,12 @@ static int fts5ExprSynonymAdvanceto( Fts5ExprTerm *pTerm, /* Term iterator to advance */ int bDesc, /* True if iterator is "rowid DESC" */ i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ - int *pRc, /* OUT: Error code */ - int *pbEof /* OUT: Set to true if EOF */ + int *pRc /* OUT: Error code */ ){ int rc = SQLITE_OK; i64 iLast = *piLast; Fts5ExprTerm *p; + int bEof = 0; for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){ if( sqlite3Fts5IterEof(p->pIter)==0 ){ @@ -759,12 +759,12 @@ static int fts5ExprSynonymAdvanceto( } if( rc!=SQLITE_OK ){ - *pbEof = 1; + *pRc = rc; + bEof = 1; }else{ - *piLast = fts5ExprSynonymRowid(pTerm, bDesc, pbEof); + *piLast = fts5ExprSynonymRowid(pTerm, bDesc, &bEof); } - - return rc; + return bEof; } /* @@ -952,10 +952,10 @@ static int fts5ExprNearNextMatch( i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0); if( iRowid==iLast ) continue; bMatch = 0; - if( fts5ExprSynonymAdvanceto(pTerm,bDesc,&iLast,&rc,&pNode->bEof) ){ + if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){ + pNode->bEof = 1; return rc; } - }else{ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; i64 iRowid = sqlite3Fts5IterRowid(pIter); diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 33d20a014e..a0fb43eca6 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -333,7 +333,16 @@ proc tcl_tokenize {tflags text} { } } -do_test 6.0 { +do_execsql_test 6.0.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl); + INSERT INTO t1 VALUES('yy xx qq'); + INSERT INTO t1 VALUES('yy xx xx'); +} +do_execsql_test 6.0.2 { + SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)'; +} {{yy xx qq}} + +do_test 6.0.3 { execsql { CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl) } @@ -372,6 +381,11 @@ foreach {tn q res} { 6 {iii dddd hh [qqqq] ddd [ooo]} {ttt d c b [aaaaaa] [qqqq]} 9 {kkkkk qqq [oooo] e tttttt mmm} {e ss qqqqqq hhhh llllll gg} } + + 4 {NEAR(q y, 20)} { + 1 {[yyyy] vvvvv [qq] oo [yyyyyy] vvvv eee} {ffff uu r qq aaaa} + 2 {ww oooooo bbbbb ssssss mm} {ffffff [yy] iiii rr s ccc [qqqqq]} + } } { do_execsql_test 6.1.$tn { SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']') diff --git a/manifest b/manifest index 30f63e1524..672b113552 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sfts5\ssynonyms\sand\sphrase\squeries.\sAlso\sfix\san\sOOM\shandling\sbug\sin\sfts5. -D 2015-09-02T17:34:22.663 +C Fix\san\sissue\swith\sfts5\ssynonyms\sand\sNEAR(...)\squeries. +D 2015-09-02T18:56:01.484 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 408384b2f540dcf72a363e1cde4c3e9896e8d41d +F ext/fts5/fts5_expr.c 1a6dbdab7a1035210b04f9ddccc019fccefe77d5 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test 88efc3de51a2c46aff984d74f06c833e4def0493 +F ext/fts5/test/fts5synonym.test 4bc555107abb16df3873d23d1f735eeab4d24010 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 11fa980897c6c7be218bbd9c4cd8253272d2c300 -R 6df2cc151a0891be96a1f7bf13aeaf04 +P a4c35fa2c94fe34b376670244fe72303c99868c1 +R 95a88bfdbba18ffe1c3b172dc2ad91be U dan -Z 72b6e5381768fc37d01c74eff21e6a40 +Z d85f21bea0d3a94f93f876bfbe641651 diff --git a/manifest.uuid b/manifest.uuid index 2675a32e17..14089c65e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4c35fa2c94fe34b376670244fe72303c99868c1 \ No newline at end of file +f2e590700d8eb30da8ac3a92f6ca92a2bbe9c1fe \ No newline at end of file From 9c671b741c220ebb7c4667c22eede207e801f100 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 2 Sep 2015 19:48:55 +0000 Subject: [PATCH 11/18] Further tests to raise coverage of fts5 synonym code to 100%. Fix a dropped error code in the same. FossilOrigin-Name: bdedd838bb3028c586bcc9f643852ce1364adb49 --- ext/fts5/fts5_expr.c | 2 +- ext/fts5/fts5_tokenize.c | 2 +- ext/fts5/test/fts5aa.test | 10 ++++++++++ ext/fts5/test/fts5fault6.test | 31 ++++++++++++++++++++++++++++--- ext/fts5/test/fts5synonym.test | 12 +++++++++++- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- 7 files changed, 62 insertions(+), 17 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 7cdb5e7632..91753652d1 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -662,7 +662,7 @@ static int fts5ExprNearAdvanceFirst( i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0); /* Advance each iterator that currently points to iRowid. Or, if iFrom - ** is valid - each iterator that points to a rowid before iFrom. */ + ** is valid - each iterator that points to a rowid before iFrom. */ for(p=pTerm; p; p=p->pSynonym){ if( sqlite3Fts5IterEof(p->pIter)==0 ){ i64 ii = sqlite3Fts5IterRowid(p->pIter); diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index 6b1129c24d..29a4579b44 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -1229,7 +1229,7 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){ ); } - return SQLITE_OK; + return rc; } diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index 22aa1b0f35..832a972cfc 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -506,6 +506,16 @@ do_execsql_test 18.3 { SELECT t1.rowid, t2.rowid FROM t2, t1 WHERE t2 MATCH t1.a AND t1.rowid = t2.c } {1 1} +#-------------------------------------------------------------------- +# fts5 table in the temp schema. +# +reset_db +do_execsql_test 19.0 { + CREATE VIRTUAL TABLE temp.t1 USING fts5(x); + INSERT INTO t1 VALUES('x y z'); + INSERT INTO t1 VALUES('w x 1'); + SELECT rowid FROM t1 WHERE t1 MATCH 'x'; +} {1 2} finish_test diff --git a/ext/fts5/test/fts5fault6.test b/ext/fts5/test/fts5fault6.test index b739e87418..c79cf7ab70 100644 --- a/ext/fts5/test/fts5fault6.test +++ b/ext/fts5/test/fts5fault6.test @@ -22,7 +22,6 @@ ifcapable !fts5 { return } -if 1 { #------------------------------------------------------------------------- # OOM while rebuilding an FTS5 table. @@ -150,8 +149,6 @@ do_faultsim_test 4.1 -faults oom-t* -prep { faultsim_test_result {0 {}} } -} - #------------------------------------------------------------------------- # # 5.2.* OOM while running a query that includes synonyms and matchinfo(). @@ -224,6 +221,8 @@ do_test 5.0 { 47 {dd dd dddddd ddddd fffff dddddd ddd ddddd eeeeee ffff eeee eee ee} 48 {ffff ffff e dddd ffffff dd dd dddd f fffff} 49 {ffffff d dddddd ffff eeeee f ffff ffff d dd fffff eeeee} + + 50 {x e} } { execsql { INSERT INTO t1(rowid, a) VALUES($rowid, $text) } } @@ -268,5 +267,31 @@ do_faultsim_test 5.3 -faults oom* -prep { faultsim_test_result {0 29} } +do_faultsim_test 5.4 -faults oom* -prep { + faultsim_restore_and_reopen + sqlite3_fts5_create_tokenizer db tcl tcl_create +} -body { + db eval { + SELECT count(*) FROM t1 WHERE t1 MATCH 'x + e' + } +} -test { + faultsim_test_result {0 1} +} + +#------------------------------------------------------------------------- +catch { db close } +breakpoint +do_faultsim_test 6 -faults oom* -prep { + sqlite_orig db test.db + sqlite3_db_config_lookaside db 0 0 0 +} -body { + load_static_extension db fts5 +} -test { + faultsim_test_result {0 {}} {1 {initialization of fts5 failed: }} + if {$testrc==0} { + db eval { CREATE VIRTUAL TABLE temp.t1 USING fts5(x) } + } + db close +} finish_test diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index a0fb43eca6..3c44bd7d17 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -387,10 +387,20 @@ foreach {tn q res} { 2 {ww oooooo bbbbb ssssss mm} {ffffff [yy] iiii rr s ccc [qqqqq]} } } { - do_execsql_test 6.1.$tn { + do_execsql_test 6.1.$tn.asc { SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']') FROM t2 WHERE t2 MATCH $q } $res + + set res2 [list] + foreach {rowid a b} $res { + set res2 [concat [list $rowid $a $b] $res2] + } + + do_execsql_test 6.1.$tn.desc { + SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']') + FROM t2 WHERE t2 MATCH $q ORDER BY rowid DESC + } $res2 } do_execsql_test 6.2.1 { diff --git a/manifest b/manifest index 672b113552..e7ac7255f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sissue\swith\sfts5\ssynonyms\sand\sNEAR(...)\squeries. -D 2015-09-02T18:56:01.484 +C Further\stests\sto\sraise\scoverage\sof\sfts5\ssynonym\scode\sto\s100%.\sFix\sa\sdropped\serror\scode\sin\sthe\ssame. +D 2015-09-02T19:48:55.424 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,21 +110,21 @@ F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 1a6dbdab7a1035210b04f9ddccc019fccefe77d5 +F ext/fts5/fts5_expr.c 45a1d1865b648ca0797f98a5f3a50a0f3a282fbb F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e F ext/fts5/fts5_storage.c c888defbb961d64c12299b3d1725a24a770b047e F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf -F ext/fts5/fts5_tokenize.c 710541513ecf3fe6d9365326fc85aee6efe97229 +F ext/fts5/fts5_tokenize.c f380f46f341af9c9a9908e1aade685ba1eaa157a F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl b6e6a40ef5d069c8e86ca4fbad491e1195485dbc -F ext/fts5/test/fts5aa.test caa44c528f7270aa4e325c4f2c28d355c1e8c307 +F ext/fts5/test/fts5aa.test 86def30ff5e76eb6db66f0833c4aefe0b765d590 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c F ext/fts5/test/fts5ad.test b2edee8b7de0c21d2c88f8a18c195034aad6952d @@ -156,7 +156,7 @@ F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7 F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875 -F ext/fts5/test/fts5fault6.test 93e9d43ab1e05ae5efa30cfde17649b3bb6a82c3 +F ext/fts5/test/fts5fault6.test 97bce1a36b7a64e3203fea504ae8e5cfd5ada423 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6 F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test 4bc555107abb16df3873d23d1f735eeab4d24010 +F ext/fts5/test/fts5synonym.test 4b6387dcc12fc4e7216db06402375326ae4a28aa F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a4c35fa2c94fe34b376670244fe72303c99868c1 -R 95a88bfdbba18ffe1c3b172dc2ad91be +P f2e590700d8eb30da8ac3a92f6ca92a2bbe9c1fe +R 5fe3995e4cfcebadaed779600452a235 U dan -Z d85f21bea0d3a94f93f876bfbe641651 +Z 535c1d64e133dff7ac9edd44481e77b4 diff --git a/manifest.uuid b/manifest.uuid index 14089c65e5..6008a07af2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f2e590700d8eb30da8ac3a92f6ca92a2bbe9c1fe \ No newline at end of file +bdedd838bb3028c586bcc9f643852ce1364adb49 \ No newline at end of file From 2a6ecb3141436fe7c0f2b11d80a0e1d1ae9de4c6 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Sep 2015 10:27:02 +0000 Subject: [PATCH 12/18] Fix a memory leak in fts5_expr.c. FossilOrigin-Name: 399932a181437d84cd57234e76c4b65da1e4ac5f --- ext/fts5/fts5_expr.c | 1 + ext/fts5/test/fts5synonym.test | 22 ++++++++++++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 91753652d1..ea5fd8a09e 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -344,6 +344,7 @@ static int fts5ExprSynonymPoslist( } memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter); nAlloc = nAlloc*2; + if( aIter!=aStatic ) sqlite3_free(aIter); aIter = aNew; } sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter]); diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 3c44bd7d17..81073dec8b 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -417,6 +417,28 @@ do_execsql_test 6.2.1 { {(y) (yy) (yyy) (yyyy) (yyyyy) (yyyyyy) (yyyyyyy)} } +#------------------------------------------------------------------------- +# Test that the xColumnSize() API is not confused by colocated tokens. +# +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +fts5_aux_test_functions db +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {[string length $w]==1} { + for {set i 2} {$i<=10} {incr i} { + sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd + } + } + } +} +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=0, tokenize=tcl); + INSERT INTO t1 VALUES('0 2 3', '4 5 6 7'); + INSERT INTO t1 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0'); + SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH '000 AND 00 AND 0'; +} {{3 4} {2 10}} finish_test diff --git a/manifest b/manifest index e7ac7255f4..efbe08818c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\stests\sto\sraise\scoverage\sof\sfts5\ssynonym\scode\sto\s100%.\sFix\sa\sdropped\serror\scode\sin\sthe\ssame. -D 2015-09-02T19:48:55.424 +C Fix\sa\smemory\sleak\sin\sfts5_expr.c. +D 2015-09-03T10:27:02.747 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 45a1d1865b648ca0797f98a5f3a50a0f3a282fbb +F ext/fts5/fts5_expr.c 0c36c1db8eccdeb006e3c8d1499d05015f6e11a6 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test 4b6387dcc12fc4e7216db06402375326ae4a28aa +F ext/fts5/test/fts5synonym.test e9ef0e6d59e6050020e0a805c37edde623bf289e F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f2e590700d8eb30da8ac3a92f6ca92a2bbe9c1fe -R 5fe3995e4cfcebadaed779600452a235 +P bdedd838bb3028c586bcc9f643852ce1364adb49 +R 780ceea4c7fd3021fab9ec34a27ddefc U dan -Z 535c1d64e133dff7ac9edd44481e77b4 +Z ba334b6166e1647cafa078f408fcb3d8 diff --git a/manifest.uuid b/manifest.uuid index 6008a07af2..f12f376ddc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bdedd838bb3028c586bcc9f643852ce1364adb49 \ No newline at end of file +399932a181437d84cd57234e76c4b65da1e4ac5f \ No newline at end of file From 9a9e3531f7ba6d6b1ccad3bdde65cd3483c0fe7b Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Sep 2015 11:00:50 +0000 Subject: [PATCH 13/18] Fix the fts5 integrity-check so that it works with columnsize=0 tables. FossilOrigin-Name: 11b887b15eaee57ea2577c763e70494f1e251275 --- ext/fts5/fts5_storage.c | 19 ++++++++----- ext/fts5/test/fts5columnsize.test | 13 +++++++++ ext/fts5/test/fts5fault7.test | 45 +++++++++++++++++++++++++++++++ ext/fts5/test/fts5synonym.test | 20 ++++++++++++-- manifest | 17 ++++++------ manifest.uuid | 2 +- 6 files changed, 99 insertions(+), 17 deletions(-) create mode 100644 ext/fts5/test/fts5fault7.test diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 6ff81d428d..c91e7eeba3 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -891,7 +891,9 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ int i; ctx.iRowid = sqlite3_column_int64(pScan, 0); ctx.szCol = 0; - rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); + if( pConfig->bColumnsize ){ + rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); + } for(i=0; rc==SQLITE_OK && inCol; i++){ if( pConfig->abUnindexed[i] ) continue; ctx.iCol = i; @@ -903,7 +905,9 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ (void*)&ctx, fts5StorageIntegrityCallback ); - if( ctx.szCol!=aColSize[i] ) rc = FTS5_CORRUPT; + if( pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ + rc = FTS5_CORRUPT; + } aTotalSize[i] += ctx.szCol; } if( rc!=SQLITE_OK ) break; @@ -928,7 +932,7 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ rc = fts5StorageCount(p, "content", &nRow); if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && pConfig->bColumnsize ){ i64 nRow; rc = fts5StorageCount(p, "docsize", &nRow); if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; @@ -1012,9 +1016,12 @@ static int fts5StorageDecodeSizeArray( ** otherwise. */ int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){ - int nCol = p->pConfig->nCol; - sqlite3_stmt *pLookup = 0; - int rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); + int nCol = p->pConfig->nCol; /* Number of user columns in table */ + sqlite3_stmt *pLookup = 0; /* Statement to query %_docsize */ + int rc; /* Return Code */ + + assert( p->pConfig->bColumnsize ); + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); if( rc==SQLITE_OK ){ int bCorrupt = 1; sqlite3_bind_int64(pLookup, 1, iRowid); diff --git a/ext/fts5/test/fts5columnsize.test b/ext/fts5/test/fts5columnsize.test index ed0edd677e..dec9b58d3d 100644 --- a/ext/fts5/test/fts5columnsize.test +++ b/ext/fts5/test/fts5columnsize.test @@ -134,5 +134,18 @@ do_execsql_test 3.2.1 { 1 {-1 0 -1} 2 {-1 0 -1} } +#------------------------------------------------------------------------- +# Test the integrity-check +# +do_execsql_test 4.1.1 { + CREATE VIRTUAL TABLE t5 USING fts5(x, columnsize=0); + INSERT INTO t5 VALUES('1 2 3 4'); + INSERT INTO t5 VALUES('2 4 6 8'); +} + +breakpoint +do_execsql_test 4.1.2 { + INSERT INTO t5(t5) VALUES('integrity-check'); +} finish_test diff --git a/ext/fts5/test/fts5fault7.test b/ext/fts5/test/fts5fault7.test new file mode 100644 index 0000000000..381f9ea6ad --- /dev/null +++ b/ext/fts5/test/fts5fault7.test @@ -0,0 +1,45 @@ +# 2015 September 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5fault2 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +#------------------------------------------------------------------------- +# Test fault-injection on a query that uses xColumnSize() on columnsize=0 +# table. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0); + INSERT INTO t1 VALUES('a b c d e f g'); + INSERT INTO t1 VALUES('a b c d'); + INSERT INTO t1 VALUES('a b c d e f g h i j'); +} + + +fts5_aux_test_functions db +do_faultsim_test 1 -faults oom* -body { + execsql { SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH 'b' } +} -test { + faultsim_test_result {0 {7 4 10}} {1 SQLITE_NOMEM} +} + +finish_test + diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 81073dec8b..008d2fc543 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -433,12 +433,28 @@ proc tcl_tokenize {tflags text} { } } } -do_execsql_test 7.0 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=0, tokenize=tcl); + +do_execsql_test 7.0.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl); INSERT INTO t1 VALUES('0 2 3', '4 5 6 7'); INSERT INTO t1 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0'); SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH '000 AND 00 AND 0'; } {{3 4} {2 10}} +do_execsql_test 7.0.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 7.1.1 { + CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl); + INSERT INTO t2 VALUES('0 2 3', '4 5 6 7'); + INSERT INTO t2 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0'); + SELECT fts5_test_columnsize(t2) FROM t2 WHERE t2 MATCH '000 AND 00 AND 0'; +} {{3 4} {2 10}} + +do_execsql_test 7.1.2 { + INSERT INTO t2(t2) VALUES('integrity-check'); +} + finish_test diff --git a/manifest b/manifest index efbe08818c..5ea1f7c040 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\smemory\sleak\sin\sfts5_expr.c. -D 2015-09-03T10:27:02.747 +C Fix\sthe\sfts5\sintegrity-check\sso\sthat\sit\sworks\swith\scolumnsize=0\stables. +D 2015-09-03T11:00:50.518 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F ext/fts5/fts5_expr.c 0c36c1db8eccdeb006e3c8d1499d05015f6e11a6 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e -F ext/fts5/fts5_storage.c c888defbb961d64c12299b3d1725a24a770b047e +F ext/fts5/fts5_storage.c 4b883f592ffdc6bcefba6fa03580228379a1333f F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf F ext/fts5/fts5_tokenize.c f380f46f341af9c9a9908e1aade685ba1eaa157a @@ -141,7 +141,7 @@ F ext/fts5/test/fts5auto.test caa5bcf917db11944655a2a9bd38c67c520376ca F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb -F ext/fts5/test/fts5columnsize.test 97dc6bd66c91009d00407aa078dd5e9e8eb22f99 +F ext/fts5/test/fts5columnsize.test a8cfef21ffa1c264b9f670a7d94eeaccb5341c07 F ext/fts5/test/fts5config.test ad2ff42ddc856aed2d05bf89dc1c578c8a39ea3b F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1 F ext/fts5/test/fts5corrupt.test 928c9c91d40690d301f943a7ed0ffc19e0d0e7b6 @@ -157,6 +157,7 @@ F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7 F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875 F ext/fts5/test/fts5fault6.test 97bce1a36b7a64e3203fea504ae8e5cfd5ada423 +F ext/fts5/test/fts5fault7.test f4a9b796f8b20c78ec7cf9f4e11144d15d7c3fd4 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6 F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145 @@ -172,7 +173,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test e9ef0e6d59e6050020e0a805c37edde623bf289e +F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bdedd838bb3028c586bcc9f643852ce1364adb49 -R 780ceea4c7fd3021fab9ec34a27ddefc +P 399932a181437d84cd57234e76c4b65da1e4ac5f +R 71ce956d6d1c9e126c469db6cf4c4e29 U dan -Z ba334b6166e1647cafa078f408fcb3d8 +Z 54ee3fa6162283338bf35bd792c52ef7 diff --git a/manifest.uuid b/manifest.uuid index f12f376ddc..5fc6e92798 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -399932a181437d84cd57234e76c4b65da1e4ac5f \ No newline at end of file +11b887b15eaee57ea2577c763e70494f1e251275 \ No newline at end of file From 5d144875bc3f6390f67575e0710a08cfe1124ae1 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Sep 2015 11:17:52 +0000 Subject: [PATCH 14/18] Remove dead code from fts5_index.c. FossilOrigin-Name: 8a0a9b01e74072ee52fe393311ad591208fbbf7c --- ext/fts5/fts5_index.c | 295 +++++++----------------------------------- manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 51 insertions(+), 258 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index a7ab7c4c05..06a5d56bbc 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -293,7 +293,6 @@ typedef struct Fts5Data Fts5Data; typedef struct Fts5DlidxIter Fts5DlidxIter; typedef struct Fts5DlidxLvl Fts5DlidxLvl; typedef struct Fts5DlidxWriter Fts5DlidxWriter; -typedef struct Fts5NodeIter Fts5NodeIter; typedef struct Fts5PageWriter Fts5PageWriter; typedef struct Fts5SegIter Fts5SegIter; typedef struct Fts5DoclistIter Fts5DoclistIter; @@ -526,24 +525,6 @@ struct Fts5IndexIter { }; -/* -** Object for iterating through the conents of a single internal node in -** memory. -*/ -struct Fts5NodeIter { - /* Internal. Set and managed by fts5NodeIterXXX() functions. Except, - ** the EOF test for the iterator is (Fts5NodeIter.aData==0). */ - const u8 *aData; - int nData; - int iOff; - - /* Output variables */ - Fts5Buffer term; - int nEmpty; - int iChild; - int bDlidx; -}; - /* ** An instance of the following type is used to iterate through the contents ** of a doclist-index record. @@ -617,6 +598,7 @@ static void *fts5IdxMalloc(Fts5Index *p, int nByte){ ** ** res = *pLeft - *pRight */ +#ifdef SQLITE_DEBUG static int fts5BufferCompareBlob( Fts5Buffer *pLeft, /* Left hand side of comparison */ const u8 *pRight, int nRight /* Right hand side of comparison */ @@ -625,7 +607,7 @@ static int fts5BufferCompareBlob( int res = memcmp(pLeft->p, pRight, nCmp); return (res==0 ? (pLeft->n - nRight) : res); } - +#endif /* ** Compare the contents of the two buffers using memcmp(). If one buffer @@ -1228,62 +1210,6 @@ static void fts5StructurePromote( } -/* -** If the pIter->iOff offset currently points to an entry indicating one -** or more term-less nodes, advance past it and set pIter->nEmpty to -** the number of empty child nodes. -*/ -static void fts5NodeIterGobbleNEmpty(Fts5NodeIter *pIter){ - if( pIter->iOffnData && 0==(pIter->aData[pIter->iOff] & 0xfe) ){ - pIter->bDlidx = pIter->aData[pIter->iOff] & 0x01; - pIter->iOff++; - pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], pIter->nEmpty); - }else{ - pIter->nEmpty = 0; - pIter->bDlidx = 0; - } -} - -/* -** Advance to the next entry within the node. -*/ -static void fts5NodeIterNext(int *pRc, Fts5NodeIter *pIter){ - if( pIter->iOff>=pIter->nData ){ - pIter->aData = 0; - pIter->iChild += pIter->nEmpty; - }else{ - int nPre, nNew; - pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], nPre); - pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], nNew); - pIter->term.n = nPre-2; - fts5BufferAppendBlob(pRc, &pIter->term, nNew, pIter->aData+pIter->iOff); - pIter->iOff += nNew; - pIter->iChild += (1 + pIter->nEmpty); - fts5NodeIterGobbleNEmpty(pIter); - if( *pRc ) pIter->aData = 0; - } -} - - -/* -** Initialize the iterator object pIter to iterate through the internal -** segment node in pData. -*/ -static void fts5NodeIterInit(const u8 *aData, int nData, Fts5NodeIter *pIter){ - memset(pIter, 0, sizeof(*pIter)); - pIter->aData = aData; - pIter->nData = nData; - pIter->iOff = fts5GetVarint32(aData, pIter->iChild); - fts5NodeIterGobbleNEmpty(pIter); -} - -/* -** Free any memory allocated by the iterator object. -*/ -static void fts5NodeIterFree(Fts5NodeIter *pIter){ - fts5BufferFree(&pIter->term); -} - /* ** Advance the iterator passed as the only argument. If the end of the ** doclist-index page is reached, return non-zero. @@ -2041,119 +1967,6 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); } -#ifdef SQLITE_DEBUG -static void fts5AssertNodeSeekOk( - Fts5Buffer *pNode, - const u8 *pTerm, int nTerm, /* Term to search for */ - int iExpectPg, - int bExpectDlidx -){ - int bDlidx; - int iPg; - int rc = SQLITE_OK; - Fts5NodeIter node; - - fts5NodeIterInit(pNode->p, pNode->n, &node); - assert( node.term.n==0 ); - iPg = node.iChild; - bDlidx = node.bDlidx; - for(fts5NodeIterNext(&rc, &node); - node.aData && fts5BufferCompareBlob(&node.term, pTerm, nTerm)<=0; - fts5NodeIterNext(&rc, &node) - ){ - iPg = node.iChild; - bDlidx = node.bDlidx; - } - fts5NodeIterFree(&node); - - assert( rc!=SQLITE_OK || iPg==iExpectPg ); - assert( rc!=SQLITE_OK || bDlidx==bExpectDlidx ); -} -#else -#define fts5AssertNodeSeekOk(v,w,x,y,z) -#endif - -/* -** Argument pNode is an internal b-tree node. This function searches -** within the node for the largest term that is smaller than or equal -** to (pTerm/nTerm). -** -** It returns the associated page number. Or, if (pTerm/nTerm) is smaller -** than all terms within the node, the leftmost child page number. -** -** Before returning, (*pbDlidx) is set to true if the last term on the -** returned child page number has a doclist-index. Or left as is otherwise. -*/ -static int fts5NodeSeek( - Fts5Buffer *pNode, /* Node to search */ - const u8 *pTerm, int nTerm, /* Term to search for */ - int *pbDlidx /* OUT: True if dlidx flag is set */ -){ - int iPg; - u8 *pPtr = pNode->p; - u8 *pEnd = &pPtr[pNode->n]; - int nMatch = 0; /* Number of bytes of pTerm already matched */ - - assert( *pbDlidx==0 ); - - pPtr += fts5GetVarint32(pPtr, iPg); - while( pPtr=pEnd ) break; - } - - /* Read the next "term" pointer. Set nKeep to the number of bytes to - ** keep from the previous term, and nNew to the number of bytes of - ** new data that will be appended to it. */ - nKeep = (int)*pPtr++; - nNew = (int)*pPtr++; - if( (nKeep | nNew) & 0x0080 ){ - pPtr -= 2; - pPtr += fts5GetVarint32(pPtr, nKeep); - pPtr += fts5GetVarint32(pPtr, nNew); - } - nKeep -= 2; - - /* Compare (pTerm/nTerm) to the current term on the node (the one described - ** by nKeep/nNew). If the node term is larger, break out of the while() - ** loop. - ** - ** Otherwise, if (pTerm/nTerm) is larger or the two terms are equal, - ** leave variable nMatch set to the size of the largest prefix common to - ** both terms in bytes. */ - if( nKeep==nMatch ){ - int nTst = MIN(nNew, nTerm-nMatch); - int i; - for(i=0; i pTerm[nMatch]) ) break; - }else if( nKeep=4 ){ - iRowidOff = fts5GetU16(&a[0]); - iTermOff = fts5GetU16(&a[2]); - }else{ - sqlite3Fts5BufferSet(&rc, &s, 8, (const u8*)"corrupt"); - goto decode_out; - } - - if( iRowidOff ){ - iOff = iRowidOff; - }else if( iTermOff ){ - iOff = iTermOff; - }else{ - iOff = n; - } - fts5DecodePoslist(&rc, &s, &a[4], iOff-4); - - assert( iRowidOff==0 || iOff==iRowidOff ); - if( iRowidOff ){ - iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff); - } - - assert( iTermOff==0 || iOff==iTermOff ); - while( iOff=4 ){ + iRowidOff = fts5GetU16(&a[0]); + iTermOff = fts5GetU16(&a[2]); }else{ - Fts5NodeIter ss; - for(fts5NodeIterInit(a, n, &ss); ss.aData; fts5NodeIterNext(&rc, &ss)){ - if( ss.term.n==0 ){ - sqlite3Fts5BufferAppendPrintf(&rc, &s, " left=%d", ss.iChild); - }else{ - sqlite3Fts5BufferAppendPrintf(&rc,&s, " \"%.*s\"", - ss.term.n, ss.term.p - ); - } - if( ss.nEmpty ){ - sqlite3Fts5BufferAppendPrintf(&rc, &s, " empty=%d%s", ss.nEmpty, - ss.bDlidx ? "*" : "" - ); - } - } - fts5NodeIterFree(&ss); + sqlite3Fts5BufferSet(&rc, &s, 8, (const u8*)"corrupt"); + goto decode_out; } + + if( iRowidOff ){ + iOff = iRowidOff; + }else if( iTermOff ){ + iOff = iTermOff; + }else{ + iOff = n; + } + fts5DecodePoslist(&rc, &s, &a[4], iOff-4); + + assert( iRowidOff==0 || iOff==iRowidOff ); + if( iRowidOff ){ + iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff); + } + + assert( iTermOff==0 || iOff==iTermOff ); + while( iOff Date: Thu, 3 Sep 2015 14:22:27 +0000 Subject: [PATCH 15/18] Remove some more code from fts5_index.c by consolidating similar functions. FossilOrigin-Name: 59ae30b97b40faa363c55aa2664dead9eaeeddc0 --- ext/fts5/fts5Int.h | 4 +- ext/fts5/fts5_index.c | 128 +++++++++++++++------------------------- ext/fts5/fts5_storage.c | 17 +----- manifest | 16 ++--- manifest.uuid | 2 +- 5 files changed, 58 insertions(+), 109 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 1da017fb01..837ecb1ccd 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -384,9 +384,9 @@ int sqlite3Fts5IndexErrcode(Fts5Index*); void sqlite3Fts5IndexReset(Fts5Index*); /* -** Get or set the "averages" record. +** Get or set the "averages" values. */ -int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf); +int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize); int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); /* diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 06a5d56bbc..7bb744773c 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -554,23 +554,6 @@ struct Fts5DlidxIter { Fts5DlidxLvl aLvl[1]; }; - - -/* -** The first argument passed to this macro is a pointer to an Fts5Buffer -** object. -*/ -#define fts5BufferSize(pBuf,n) { \ - if( pBuf->nSpacep, n); \ - if( pNew==0 ){ \ - sqlite3_free(pBuf->p); \ - } \ - pBuf->nSpace = n; \ - pBuf->p = pNew; \ - } \ -} - static void fts5PutU16(u8 *aOut, u16 iVal){ aOut[0] = (iVal>>8); aOut[1] = (iVal&0xFF); @@ -647,11 +630,14 @@ static void fts5CloseReader(Fts5Index *p){ } } -static Fts5Data *fts5DataReadOrBuffer( - Fts5Index *p, - Fts5Buffer *pBuf, - i64 iRowid -){ + +/* +** Retrieve a record from the %_data table. +** +** If an error occurs, NULL is returned and an error left in the +** Fts5Index object. +*/ +static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ Fts5Data *pRet = 0; if( p->rc==SQLITE_OK ){ int rc = SQLITE_OK; @@ -671,8 +657,8 @@ static Fts5Data *fts5DataReadOrBuffer( if( rc==SQLITE_ABORT ) rc = SQLITE_OK; } - /* If the blob handle is not yet open, open and seek it. Otherwise, use - ** the blob_reopen() API to reseek the existing blob handle. */ + /* If the blob handle is not open at this point, open it and seek + ** to the requested entry. */ if( p->pReader==0 && rc==SQLITE_OK ){ Fts5Config *pConfig = p->pConfig; rc = sqlite3_blob_open(pConfig->db, @@ -690,22 +676,13 @@ static Fts5Data *fts5DataReadOrBuffer( if( rc==SQLITE_OK ){ u8 *aOut = 0; /* Read blob data into this buffer */ int nByte = sqlite3_blob_bytes(p->pReader); - if( pBuf ){ - fts5BufferSize(pBuf, MAX(nByte, p->pConfig->pgsz) + 20); - pBuf->n = nByte; - aOut = pBuf->p; - if( aOut==0 ){ - rc = SQLITE_NOMEM; - } + int nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING; + pRet = (Fts5Data*)sqlite3_malloc(nAlloc); + if( pRet ){ + pRet->n = nByte; + aOut = pRet->p = (u8*)&pRet[1]; }else{ - int nSpace = nByte + FTS5_DATA_PADDING; - pRet = (Fts5Data*)sqlite3_malloc(nSpace+sizeof(Fts5Data)); - if( pRet ){ - pRet->n = nByte; - aOut = pRet->p = (u8*)&pRet[1]; - }else{ - rc = SQLITE_NOMEM; - } + rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ @@ -720,33 +697,10 @@ static Fts5Data *fts5DataReadOrBuffer( p->nRead++; } - return pRet; -} - -/* -** Retrieve a record from the %_data table. -** -** If an error occurs, NULL is returned and an error left in the -** Fts5Index object. -*/ -static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ - Fts5Data *pRet = fts5DataReadOrBuffer(p, 0, iRowid); assert( (pRet==0)==(p->rc!=SQLITE_OK) ); return pRet; } -/* -** Read a record from the %_data table into the buffer supplied as the -** second argument. -** -** If an error occurs, an error is left in the Fts5Index object. If an -** error has already occurred when this function is called, it is a -** no-op. -*/ -static void fts5DataBuffer(Fts5Index *p, Fts5Buffer *pBuf, i64 iRowid){ - (void)fts5DataReadOrBuffer(p, pBuf, iRowid); -} - /* ** Release a reference to data record returned by an earlier call to ** fts5DataRead(). @@ -1015,19 +969,18 @@ static Fts5Structure *fts5StructureRead(Fts5Index *p){ Fts5Config *pConfig = p->pConfig; Fts5Structure *pRet = 0; /* Object to return */ int iCookie; /* Configuration cookie */ + Fts5Data *pData; Fts5Buffer buf = {0, 0, 0}; - fts5DataBuffer(p, &buf, FTS5_STRUCTURE_ROWID); - if( buf.p==0 ) return 0; - assert( buf.nSpace>=(buf.n + FTS5_DATA_ZERO_PADDING) ); - memset(&buf.p[buf.n], 0, FTS5_DATA_ZERO_PADDING); - p->rc = fts5StructureDecode(buf.p, buf.n, &iCookie, &pRet); - + pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); + if( p->rc ) return 0; + memset(&pData->p[pData->n], 0, FTS5_DATA_PADDING); + p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet); if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); } - fts5BufferFree(&buf); + fts5DataRelease(pData); if( p->rc!=SQLITE_OK ){ fts5StructureRelease(pRet); pRet = 0; @@ -2490,13 +2443,13 @@ static void fts5SegIterNextFrom( } } - while( p->rc==SQLITE_OK ){ + do{ if( bMove ) fts5SegIterNext(p, pIter, 0); if( pIter->pLeaf==0 ) break; if( bRev==0 && pIter->iRowid>=iMatch ) break; if( bRev!=0 && pIter->iRowid<=iMatch ) break; bMove = 1; - } + }while( p->rc==SQLITE_OK ); } @@ -4272,13 +4225,9 @@ int sqlite3Fts5IndexRollback(Fts5Index *p){ */ int sqlite3Fts5IndexReinit(Fts5Index *p){ Fts5Structure s; - - assert( p->rc==SQLITE_OK ); - p->rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0); - memset(&s, 0, sizeof(Fts5Structure)); + fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); fts5StructureWrite(p, &s); - return fts5IndexReturn(p); } @@ -4600,13 +4549,28 @@ void sqlite3Fts5IterClose(Fts5IndexIter *pIter){ } /* -** Read the "averages" record into the buffer supplied as the second -** argument. Return SQLITE_OK if successful, or an SQLite error code -** if an error occurs. +** Read and decode the "averages" record from the database. +** +** Parameter anSize must point to an array of size nCol, where nCol is +** the number of user defined columns in the FTS table. */ -int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf){ - assert( p->rc==SQLITE_OK ); - fts5DataReadOrBuffer(p, pBuf, FTS5_AVERAGES_ROWID); +int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize){ + int nCol = p->pConfig->nCol; + Fts5Data *pData; + + *pnRow = 0; + memset(anSize, 0, sizeof(i64) * nCol); + pData = fts5DataRead(p, FTS5_AVERAGES_ROWID); + if( p->rc==SQLITE_OK && pData->n ){ + int i = 0; + int iCol; + i += fts5GetVarint(&pData->p[i], (u64*)pnRow); + for(iCol=0; in && iColp[i], (u64*)&anSize[iCol]); + } + } + + fts5DataRelease(pData); return fts5IndexReturn(p); } diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index c91e7eeba3..65c6d48a36 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -455,22 +455,7 @@ static int fts5StorageInsertDocsize( static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){ int rc = SQLITE_OK; if( p->bTotalsValid==0 ){ - int nCol = p->pConfig->nCol; - Fts5Buffer buf; - memset(&buf, 0, sizeof(buf)); - - memset(p->aTotalSize, 0, sizeof(i64) * nCol); - p->nTotalRow = 0; - rc = sqlite3Fts5IndexGetAverages(p->pIndex, &buf); - if( rc==SQLITE_OK && buf.n ){ - int i = 0; - int iCol; - i += fts5GetVarint(&buf.p[i], (u64*)&p->nTotalRow); - for(iCol=0; iaTotalSize[iCol]); - } - } - sqlite3_free(buf.p); + rc = sqlite3Fts5IndexGetAverages(p->pIndex, &p->nTotalRow, p->aTotalSize); p->bTotalsValid = bCache; } return rc; diff --git a/manifest b/manifest index 73cbe4ee86..0e7dfca7b2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sdead\scode\sfrom\sfts5_index.c. -D 2015-09-03T11:17:52.904 +C Remove\ssome\smore\scode\sfrom\sfts5_index.c\sby\sconsolidating\ssimilar\sfunctions. +D 2015-09-03T14:22:27.810 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -106,15 +106,15 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42 -F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b +F ext/fts5/fts5Int.h f65d41f66accad0a289d6bd66b13c07d2932f9be F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 F ext/fts5/fts5_expr.c 0c36c1db8eccdeb006e3c8d1499d05015f6e11a6 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 -F ext/fts5/fts5_index.c df98f39c0f6e9d06e144dde5a0751ab14ee9d0fd +F ext/fts5/fts5_index.c febb68173333ae3248eb15928a18b21112d45135 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e -F ext/fts5/fts5_storage.c 4b883f592ffdc6bcefba6fa03580228379a1333f +F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf F ext/fts5/fts5_tokenize.c f380f46f341af9c9a9908e1aade685ba1eaa157a @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 11b887b15eaee57ea2577c763e70494f1e251275 -R 97c9a99cb5cabc51c42bc2a29210372f +P 8a0a9b01e74072ee52fe393311ad591208fbbf7c +R e26c002d3fbf39ced0ec480a25b8c1b8 U dan -Z 9499a3a8b1a7472d387dbda943ee05af +Z cd2601a80f5a8baa4e1506c2cddafc82 diff --git a/manifest.uuid b/manifest.uuid index f57a7a34cb..03cc15587f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8a0a9b01e74072ee52fe393311ad591208fbbf7c \ No newline at end of file +59ae30b97b40faa363c55aa2664dead9eaeeddc0 \ No newline at end of file From ea623630e2f057f531f895c4501264a93b02f472 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Sep 2015 15:37:26 +0000 Subject: [PATCH 16/18] Add tests to improve coverage of fts5_varint.c. FossilOrigin-Name: 89f24f31a8f7d7cb0a66ee53523881f566dcb035 --- ext/fts5/test/fts5aa.test | 20 ++++++++++++++++++++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index 832a972cfc..5f3a9d9e1d 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -516,6 +516,26 @@ do_execsql_test 19.0 { INSERT INTO t1 VALUES('w x 1'); SELECT rowid FROM t1 WHERE t1 MATCH 'x'; } {1 2} + +#-------------------------------------------------------------------- +# Test that 6 and 7 byte varints can be read. +# +reset_db +do_execsql_test 20.0 { + CREATE VIRTUAL TABLE temp.tmp USING fts5(x); +} +set ::ids [list \ + 0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43] +] +do_test 20.1 { + foreach id $::ids { + execsql { INSERT INTO tmp(rowid, x) VALUES($id, 'x y z') } + } + execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' } +} $::ids + + + finish_test diff --git a/manifest b/manifest index 0e7dfca7b2..c46ab77428 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\smore\scode\sfrom\sfts5_index.c\sby\sconsolidating\ssimilar\sfunctions. -D 2015-09-03T14:22:27.810 +C Add\stests\sto\simprove\scoverage\sof\sfts5_varint.c. +D 2015-09-03T15:37:26.095 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,7 +124,7 @@ F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl b6e6a40ef5d069c8e86ca4fbad491e1195485dbc -F ext/fts5/test/fts5aa.test 86def30ff5e76eb6db66f0833c4aefe0b765d590 +F ext/fts5/test/fts5aa.test be961d10b0eff5e973c37b3b7a32a18cda87a77c F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c F ext/fts5/test/fts5ad.test b2edee8b7de0c21d2c88f8a18c195034aad6952d @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8a0a9b01e74072ee52fe393311ad591208fbbf7c -R e26c002d3fbf39ced0ec480a25b8c1b8 +P 59ae30b97b40faa363c55aa2664dead9eaeeddc0 +R e54371f57da08ab0c80326ea4baedd7e U dan -Z cd2601a80f5a8baa4e1506c2cddafc82 +Z 416751bdad2bf139cde3272ad789aa1a diff --git a/manifest.uuid b/manifest.uuid index 03cc15587f..3c272efde7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -59ae30b97b40faa363c55aa2664dead9eaeeddc0 \ No newline at end of file +89f24f31a8f7d7cb0a66ee53523881f566dcb035 \ No newline at end of file From ea8b93ee2041a1f4e22132cb937eff137127f4b5 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Sep 2015 18:05:09 +0000 Subject: [PATCH 17/18] Add documentation for fts5 synonym support. FossilOrigin-Name: 58aa1f435959852df74f1bca8e0bdbc4f47c256a --- ext/fts5/extract_api_docs.tcl | 8 +- ext/fts5/fts5.h | 150 ++++++++++++++++++++++++++++++++-- manifest | 14 ++-- manifest.uuid | 2 +- 4 files changed, 157 insertions(+), 17 deletions(-) diff --git a/ext/fts5/extract_api_docs.tcl b/ext/fts5/extract_api_docs.tcl index afb2699be5..2320d70b7d 100644 --- a/ext/fts5/extract_api_docs.tcl +++ b/ext/fts5/extract_api_docs.tcl @@ -108,13 +108,15 @@ proc get_tokenizer_docs {data} { append res "
$line

\n" continue } + if {[regexp {SYNONYM SUPPORT} $line]} { + set line "

Synonym Support

" + } if {[string trim $line] == ""} { append res "

\n" } else { append res "$line\n" } } - append res "\n" set res } @@ -208,6 +210,10 @@ proc main {data} { fts5_tokenizer { output [get_fts5_struct $data "typedef struct Fts5Tokenizer" "^\};"] + output [get_fts5_struct $data \ + "Flags that may be passed as the third argument to xTokenize()" \ + "#define FTS5_TOKEN_COLOCATED" + ] } fts5_extension { diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index c9eb91d4cc..6872918e94 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -278,18 +278,46 @@ struct Fts5ExtensionApi { ** ** xTokenize: ** This function is expected to tokenize the nText byte string indicated -** by argument pText. pText may not be nul-terminated. The first argument -** passed to this function is a pointer to an Fts5Tokenizer object returned -** by an earlier call to xCreate(). +** by argument pText. pText may or may not be nul-terminated. The first +** argument passed to this function is a pointer to an Fts5Tokenizer object +** returned by an earlier call to xCreate(). +** +** The second argument indicates the reason that FTS5 is requesting +** tokenization of the supplied text. This is always one of the following +** four values: +** +**

  • FTS5_TOKENIZE_DOCUMENT - A document is being inserted into +** or removed from the FTS table. The tokenizer is being invoked to +** determine the set of tokens to add to (or delete from) the +** FTS index. +** +**
  • FTS5_TOKENIZE_QUERY - A MATCH query is being executed +** against the FTS index. The tokenizer is being called to tokenize +** a bareword or quoted string specified as part of the query. +** +**
  • (FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) - Same as +** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is +** followed by a "*" character, indicating that the last token +** returned by the tokenizer will be treated as a token prefix. +** +**
  • FTS5_TOKENIZE_AUX - The tokenizer is being invoked to +** satisfy an fts5_api.xTokenize() request made by an auxiliary +** function. Or an fts5_api.xColumnSize() request made by the same +** on a columnsize=0 database. +**
** ** For each token in the input string, the supplied callback xToken() must ** be invoked. The first argument to it should be a copy of the pointer -** passed as the second argument to xTokenize(). The next two arguments -** are a pointer to a buffer containing the token text, and the size of -** the token in bytes. The 4th and 5th arguments are the byte offsets of -** the first byte of and first byte immediately following the text from +** passed as the second argument to xTokenize(). The third and fourth +** arguments are a pointer to a buffer containing the token text, and the +** size of the token in bytes. The 4th and 5th arguments are the byte offsets +** of the first byte of and first byte immediately following the text from ** which the token is derived within the input. ** +** The second argument passed to the xToken() callback ("tflags") should +** normally be set to 0. The exception is if the tokenizer supports +** synonyms. In this case see the discussion below for details. +** ** FTS5 assumes the xToken() callback is invoked for each token in the ** order that they occur within the input text. ** @@ -301,6 +329,112 @@ struct Fts5ExtensionApi { ** may abandon the tokenization and return any error code other than ** SQLITE_OK or SQLITE_DONE. ** +** SYNONYM SUPPORT +** +** Custom tokenizers may also support synonyms. Consider a case in which a +** user wishes to query for a phrase such as "first place". Using the +** built-in tokenizers, the FTS5 query 'first + place' will match instances +** of "first place" within the document set, but not alternative forms +** such as "1st place". In some applications, it would be better to match +** all instances of "first place" or "1st place" regardless of which form +** the user specified in the MATCH query text. +** +** There are several ways to approach this in FTS5: +** +**
  1. By mapping all synonyms to a single token. In this case, the +** In the above example, this means that the tokenizer returns the +** same token for inputs "first" and "1st". Say that token is in +** fact "first", so that when the user inserts the document "I won +** 1st place" entries are added to the index for tokens "i", "won", +** "first" and "place". If the user then queries for '1st + place', +** the tokenizer substitutes "first" for "1st" and the query works +** as expected. +** +**
  2. By adding multiple synonyms for a single term to the FTS index. +** In this case, when tokenizing query text, the tokenizer may +** provide multiple synonyms for a single term within the document. +** FTS5 then queries the index for each synonym individually. For +** example, faced with the query: +** +** +** ... MATCH 'first place' +** +** the tokenizer offers both "1st" and "first" as synonyms for the +** first token in the MATCH query and FTS5 effectively runs a query +** similar to: +** +** +** ... MATCH '(first OR 1st) place' +** +** except that, for the purposes of auxiliary functions, the query +** still appears to contain just two phrases - "(first OR 1st)" +** being treated as a single phrase. +** +**
  3. By adding multiple synonyms for a single term to the FTS index. +** Using this method, when tokenizing document text, the tokenizer +** provides multiple synonyms for each token. So that when a +** document such as "I won first place" is tokenized, entries are +** added to the FTS index for "i", "won", "first", "1st" and +** "place". +** +** This way, even if the tokenizer does not provide synonyms +** when tokenizing query text (it should not - to do would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entires in the +** FTS index corresponding to both forms of the first token. +**
+** +** Whether is is parsing document or query text, any call to xToken that +** specifies a tflags argument with the FTS5_TOKEN_COLOCATED bit +** is considered to supply a synonym for the previous token. For example, +** when parsing the document "I won first place", a tokenizer that supports +** synonyms would call xToken() 5 times, as follows: +** +** +** xToken(pCtx, 0, "i", 1, 0, 1); +** xToken(pCtx, 0, "won", 3, 2, 5); +** xToken(pCtx, 0, "first", 5, 6, 11); +** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); +** xToken(pCtx, 0, "place", 5, 12, 17); +** +** +** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time +** xToken() is called. Multiple synonyms may be specified for a single token +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** There is no limit to the number of synonyms that may be provided for a +** single token. +** +** In many cases, method (1) above is the best approach. It does not add +** extra data to the FTS index or require FTS5 to query for multiple terms, +** so it is efficient in terms of disk space and query speed. However, it +** does not support prefix queries very well. If, as suggested above, the +** token "first" is subsituted for "1st" by the tokenizer, then the query: +** +** +** ... MATCH '1s*' +** +** will not match documents that contain the token "1st" (as the tokenizer +** will probably not map "1s" to any prefix of "first"). +** +** For full prefix support, method (3) may be preferred. In this case, +** because the index contains entries for both "first" and "1st", prefix +** queries such as 'fi*' or '1s*' will match correctly. However, because +** extra entries are added to the FTS index, this method uses more space +** within the database. +** +** Method (2) offers a midpoint between (1) and (3). Using this method, +** a query such as '1s*' will match documents that contain the literal +** token "1st", but not "first" (assuming the tokenizer is not able to +** provide synonyms for prefixes). However, a non-prefix query like '1st' +** will match against "1st" and "first". This method does not require +** extra disk space, as no extra entries are added to the FTS index. +** On the other hand, it may require more CPU cycles to run MATCH queries, +** as separate queries of the FTS index are required for each synonym. +** +** When using methods (2) or (3), it is important that the tokenizer only +** provide synonyms when tokenizing document text (method (2)) or query +** text (method (3)), not both. Doing so will not cause any errors, but is +** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; typedef struct fts5_tokenizer fts5_tokenizer; @@ -309,7 +443,7 @@ struct fts5_tokenizer { void (*xDelete)(Fts5Tokenizer*); int (*xTokenize)(Fts5Tokenizer*, void *pCtx, - int flags, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ const char *pText, int nText, int (*xToken)( void *pCtx, /* Copy of 2nd argument to xTokenize() */ diff --git a/manifest b/manifest index c46ab77428..2777d36021 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\simprove\scoverage\sof\sfts5_varint.c. -D 2015-09-03T15:37:26.095 +C Add\sdocumentation\sfor\sfts5\ssynonym\ssupport. +D 2015-09-03T18:05:09.505 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -104,8 +104,8 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 -F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 -F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42 +F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 +F ext/fts5/fts5.h f04659e0df5af83731b102189a32280f74f4a6bc F ext/fts5/fts5Int.h f65d41f66accad0a289d6bd66b13c07d2932f9be F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 59ae30b97b40faa363c55aa2664dead9eaeeddc0 -R e54371f57da08ab0c80326ea4baedd7e +P 89f24f31a8f7d7cb0a66ee53523881f566dcb035 +R 763079caf8ff69ca16b541c2d3f4f0ad U dan -Z 416751bdad2bf139cde3272ad789aa1a +Z 726cf8268b191707cc8a3f300ac29036 diff --git a/manifest.uuid b/manifest.uuid index 3c272efde7..fc803bdc2c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -89f24f31a8f7d7cb0a66ee53523881f566dcb035 \ No newline at end of file +58aa1f435959852df74f1bca8e0bdbc4f47c256a \ No newline at end of file From f8efe64649d4d4a646007bf8a76897a0b20ce95e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Sep 2015 19:48:02 +0000 Subject: [PATCH 18/18] Rearrange code in fts5_expr.c so that synonym support does not slow down the common case. FossilOrigin-Name: 801882817f1d895aef1426f9a7196bd140b807c3 --- ext/fts5/fts5_expr.c | 11 +++++++++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index ea5fd8a09e..b68634694f 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1181,10 +1181,17 @@ static int fts5ExprNodeNext( }; case FTS5_TERM: { - rc = fts5ExprNearAdvanceFirst(pExpr, pNode, bFromValid, iFrom); - if( pNode->bEof==0 ){ + Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter; + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(pIter); + } + if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){ assert( rc==SQLITE_OK ); rc = fts5ExprTokenTest(pExpr, pNode); + }else{ + pNode->bEof = 1; } return rc; }; diff --git a/manifest b/manifest index b7ec71865d..20d7c879fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\senhancements\sfrom\strunk. -D 2015-09-03T18:57:52.309 +C Rearrange\scode\sin\sfts5_expr.c\sso\sthat\ssynonym\ssupport\sdoes\snot\sslow\sdown\sthe\scommon\scase. +D 2015-09-03T19:48:02.456 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h f65d41f66accad0a289d6bd66b13c07d2932f9be F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 0c36c1db8eccdeb006e3c8d1499d05015f6e11a6 +F ext/fts5/fts5_expr.c 1c24e1a2ffb286bfe37e537a43b7fadabfe993d4 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c febb68173333ae3248eb15928a18b21112d45135 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 58aa1f435959852df74f1bca8e0bdbc4f47c256a 618d8dd4ff44cce10cc4688a2134715ff66cc562 -R 38e68c75949f5efa3d52c01a80d7f0aa +P 4b49fe996989fe42d2bb1c24e7193fef09c5fc50 +R 0fe79065145fa2e8621202f9c4e41627 U dan -Z 9f85937304cd0ef1147cd11d7113913e +Z fe72ec30c1ce282b0a02c9b8970f9360 diff --git a/manifest.uuid b/manifest.uuid index d9cd6ed680..005faca754 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4b49fe996989fe42d2bb1c24e7193fef09c5fc50 \ No newline at end of file +801882817f1d895aef1426f9a7196bd140b807c3 \ No newline at end of file