paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix bugs in fts5 synonym processing for detail=col and other modes.

Browse Source 
FossilOrigin-Name: 0e3c545423246231ddac8fed2f103d71d556e17f
This commit is contained in:
dan
2016-01-11 17:30:28 +00:00
parent 4f40cc6f3d
commit aaf17c41ec
4 changed files with 106 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ext/fts5/fts5_expr.c
+78 -28
View File
@@ -40,6 +40,7 @@ void sqlite3Fts5ParserTrace(FILE*, char*);
struct Fts5Expr {
Fts5Index *pIndex;
Fts5Config *pConfig;
Fts5ExprNode *pRoot;
int bDesc; /* Iterate in descending rowid order */
int nPhrase; /* Number of phrases in expression */
@@ -235,6 +236,7 @@ int sqlite3Fts5ExprNew(
}else{
pNew->pRoot = sParse.pExpr;
pNew->pIndex = 0;
pNew->pConfig = pConfig;
pNew->apExprPhrase = sParse.apPhrase;
pNew->nPhrase = sParse.nPhrase;
sParse.apPhrase = 0;
@@ -299,8 +301,9 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){
/*
** Argument pTerm must be a synonym iterator.
*/
static int fts5ExprSynonymPoslist(
static int fts5ExprSynonymList(
Fts5ExprTerm *pTerm,
int bCollist,
Fts5Colset *pColset,
i64 iRowid,
int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */
@@ -319,9 +322,16 @@ static int fts5ExprSynonymPoslist(
if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){
const u8 *a;
int n;
i64 dummy;
rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy);
if( bCollist ){
rc = sqlite3Fts5IterCollist(pIter, &a, &n);
}else{
i64 dummy;
rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy);
}
if( rc!=SQLITE_OK ) goto synonym_poslist_out;
if( n==0 ) continue;
if( nIter==nAlloc ){
int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2;
Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte);
@@ -422,8 +432,8 @@ static int fts5ExprPhraseIsMatch(
int bFlag = 0;
const u8 *a = 0;
if( pTerm->pSynonym ){
rc = fts5ExprSynonymPoslist(
pTerm, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n
rc = fts5ExprSynonymList(
pTerm, 0, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n
);
}else{
rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy);
@@ -757,30 +767,51 @@ static int fts5ExprNearTest(
){
Fts5ExprNearset *pNear = pNode->pNear;
int rc = *pRc;
int i;
/* Check that each phrase in the nearset matches the current row.
** Populate the pPhrase->poslist buffers at the same time. If any
** phrase is not a match, break out of the loop early. */
for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
int bMatch = 0;
rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
if( bMatch==0 ) break;
}else{
rc = sqlite3Fts5IterPoslistBuffer(
pPhrase->aTerm[0].pIter, &pPhrase->poslist
);
if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){
Fts5ExprTerm *pTerm;
Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
pPhrase->poslist.n = 0;
for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
Fts5IndexIter *pIter = pTerm->pIter;
if( sqlite3Fts5IterEof(pIter)==0 ){
int n;
i64 iRowid;
rc = sqlite3Fts5IterPoslist(pIter, pNear->pColset, 0, &n, &iRowid);
if( rc!=SQLITE_OK ){
*pRc = rc;
return 0;
}else if( iRowid==pNode->iRowid && n>0 ){
pPhrase->poslist.n = 1;
}
}
}
}
return pPhrase->poslist.n;
}else{
int i;
*pRc = rc;
if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
return 1;
}
/* Check that each phrase in the nearset matches the current row.
** Populate the pPhrase->poslist buffers at the same time. If any
** phrase is not a match, break out of the loop early. */
for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
int bMatch = 0;
rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
if( bMatch==0 ) break;
}else{
rc = sqlite3Fts5IterPoslistBuffer(
pPhrase->aTerm[0].pIter, &pPhrase->poslist
);
}
}
return 0;
*pRc = rc;
if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
return 1;
}
return 0;
}
}
static int fts5ExprTokenTest(
@@ -1606,6 +1637,7 @@ int sqlite3Fts5ExprClonePhrase(
if( rc==SQLITE_OK ){
/* All the allocations succeeded. Put the expression object together. */
pNew->pIndex = pExpr->pIndex;
pNew->pConfig = pExpr->pConfig;
pNew->nPhrase = 1;
pNew->apExprPhrase[0] = sCtx.pPhrase;
pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase;
@@ -2453,17 +2485,35 @@ int sqlite3Fts5ExprPhraseCollist(
){
Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase];
Fts5ExprNode *pNode = pPhrase->pNode;
assert( iPhrase>=0 && iPhrase<pExpr->nPhrase );
int rc = SQLITE_OK;
assert( iPhrase>=0 && iPhrase<pExpr->nPhrase );
if( pNode->bEof==0
&& pNode->iRowid==pExpr->pRoot->iRowid
&& pPhrase->poslist.n>0
){
sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist);
Fts5ExprTerm *pTerm = &pPhrase->aTerm[0];
if( pTerm->pSynonym ){
int bDel = 0;
u8 *a;
rc = fts5ExprSynonymList(
pTerm, 1, 0, pNode->iRowid, &bDel, &a, pnCollist
);
if( bDel ){
sqlite3Fts5BufferSet(&rc, &pPhrase->poslist, *pnCollist, a);
*ppCollist = pPhrase->poslist.p;
sqlite3_free(a);
}else{
*ppCollist = a;
}
}else{
sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist);
}
}else{
*ppCollist = 0;
*pnCollist = 0;
}
return SQLITE_OK;
return rc;
}
ext/fts5/test/fts5synonym2.test
+20 -5
View File
@@ -13,7 +13,7 @@
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5synonym
set testprefix fts5synonym2
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
@@ -47,7 +47,7 @@ proc tcl_tokenize {tflags text} {
foreach {w iStart iEnd} [fts5_tokenize_split $text] {
sqlite3_fts5_token $w $iStart $iEnd
if {$tflags == "query"} {
foreach s $::SYNDICT($w) { sqlite3_fts5_token -colo $s $iStart $iEnd }
foreach s $::syn($w) { sqlite3_fts5_token -colo $s $iStart $iEnd }
}
}
}
@@ -67,7 +67,6 @@ fts5_aux_test_functions db
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE ss USING fts5(a, b, tokenize=tcl, detail=%DETAIL%);
INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i');
INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi');
INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3');
@@ -120,14 +119,30 @@ do_execsql_test 1.0 {
}
foreach {tn expr} {
1 "eight"
1.1 "one" 1.2 "two" 1.3 "three" 1.4 "four"
1.5 "v" 1.6 "vi" 1.7 "vii" 1.8 "viii"
1.9 "9" 1.10 "0" 1.11 "1" 1.12 "2"
2.1 "one OR two OR three OR four"
2.2 "(one AND two) OR (three AND four)"
2.3 "(one AND two) OR (three AND four) NOT five"
2.4 "(one AND two) NOT 6"
3.1 "b:one AND a:two"
3.2 "b:one OR a:two"
3.3 "a:one OR b:1 OR {a b} : i"
4.1 "NEAR(one two, 2)"
4.2 "NEAR(one two three, 2)"
4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)"
} {
if {[fts5_expr_ok $expr ss]==0} {
do_test 1.$tn.OMITTED { list } [list]
continue
}
set res [fts5_query_data $expr ss ASC ::SYNDICT]
set res [fts5_query_data $expr ss ASC ::syn]
breakpoint
do_execsql_test 1.$tn.[llength $res].asc {
SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
} $res
manifest
+7 -7
View File
@@ -1,5 +1,5 @@
C Begin\sadding\sfts5\stests\sinvolving\ssynonyms\sand\sdetail=none/col\stables.
D 2016-01-08T17:21:18.901
C Fix\sbugs\sin\sfts5\ssynonym\sprocessing\sfor\sdetail=col\sand\sother\smodes.
D 2016-01-11T17:30:28.056
F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042
@@ -101,7 +101,7 @@ F ext/fts5/fts5Int.h 3918045ebceb1d600c9c1c1b460489ff0c788e96
F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e
F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1
F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6
F ext/fts5/fts5_expr.c 913ab9f9a7651d76f055f9847ca0ae0e2f5da78d
F ext/fts5/fts5_expr.c 6eba2220747ea1b20a358fb3b34b2ab78323e285
F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
F ext/fts5/fts5_index.c 04fe098ffab66d9424ba4e7c0b88ce7c98748cb1
F ext/fts5/fts5_main.c 03bd44e4bd0ba16213ca9259ad5df1d4d743fd7e
@@ -173,7 +173,7 @@ F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821
F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9
F ext/fts5/test/fts5simple2.test 843f1f7fe439ff32bf74f4fd6430632f9636ef3a
F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671
F ext/fts5/test/fts5synonym2.test 6aa842d0e5bd019db0c3597e0860eb68eb2867e4
F ext/fts5/test/fts5synonym2.test d2d9099d9d105b55ea03fd52d61ae2847d534129
F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
@@ -1410,7 +1410,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P d9135cc723fc4227aace6dcf4ffa4630c9d23aa0
R c443735bd0d019a3a4798f3f3350cb78
P b3e6f15ec2d9a834e2c80b91ffd7097553816228
R f239c4ba67c1753046748953bb2cde92
U dan
Z 48c62f9fa751846d5897bddd7cc5158f
Z 915aa1f2ec9df261dd260c37cdbace05
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
b3e6f15ec2d9a834e2c80b91ffd7097553816228
0e3c545423246231ddac8fed2f103d71d556e17f