paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix a problem with fts5 synonyms and phrase queries. Also fix an OOM handling bug in fts5.

Browse Source 
FossilOrigin-Name: a4c35fa2c94fe34b376670244fe72303c99868c1
This commit is contained in:
dan
2015-09-02 17:34:22 +00:00
parent 583611df65
commit bea34fc53a
6 changed files with 151 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ext/fts5/fts5_expr.c
+41 -20
View File
@@ -288,8 +288,8 @@ static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){
** Argument pTerm must be a synonym iterator. Return the current rowid
** that it points to.
*/
static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){
i64 iRet;
static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){
i64 iRet = 0;
int bRetValid = 0;
Fts5ExprTerm *p;
@@ -305,7 +305,7 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){
}
}
assert( bRetValid );
if( pbEof && bRetValid==0 ) *pbEof = 1;
return iRet;
}
@@ -346,9 +346,9 @@ static int fts5ExprSynonymPoslist(
nAlloc = nAlloc*2;
aIter = aNew;
}
if( sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter])==0 ){
nIter++;
}
sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter]);
assert( aIter[nIter].bEof==0 );
nIter++;
}
}
@@ -659,7 +659,7 @@ static int fts5ExprNearAdvanceFirst(
Fts5ExprTerm *p;
/* Find the firstest rowid any synonym points to. */
i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc);
i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0);
/* Advance each iterator that currently points to iRowid. Or, if iFrom
** is valid - each iterator that points to a rowid before iFrom. */
@@ -738,6 +738,35 @@ static int fts5ExprAdvanceto(
return 0;
}
static int fts5ExprSynonymAdvanceto(
Fts5ExprTerm *pTerm, /* Term iterator to advance */
int bDesc, /* True if iterator is "rowid DESC" */
i64 *piLast, /* IN/OUT: Lastest rowid seen so far */
int *pRc, /* OUT: Error code */
int *pbEof /* OUT: Set to true if EOF */
){
int rc = SQLITE_OK;
i64 iLast = *piLast;
Fts5ExprTerm *p;
for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){
if( sqlite3Fts5IterEof(p->pIter)==0 ){
i64 iRowid = sqlite3Fts5IterRowid(p->pIter);
if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){
rc = sqlite3Fts5IterNextFrom(p->pIter, iLast);
}
}
}
if( rc!=SQLITE_OK ){
*pbEof = 1;
}else{
*piLast = fts5ExprSynonymRowid(pTerm, bDesc, pbEof);
}
return rc;
}
/*
** IN/OUT parameter (*pa) points to a position list n bytes in size. If
** the position list contains entries for column iCol, then (*pa) is set
@@ -906,7 +935,7 @@ static int fts5ExprNearNextMatch(
** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it
** means the minimum rowid. */
if( pLeft->aTerm[0].pSynonym ){
iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc);
iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0);
}else{
iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter);
}
@@ -920,21 +949,13 @@ static int fts5ExprNearNextMatch(
if( pTerm->pSynonym ){
Fts5ExprTerm *p;
int bEof = 1;
i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc);
i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0);
if( iRowid==iLast ) continue;
for(p=pTerm; p; p=p->pSynonym){
Fts5IndexIter *pIter = p->pIter;
int dummy;
if( 0==sqlite3Fts5IterEof(pIter)
&& 0==fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &dummy)==0
){
bEof = 0;
}
}
if( bEof || rc ){
pNode->bEof = 1;
bMatch = 0;
if( fts5ExprSynonymAdvanceto(pTerm,bDesc,&iLast,&rc,&pNode->bEof) ){
return rc;
}
}else{
Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
i64 iRowid = sqlite3Fts5IterRowid(pIter);
ext/fts5/test/fts5fault6.test
+56 -2
View File
@@ -22,6 +22,8 @@ ifcapable !fts5 {
return
}
if 1 {
#-------------------------------------------------------------------------
# OOM while rebuilding an FTS5 table.
#
@@ -148,8 +150,14 @@ do_faultsim_test 4.1 -faults oom-t* -prep {
faultsim_test_result {0 {}}
}
}
#-------------------------------------------------------------------------
# OOM while running a query that includes synonyms and matchinfo().
#
# 5.2.* OOM while running a query that includes synonyms and matchinfo().
#
# 5.3.* OOM while running a query that returns a row containing instances
# of more than 4 synonyms for a single term.
#
proc mit {blob} {
set scan(littleEndian) i*
@@ -174,6 +182,7 @@ db func mit mit
sqlite3_fts5_register_matchinfo db
do_test 5.0 {
execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl) }
execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) }
foreach {rowid text} {
1 {aaaa cc b aaaaa cc aa}
2 {aa aa bb a bbb}
@@ -184,6 +193,37 @@ do_test 5.0 {
7 {aaaaa aa aa ccccc bb}
8 {ccc bbbbb ccccc bbb c}
9 {cccccc bbbb a aaa cccc c}
20 {ddd f ddd eeeee fff ffff eeee ddd fff eeeee dddddd eeee}
21 {fffff eee dddd fffff dd ee ee eeeee eee eeeeee ee dd e}
22 {fffff d eeee dddd fffff dddddd ffff ddddd eeeee ee eee dddd ddddd}
23 {ddddd fff ddd eeeee ffff eeee ddd ff ff ffffff eeeeee dddd ffffff}
24 {eee dd ee dddd dddd eeeeee e eee fff ffff}
25 {ddddd ffffff dddddd fff ddd ddddd ddd f eeee fff dddd f}
26 {f ffff fff fff eeeeee dddd d dddddd ddddd eee ff eeeee}
27 {eee fff dddddd eeeee eeeee dddd ddddd ffff f eeeee eee dddddd ddddd d}
28 {dd ddddd d ddd d fff d dddd ee dddd ee ddd dddddd dddddd}
29 {eeee dddd ee dddd eeee dddd dd fffff f ddd eeeee ddd ee}
30 {ff ffffff eeeeee eeeee eee ffffff ff ffff f fffff eeeee}
31 {fffff eeeeee dddd eeee eeee eeeeee eee fffff d ddddd ffffff ffff dddddd}
32 {dddddd fffff ee eeeeee eeee ee fff dddd fff eeee ffffff eeeeee ffffff}
33 {ddddd eeee dd ffff dddddd fff eeee ddddd ffff eeee ddd}
34 {ee dddd ddddd dddddd eeee eeeeee f dd ee dddddd ffffff}
35 {ee dddd dd eeeeee ddddd eee d eeeeee dddddd eee dddd fffff}
36 {eee ffffff ffffff e fffff eeeee ff dddddd dddddd fff}
37 {eeeee fffff dddddd dddd ffffff fff f dd ee dd dd eeeee}
38 {eeeeee ee d ff eeeeee eeeeee eee eeeee ee ffffff dddd eeee dddddd ee}
39 {eeeeee ddd fffff e dddd ee eee eee ffffff ee f d dddd}
40 {ffffff dddddd eee ee ffffff eee eeee ddddd ee eeeeee f}
41 {ddd ddd fff fffff ee fffff f fff ddddd fffff}
42 {dddd ee ff d f ffffff fff ffffff ff dd dddddd f eeee}
43 {d dd fff fffff d f fff e dddd ee ee}
44 {ff ffff eee ddd d dd ffff dddd d eeee d eeeeee}
45 {eeee f eeeee ee e ffff f ddd e fff}
46 {ffff d ffff eeee ffff eeeee f ffff ddddd eee}
47 {dd dd dddddd ddddd fffff dddddd ddd ddddd eeeeee ffff eeee eee ee}
48 {ffff ffff e dddd ffffff dd dd dddd f fffff}
49 {ffffff d dddddd ffff eeeee f ffff ffff d dd fffff eeeee}
} {
execsql { INSERT INTO t1(rowid, a) VALUES($rowid, $text) }
}
@@ -196,9 +236,12 @@ set res [list {*}{
7 {3 24 8 1 12 6}
9 {2 24 8 3 12 6}
}]
do_execsql_test 5.1 {
do_execsql_test 5.1.1 {
SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c'
} $res
do_execsql_test 5.1.2 {
SELECT count(*) FROM t1 WHERE t1 MATCH 'd e f'
} 29
faultsim_save_and_close
do_faultsim_test 5.2 -faults oom* -prep {
@@ -214,5 +257,16 @@ do_faultsim_test 5.2 -faults oom* -prep {
faultsim_test_result [list 0 $::res]
}
do_faultsim_test 5.3 -faults oom* -prep {
faultsim_restore_and_reopen
sqlite3_fts5_create_tokenizer db tcl tcl_create
} -body {
db eval {
SELECT count(*) FROM t1 WHERE t1 MATCH 'd AND e AND f'
}
} -test {
faultsim_test_result {0 29}
}
finish_test
ext/fts5/test/fts5synonym.test
+19
View File
@@ -93,6 +93,7 @@ foreach {tn expr res} {
2 "eight OR 8 OR 5" {2 3}
3 "10" {}
4 "1*" {1}
5 "1 + 2" {1}
} {
do_execsql_test 2.1.$tn {
SELECT rowid FROM ft WHERE ft MATCH $expr
@@ -233,6 +234,7 @@ do_test 5.0 {
}
} {}
foreach {tn q res} {
1 {one} {
1 {four v 4 [i] three} {[1] 3 five five 4 [one]}
@@ -265,6 +267,23 @@ foreach {tn q res} {
8 {[2] [ii] [i] [two] [3] [three] [2]} {[two] [iv] [v] [iii] [3] [five]}
9 {[i] [2] [iv] [3] [five] [four] [v]} {[iii] [4] [three] [i] [three] [ii] [1]}
}
4 {5 + 1} {
2 {[5 1] 3 4 i} {2 2 v two 4}
3 {[5 i] 5 2 four 4 1} {iii ii five two 1}
4 {ii four 4 one 5 three five} {one [5 1] iii 4 3}
5 {three i [v i] four 4 1} {ii five five five iii}
}
5 {one + two + three} {
7 {ii ii two three 2 5} {iii [i ii iii] iii one one}
8 {2 ii [i two 3] three 2} {two iv v iii 3 five}
}
6 {"v v"} {
1 {four v 4 i three} {1 3 [five five] 4 one}
5 {three i v i four 4 1} {ii [five five five] iii}
}
} {
do_execsql_test 5.1.$tn {
SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']')