paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add syntax to fts5 used to specify that a phrase or NEAR group should match a subset of columns. For example "[col1 col2 ...] : <phrase>".

Browse Source 
FossilOrigin-Name: 0fc0ea20920615f3e48ea2dbe2b7dcd979b0993e
This commit is contained in:
dan
2015-05-29 15:55:30 +00:00
parent e96efb6426
commit 6d21f42db2
7 changed files with 217 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ext/fts5/fts5Int.h
+8 -1
View File
@@ -511,6 +511,7 @@ typedef struct Fts5Parse Fts5Parse;
typedef struct Fts5Token Fts5Token;
typedef struct Fts5ExprPhrase Fts5ExprPhrase;
typedef struct Fts5ExprNearset Fts5ExprNearset;
typedef struct Fts5ExprColset Fts5ExprColset;
struct Fts5Token {
const char *p; /* Token text (not NULL terminated) */
@@ -578,12 +579,18 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
Fts5ExprPhrase*
);
Fts5ExprColset *sqlite3Fts5ParseColset(
Fts5Parse*,
Fts5ExprColset*,
Fts5Token *
);
void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseSetColumn(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5ExprColset*);
void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
ext/fts5/fts5_expr.c
+140 -32
View File
@@ -79,13 +79,23 @@ struct Fts5ExprPhrase {
Fts5ExprTerm aTerm[0]; /* Terms that make up this phrase */
};
/*
** If a NEAR() clump may only match a specific set of columns, then
** Fts5ExprNearset.pColset points to an object of the following type.
** Each entry in the aiCol[] array
*/
struct Fts5ExprColset {
int nCol;
int aiCol[1];
};
/*
** One or more phrases that must appear within a certain token distance of
** each other within each matching document.
*/
struct Fts5ExprNearset {
int nNear; /* NEAR parameter */
int iCol; /* Column to search (-1 -> all columns) */
Fts5ExprColset *pColset; /* Columns to search (NULL -> all columns) */
int nPhrase; /* Number of entries in aPhrase[] array */
Fts5ExprPhrase *apPhrase[0]; /* Array of phrase pointers */
};
@@ -136,6 +146,8 @@ static int fts5ExprGetToken(
switch( *z ){
case '(': tok = FTS5_LP; break;
case ')': tok = FTS5_RP; break;
case '[': tok = FTS5_LSP; break;
case ']': tok = FTS5_RSP; break;
case ':': tok = FTS5_COLON; break;
case ',': tok = FTS5_COMMA; break;
case '+': tok = FTS5_PLUS; break;
@@ -275,7 +287,6 @@ int sqlite3Fts5ExprPhraseExpr(
pNode->eType = FTS5_STRING;
pNode->pNear = pNear;
pNear->iCol = -1;
pNear->nPhrase = 1;
pNear->apPhrase[0] = pCopy;
@@ -335,7 +346,7 @@ void sqlite3Fts5ExprFree(Fts5Expr *p){
*/
static int fts5ExprPhraseIsMatch(
Fts5Expr *pExpr, /* Expression pPhrase belongs to */
int iCol, /* If >=0, search for matches in iCol only */
Fts5ExprColset *pColset, /* Restrict matches to these columns */
Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */
int *pbMatch /* OUT: Set to true if really a match */
){
@@ -344,6 +355,7 @@ static int fts5ExprPhraseIsMatch(
Fts5PoslistReader *aIter = aStatic;
int i;
int rc = SQLITE_OK;
int iCol = pColset ? pColset->aiCol[0] : -1;
fts5BufferZero(&pPhrase->poslist);
@@ -664,7 +676,6 @@ static int fts5ExprExtractCol(
int n, /* IN: Size of poslist in bytes */
int iCol /* Column to extract from poslist */
){
int ii;
int iCurrent = 0;
const u8 *p = *pa;
const u8 *pEnd = &p[n]; /* One byte past end of position list */
@@ -716,7 +727,6 @@ static int fts5ExprNearNextMatch(
int rc = SQLITE_OK;
while( 1 ){
int i;
if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){
/* If this "NEAR" object is actually a single phrase that consists
@@ -726,10 +736,11 @@ static int fts5ExprNearNextMatch(
** complicated phrase or NEAR expressions. */
Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
assert( pPhrase->poslist.nSpace==0 );
rc = sqlite3Fts5IterPoslist(pIter,
(const u8**)&pPhrase->poslist.p, &pPhrase->poslist.n, &pNode->iRowid
);
Fts5ExprColset *pColset = pNear->pColset;
const u8 *pPos;
int nPos;
rc = sqlite3Fts5IterPoslist(pIter, &pPos, &nPos, &pNode->iRowid);
/* If the term may match any column, then this must be a match.
** Return immediately in this case. Otherwise, try to find the
@@ -737,15 +748,31 @@ static int fts5ExprNearNextMatch(
** If it can be found, return. If it cannot, the next iteration
** of the loop will test the next rowid in the database for this
** term. */
if( pNear->iCol<0 ) return rc;
if( pColset==0 ){
assert( pPhrase->poslist.nSpace==0 );
pPhrase->poslist.p = (u8*)pPos;
pPhrase->poslist.n = nPos;
}else if( pColset->nCol==1 ){
assert( pPhrase->poslist.nSpace==0 );
pPhrase->poslist.n = fts5ExprExtractCol(&pPos, nPos, pColset->aiCol[0]);
pPhrase->poslist.p = (u8*)pPos;
}else{
int i;
fts5BufferZero(&pPhrase->poslist);
for(i=0; i<pColset->nCol; i++){
const u8 *pSub = pPos;
int nSub = fts5ExprExtractCol(&pSub, nPos, pColset->aiCol[i]);
if( nSub ){
fts5BufferAppendBlob(&rc, &pPhrase->poslist, nSub, pSub);
}
}
}
pPhrase->poslist.n = fts5ExprExtractCol(
(const u8**)&pPhrase->poslist.p,
pPhrase->poslist.n,
pNear->iCol
);
if( pPhrase->poslist.n ) return rc;
}else{
int i;
assert( pNear->pColset==0 || pNear->pColset->nCol==1 );
/* Advance the iterators until they all point to the same rowid */
rc = fts5ExprNearNextRowidMatch(pExpr, pNode);
@@ -756,14 +783,14 @@ static int fts5ExprNearNextMatch(
** phrase is not a match, break out of the loop early. */
for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
if( pPhrase->nTerm>1 || pNear->iCol>=0 ){
if( pPhrase->nTerm>1 || pNear->pColset ){
int bMatch = 0;
rc = fts5ExprPhraseIsMatch(pExpr, pNear->iCol, pPhrase, &bMatch);
rc = fts5ExprPhraseIsMatch(pExpr, pNear->pColset, pPhrase, &bMatch);
if( bMatch==0 ) break;
}else{
rc = sqlite3Fts5IterPoslistBuffer(
pPhrase->aTerm[0].pIter, &pPhrase->poslist
);
);
}
}
@@ -1152,7 +1179,6 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
pParse->rc = SQLITE_NOMEM;
}else{
memset(pRet, 0, nByte);
pRet->iCol = -1;
}
}else if( (pNear->nPhrase % SZALLOC)==0 ){
int nNew = pNear->nPhrase + SZALLOC;
@@ -1235,6 +1261,7 @@ void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset *pNear){
for(i=0; i<pNear->nPhrase; i++){
fts5ExprPhraseFree(pNear->apPhrase[i]);
}
sqlite3_free(pNear->pColset);
sqlite3_free(pNear);
}
}
@@ -1313,7 +1340,7 @@ void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token *pTok){
void sqlite3Fts5ParseSetDistance(
Fts5Parse *pParse,
Fts5ExprNearset *pNear,
Fts5ExprNearset *pNear,
Fts5Token *p
){
int nNear = 0;
@@ -1335,30 +1362,100 @@ void sqlite3Fts5ParseSetDistance(
pNear->nNear = nNear;
}
void sqlite3Fts5ParseSetColumn(
Fts5Parse *pParse,
Fts5ExprNearset *pNear,
/*
** The second argument passed to this function may be NULL, or it may be
** an existing Fts5ExprColset object. This function returns a pointer to
** a new colset object containing the contents of (p) with new value column
** number iCol appended.
**
** If an OOM error occurs, store an error code in pParse and return NULL.
** The old colset object (if any) is not freed in this case.
*/
static Fts5ExprColset *fts5ParseColset(
Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */
Fts5ExprColset *p, /* Existing colset object */
int iCol /* New column to add to colset object */
){
int nCol = p ? p->nCol : 0; /* Num. columns already in colset object */
Fts5ExprColset *pNew; /* New colset object to return */
assert( pParse->rc==SQLITE_OK );
assert( iCol>=0 && iCol<pParse->pConfig->nCol );
pNew = sqlite3_realloc(p, sizeof(Fts5ExprColset) + sizeof(int)*nCol);
if( pNew==0 ){
pParse->rc = SQLITE_NOMEM;
}else{
int *aiCol = pNew->aiCol;
int i, j;
for(i=0; i<nCol; i++){
if( aiCol[i]==iCol ) return pNew;
if( aiCol[i]>iCol ) break;
}
for(j=nCol; j>i; j--){
aiCol[j] = aiCol[j-1];
}
aiCol[i] = iCol;
pNew->nCol = nCol+1;
#ifndef NDEBUG
/* Check that the array is in order and contains no duplicate entries. */
for(i=1; i<pNew->nCol; i++) assert( pNew->aiCol[i]>pNew->aiCol[i-1] );
#endif
}
return pNew;
}
Fts5ExprColset *sqlite3Fts5ParseColset(
Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */
Fts5ExprColset *pColset, /* Existing colset object */
Fts5Token *p
){
Fts5ExprColset *pRet = 0;
if( pParse->rc==SQLITE_OK ){
int iCol;
char *z = 0;
int rc = fts5ParseStringFromToken(p, &z);
if( rc==SQLITE_OK ){
Fts5Config *pConfig = pParse->pConfig;
int i;
for(i=0; i<pConfig->nCol; i++){
if( 0==sqlite3_stricmp(pConfig->azCol[i], z) ){
pNear->iCol = i;
sqlite3Fts5Dequote(z);
for(iCol=0; iCol<pConfig->nCol; iCol++){
if( 0==sqlite3_stricmp(pConfig->azCol[iCol], z) ){
break;
}
}
if( i==pConfig->nCol ){
if( iCol==pConfig->nCol ){
sqlite3Fts5ParseError(pParse, "no such column: %s", z);
}
sqlite3_free(z);
}else{
pParse->rc = rc;
}
if( pParse->rc==SQLITE_OK ){
pRet = fts5ParseColset(pParse, pColset, iCol);
}
}
if( pParse->rc!=SQLITE_OK ){
assert( pRet==0 );
sqlite3_free(pColset);
}
return pRet;
}
void sqlite3Fts5ParseSetColset(
Fts5Parse *pParse,
Fts5ExprNearset *pNear,
Fts5ExprColset *pColset
){
if( pNear ){
pNear->pColset = pColset;
}else{
sqlite3_free(pColset);
}
}
@@ -1463,8 +1560,18 @@ static char *fts5ExprPrintTcl(
zRet = fts5PrintfAppend(zRet, "[%s ", zNearsetCmd);
if( zRet==0 ) return 0;
if( pNear->iCol>=0 ){
zRet = fts5PrintfAppend(zRet, "-col %d ", pNear->iCol);
if( pNear->pColset ){
int *aiCol = pNear->pColset->aiCol;
int nCol = pNear->pColset->nCol;
if( nCol==1 ){
zRet = fts5PrintfAppend(zRet, "-col %d ", aiCol[0]);
}else{
zRet = fts5PrintfAppend(zRet, "-col {%d", aiCol[0]);
for(i=1; i<pNear->pColset->nCol; i++){
zRet = fts5PrintfAppend(zRet, " %d", aiCol[i]);
}
zRet = fts5PrintfAppend(zRet, "} ");
}
if( zRet==0 ) return 0;
}
@@ -1530,8 +1637,9 @@ static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){
int i;
int iTerm;
if( pNear->iCol>=0 ){
zRet = fts5PrintfAppend(zRet, "%s : ", pConfig->azCol[pNear->iCol]);
if( pNear->pColset ){
int iCol = pNear->pColset->aiCol[0];
zRet = fts5PrintfAppend(zRet, "%s : ", pConfig->azCol[iCol]);
if( zRet==0 ) return 0;
}
ext/fts5/fts5_index.c
-2
View File
@@ -4479,7 +4479,6 @@ int sqlite3Fts5IterEof(Fts5IndexIter *pIter){
*/
int sqlite3Fts5IterNext(Fts5IndexIter *pIter){
assert( pIter->pIndex->rc==SQLITE_OK );
fts5BufferZero(&pIter->poslist);
fts5MultiIterNext(pIter->pIndex, pIter->pMulti, 0, 0);
return fts5IndexReturn(pIter->pIndex);
}
@@ -4494,7 +4493,6 @@ int sqlite3Fts5IterNextScan(Fts5IndexIter *pIter){
assert( pIter->pIndex->rc==SQLITE_OK );
assert( pMulti );
fts5BufferZero(&pIter->poslist);
fts5MultiIterNext(p, pMulti, 0, 0);
if( p->rc==SQLITE_OK ){
Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
ext/fts5/fts5parse.y
+19 -2
View File
@@ -95,11 +95,28 @@ exprlist(A) ::= exprlist(X) cnearset(Y). {
cnearset(A) ::= nearset(X). {
A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X);
}
cnearset(A) ::= STRING(X) COLON nearset(Y). {
sqlite3Fts5ParseSetColumn(pParse, Y, &X);
cnearset(A) ::= colset(X) COLON nearset(Y). {
sqlite3Fts5ParseSetColset(pParse, Y, X);
A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y);
}
%type colset {Fts5ExprColset*}
%destructor colset { sqlite3_free($$); }
%type colsetlist {Fts5ExprColset*}
%destructor colsetlist { sqlite3_free($$); }
colset(A) ::= LSP colsetlist(X) RSP. { A = X; }
colset(A) ::= STRING(X). {
A = sqlite3Fts5ParseColset(pParse, 0, &X);
}
colsetlist(A) ::= colsetlist(Y) STRING(X). {
A = sqlite3Fts5ParseColset(pParse, Y, &X); }
colsetlist(A) ::= STRING(X). {
A = sqlite3Fts5ParseColset(pParse, 0, &X);
}
%type nearset {Fts5ExprNearset*}
%type nearphrases {Fts5ExprNearset*}
%destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
ext/fts5/test/fts5ac.test
+39 -7
View File
@@ -125,13 +125,32 @@ set data {
99 {r c v w i v h a t a c v c r e} {h h u m g o f b a e o}
}
#-------------------------------------------------------------------------
# Usage:
#
# poslist aCol ?-pc VARNAME? ?-near N? ?-col C? -- phrase1 phrase2...
#
# This command is used to test if a document (set of column values) matches
# the logical equivalent of a single FTS5 NEAR() clump and, if so, return
# the equivalent of an FTS5 position list.
#
# Parameter $aCol is passed a list of the column values for the document
# to test. Parameters $phrase1 and so on are the phrases.
#
# The result is a list of phrase hits. Each phrase hit is formatted as
# three integers separated by "." characters, in the following format:
#
# <phrase number> . <column number> . <token offset>
#
# Options:
#
# -near N (NEAR distance. Default 10)
# -col C (List of column indexes to match against)
# -pc VARNAME (variable in caller frame to use for phrase numbering)
#
proc poslist {aCol args} {
set O(-near) 10
set O(-col) -1
set O(-col) {}
set O(-pc) ""
set nOpt [lsearch -exact $args --]
@@ -161,8 +180,7 @@ proc poslist {aCol args} {
set iCol -1
foreach col $aCol {
incr iCol
if {$O(-col)>=0 && $O(-col)!=$iCol} continue
if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue
set nToken [llength $col]
set iFL [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)]
@@ -361,10 +379,24 @@ foreach {tn2 sql} {
# Queries on a specific column.
#
foreach {tn expr} {
1 "x:a"
2 "y:a"
3 "x:b"
4 "y:b"
1.1 "x:a"
1.2 "y:a"
1.3 "x:b"
1.4 "y:b"
2.1 "[x]:a"
2.2 "[y]:a"
2.3 "[x]:b"
2.4 "[y]:b"
3.1 "[x y]:a"
3.2 "[y x]:a"
3.3 "[x x]:b"
3.4 "[y y]:b"
4.1 {["x" "y"]:a}
4.2 {["y" x]:a}
4.3 {[x "x"]:b}
4.4 {["y" y]:b}
} {
set res [matchdata 1 $expr]
do_execsql_test $tn2.3.$tn.[llength $res] {
manifest
+10 -10
View File
@@ -1,5 +1,5 @@
C Optimizations\sfor\sfts5\squeries\sthat\smatch\sagainst\sa\sspecific\scolumn.
D 2015-05-28T19:57:12.367
C Add\ssyntax\sto\sfts5\sused\sto\sspecify\sthat\sa\sphrase\sor\sNEAR\sgroup\sshould\smatch\sa\ssubset\sof\scolumns.\sFor\sexample\s"[col1\scol2\s...]\s:\s<phrase>".
D 2015-05-29T15:55:30.046
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 2c28e557780395095c307a6e5cb539419027eb5e
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -107,24 +107,24 @@ F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
F ext/fts5/fts5.c 74d18b4dc7518c7cd85609f1541e83bc564619a2
F ext/fts5/fts5.h 4266c6231094005b051dbfc8dd85d2bc57243d34
F ext/fts5/fts5Int.h 2ce5c5e68852dd16de404b7a9a2a78f4f4588eb4
F ext/fts5/fts5Int.h 3bcecc469fe570ab188d123e1d33d6e5e11a5129
F ext/fts5/fts5_aux.c d53f00f31ad615ca4f139dd8751f9041afa00971
F ext/fts5/fts5_buffer.c 861599a0abe2383f0cd0352c57001140a26b0930
F ext/fts5/fts5_config.c 11f969ed711a0a8b611d47431d74c372ad78c713
F ext/fts5/fts5_expr.c c94983eaff58391d7c0d62e99de917cecd0f1dbc
F ext/fts5/fts5_expr.c c607282529c7b5747fc2bcf80770d6abc22638bb
F ext/fts5/fts5_hash.c 54dd25348a46ea62ea96322c572e08cd1fb37304
F ext/fts5/fts5_index.c a693ba741b82539da5779329214e5d2609e82e5f
F ext/fts5/fts5_index.c 59b8a3dfde24ddb80c31088148a3dfc779db22ab
F ext/fts5/fts5_storage.c 5d2b51adb304643d8f825ba89283d628418b20c2
F ext/fts5/fts5_tcl.c 7ea165878e4ae3598e89acd470a0ee1b5a00e33c
F ext/fts5/fts5_tokenize.c 24649425adfea2c4877d8f69f2754b70374940ec
F ext/fts5/fts5_unicode2.c da3cf712f05cd8347c8c5bc00964cc0361c88da9
F ext/fts5/fts5_vocab.c 1f8543b2c1ae4427f127a911bc8e60873fcd7bf9
F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
F ext/fts5/fts5parse.y 4ee667932d561a150d96483cf563281b95a9e523
F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
F ext/fts5/test/fts5_common.tcl 6d663e8c3d8409857363f66560df96b8ca813e79
F ext/fts5/test/fts5aa.test 5f73afe6a1394fdba9bc18302876ded81021bee6
F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad
F ext/fts5/test/fts5ac.test 05008e00bd2761cc45df838a0988ecf318cbe1fd
F ext/fts5/test/fts5ac.test d35bbe22dd23b3dbac3e1d3f07eed0206213a480
F ext/fts5/test/fts5ad.test 312f3c8ed9592533499c5b94d2059ae6382913a0
F ext/fts5/test/fts5ae.test 9175201baf8c885fc1cbb2da11a0c61fd11224db
F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a
@@ -1331,7 +1331,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 0f9df202cc58097afddb8dad662b7c7fdc2c7d0c
R f25569d3bfd1393da78f5986e0e8acff
P b29ac50af0491a780a5a4c0985d88d0e5e014ba3
R 34ff180b006ad5f21871399f838e5dbb
U dan
Z d652fde1b36e85f62688dc3a9737ccda
Z f15f3e2fe41d81aa9045dd94b23ec6a4
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
b29ac50af0491a780a5a4c0985d88d0e5e014ba3
0fc0ea20920615f3e48ea2dbe2b7dcd979b0993e