paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve performance of prefix queries without a prefix index on fts5 tables.

Browse Source 
FossilOrigin-Name: f2f0184e9e1c9f121ee2ac864cd28c8cd8efecb5
This commit is contained in:
dan
2015-10-05 19:41:16 +00:00
parent f24bebe37a
commit 6f832d3dff
4 changed files with 98 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ext/fts5/fts5_buffer.c
+8 -6
View File
@@ -233,13 +233,15 @@ int sqlite3Fts5PoslistWriterAppend(
){
static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
int rc = SQLITE_OK;
if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
fts5BufferAppendVarint(&rc, pBuf, 1);
fts5BufferAppendVarint(&rc, pBuf, (iPos >> 32));
pWriter->iPrev = (iPos & colmask);
if( 0==sqlite3Fts5BufferGrow(&rc, pBuf, 5+5+5) ){
if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
pBuf->p[pBuf->n++] = 1;
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
pWriter->iPrev = (iPos & colmask);
}
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2);
pWriter->iPrev = iPos;
}
fts5BufferAppendVarint(&rc, pBuf, (iPos - pWriter->iPrev) + 2);
pWriter->iPrev = iPos;
return rc;
}
ext/fts5/fts5_index.c
+82 -54
View File
@@ -307,9 +307,7 @@ struct Fts5Index {
};
struct Fts5DoclistIter {
u8 *a;
int n;
int i;
u8 *aEof; /* Pointer to 1 byte past end of doclist */
/* Output variables. aPoslist==0 at EOF */
i64 iRowid;
@@ -3706,10 +3704,15 @@ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){
return ret;
}
#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \
assert( pBuf->nSpace>=(pBuf->n+nBlob) ); \
memcpy(&pBuf->p[pBuf->n], pBlob, nBlob); \
pBuf->n += nBlob; \
#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \
assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) ); \
memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob); \
(pBuf)->n += nBlob; \
}
#define fts5BufferSafeAppendVarint(pBuf, iVal) { \
(pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal)); \
assert( (pBuf)->nSpace>=(pBuf)->n ); \
}
/*
@@ -3989,22 +3992,28 @@ static void fts5MultiIterPoslist(
}
static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
if( pIter->i<pIter->n ){
int bDummy;
if( pIter->i ){
i64 iDelta;
pIter->i += fts5GetVarint(&pIter->a[pIter->i], (u64*)&iDelta);
pIter->iRowid += iDelta;
}else{
pIter->i += fts5GetVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid);
}
pIter->i += fts5GetPoslistSize(
&pIter->a[pIter->i], &pIter->nPoslist, &bDummy
);
pIter->aPoslist = &pIter->a[pIter->i];
pIter->i += pIter->nPoslist;
}else{
u8 *p = pIter->aPoslist + pIter->nPoslist;
assert( pIter->aPoslist );
if( p>=pIter->aEof ){
pIter->aPoslist = 0;
}else{
i64 iDelta;
p += fts5GetVarint(p, (u64*)&iDelta);
pIter->iRowid += iDelta;
/* Read position list size */
if( p[0] & 0x80 ){
int nPos;
p += fts5GetVarint32(p, nPos);
pIter->nPoslist = (nPos>>1);
}else{
pIter->nPoslist = ((int)(p[0])) >> 1;
p++;
}
pIter->aPoslist = p;
}
}
@@ -4013,27 +4022,34 @@ static void fts5DoclistIterInit(
Fts5DoclistIter *pIter
){
memset(pIter, 0, sizeof(*pIter));
pIter->a = pBuf->p;
pIter->n = pBuf->n;
pIter->aPoslist = pBuf->p;
pIter->aEof = &pBuf->p[pBuf->n];
fts5DoclistIterNext(pIter);
}
#if 0
/*
** Append a doclist to buffer pBuf.
**
** This function assumes that space within the buffer has already been
** allocated.
*/
static void fts5MergeAppendDocid(
int *pRc, /* IN/OUT: Error code */
Fts5Buffer *pBuf, /* Buffer to write to */
i64 *piLastRowid, /* IN/OUT: Previous rowid written (if any) */
i64 iRowid /* Rowid to append */
){
if( pBuf->n==0 ){
fts5BufferAppendVarint(pRc, pBuf, iRowid);
}else{
fts5BufferAppendVarint(pRc, pBuf, iRowid - *piLastRowid);
}
assert( pBuf->n!=0 || (*piLastRowid)==0 );
fts5BufferSafeAppendVarint(pBuf, iRowid - *piLastRowid);
*piLastRowid = iRowid;
}
#endif
#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \
assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \
fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \
(iLastRowid) = (iRowid); \
}
/*
** Buffers p1 and p2 contain doclists. This function merges the content
@@ -4057,53 +4073,60 @@ static void fts5MergePrefixLists(
memset(&out, 0, sizeof(out));
memset(&tmp, 0, sizeof(tmp));
sqlite3Fts5BufferGrow(&p->rc, &out, p1->n + p2->n);
fts5DoclistIterInit(p1, &i1);
fts5DoclistIterInit(p2, &i2);
while( p->rc==SQLITE_OK && (i1.aPoslist!=0 || i2.aPoslist!=0) ){
if( i2.aPoslist==0 || (i1.aPoslist && i1.iRowid<i2.iRowid) ){
/* Copy entry from i1 */
fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i1.iRowid);
fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
/* WRITEPOSLISTSIZE */
fts5BufferAppendVarint(&p->rc, &out, i1.nPoslist * 2);
fts5BufferAppendBlob(&p->rc, &out, i1.nPoslist, i1.aPoslist);
fts5BufferSafeAppendVarint(&out, i1.nPoslist * 2);
fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist);
fts5DoclistIterNext(&i1);
}
else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){
/* Copy entry from i2 */
fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid);
fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
/* WRITEPOSLISTSIZE */
fts5BufferAppendVarint(&p->rc, &out, i2.nPoslist * 2);
fts5BufferAppendBlob(&p->rc, &out, i2.nPoslist, i2.aPoslist);
fts5BufferSafeAppendVarint(&out, i2.nPoslist * 2);
fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist);
fts5DoclistIterNext(&i2);
}
else{
Fts5PoslistReader r1;
Fts5PoslistReader r2;
Fts5PoslistWriter writer;
i64 iPos1 = 0;
i64 iPos2 = 0;
int iOff1 = 0;
int iOff2 = 0;
Fts5PoslistWriter writer;
memset(&writer, 0, sizeof(writer));
/* Merge the two position lists. */
fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid);
fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
fts5BufferZero(&tmp);
sqlite3Fts5PoslistReaderInit(-1, i1.aPoslist, i1.nPoslist, &r1);
sqlite3Fts5PoslistReaderInit(-1, i2.aPoslist, i2.nPoslist, &r2);
while( p->rc==SQLITE_OK && (r1.bEof==0 || r2.bEof==0) ){
sqlite3Fts5PoslistNext64(i1.aPoslist, i1.nPoslist, &iOff1, &iPos1);
sqlite3Fts5PoslistNext64(i2.aPoslist, i2.nPoslist, &iOff2, &iPos2);
while( p->rc==SQLITE_OK && (iPos1>=0 || iPos2>=0) ){
i64 iNew;
if( r2.bEof || (r1.bEof==0 && r1.iPos<r2.iPos) ){
iNew = r1.iPos;
sqlite3Fts5PoslistReaderNext(&r1);
if( iPos2<0 || (iPos1>=0 && iPos1<iPos2) ){
iNew = iPos1;
sqlite3Fts5PoslistNext64(i1.aPoslist, i1.nPoslist, &iOff1, &iPos1);
}else{
iNew = r2.iPos;
sqlite3Fts5PoslistReaderNext(&r2);
if( r1.iPos==r2.iPos ) sqlite3Fts5PoslistReaderNext(&r1);
iNew = iPos2;
sqlite3Fts5PoslistNext64(i2.aPoslist, i2.nPoslist, &iOff2, &iPos2);
if( iPos1==iPos2 ){
sqlite3Fts5PoslistNext64(i1.aPoslist, i1.nPoslist, &iOff1,&iPos1);
}
}
p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
}
/* WRITEPOSLISTSIZE */
fts5BufferAppendVarint(&p->rc, &out, tmp.n * 2);
fts5BufferAppendBlob(&p->rc, &out, tmp.n, tmp.p);
fts5BufferSafeAppendVarint(&out, tmp.n * 2);
fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n);
fts5DoclistIterNext(&i1);
fts5DoclistIterNext(&i2);
}
@@ -4165,14 +4188,19 @@ static void fts5SetupPrefixIter(
fts5BufferZero(&aBuf[i]);
}
}
iLastRowid = 0;
}
fts5MergeAppendDocid(&p->rc, &doclist, &iLastRowid, iRowid);
fts5MultiIterPoslist(p, p1, 1, &doclist);
if( 0==sqlite3Fts5BufferGrow(&p->rc, &doclist, 9) ){
fts5MergeAppendDocid(&doclist, iLastRowid, iRowid);
fts5MultiIterPoslist(p, p1, 1, &doclist);
}
}
for(i=0; i<nBuf; i++){
fts5MergePrefixLists(p, &doclist, &aBuf[i]);
if( p->rc==SQLITE_OK ){
fts5MergePrefixLists(p, &doclist, &aBuf[i]);
}
fts5BufferFree(&aBuf[i]);
}
fts5MultiIterFree(p, p1);
manifest
+7 -7
View File
@@ -1,5 +1,5 @@
C Update\sfts3\sso\sthat\sexpressions\sto\sthe\sleft\sand\sright\sof\sa\sNOT\soperator\sare\sbalanced.\sThis\sprevents\srelatively\ssmall\sexpressions\s(a\sdozen\sterms\sor\sso)\sthat\sare\schildren\sof\sNOT\soperators\sfrom\striggering\sthe\s"expression\stree\sis\stoo\slarge"\serror.
D 2015-10-05T15:39:45.681
C Improve\sperformance\sof\sprefix\squeries\swithout\sa\sprefix\sindex\son\sfts5\stables.
D 2015-10-05T19:41:16.348
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 2143eeef6d0cc26006ae5fc4bb242a4a8b973412
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -108,11 +108,11 @@ F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
F ext/fts5/fts5.h 98f802fe41481f9d797fce496f0fefcad72c7782
F ext/fts5/fts5Int.h ff78a77d819a7fc04a7f8b08b0e1ce361a3395e4
F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e
F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4
F ext/fts5/fts5_buffer.c 54b18497395a19dfe1d00f63a3b403e5f93d4fd1
F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695
F ext/fts5/fts5_expr.c 667faaf14a69a5683ac383acdc8d942cf32c3f93
F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246
F ext/fts5/fts5_index.c 00d2593f94ede440ea274f8db21864cf41632aa3
F ext/fts5/fts5_index.c ca3912a44ef5a173ef098f3454465519bd4b8e88
F ext/fts5/fts5_main.c fe5243d6bbb79217394f0ec7f4f5199ddbc9e7e8
F ext/fts5/fts5_storage.c df061a5caf9e50fbbd43113009b5b248362f4995
F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37
@@ -1391,7 +1391,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 44f1ce30d1e446c9ee5f8bd8a62119e00356aa0e
R 0954493ffc80fb341b0df1cab997ace8
P d6b66cd7b89fbd964f798d160a34caac0ba7347a
R 0053d2d2b912a0e9b79250c67681b097
U dan
Z 52900cdc8d31460ed8eb8d6276be0e92
Z c981c12e0ed92ecd6400cdfa9ecf4084
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
d6b66cd7b89fbd964f798d160a34caac0ba7347a
f2f0184e9e1c9f121ee2ac864cd28c8cd8efecb5