From d12b636330335bde32dd1de144596f87683cdf9d Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 11 Oct 2015 19:46:59 +0000 Subject: [PATCH 1/7] Improved substitution logic in the query flattener. Saves code space, and (more importantly) works correctly with table-valued functions. FossilOrigin-Name: 3d0bd95e977db50c314d33ec292f99e0539d7b4a --- manifest | 14 +++++------ manifest.uuid | 2 +- src/select.c | 58 +++++++++++++++++++-------------------------- test/tabfunc01.test | 6 ++++- 4 files changed, 38 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index fa290bfaef..2aad6581bf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\saround\sa\s"security\sfeature"\sbug\sin\smemcpy()\son\sOpenBSD. -D 2015-10-10T16:41:28.969 +C Improved\ssubstitution\slogic\sin\sthe\squery\sflattener.\s\sSaves\scode\sspace,\sand\n(more\simportantly)\sworks\scorrectly\swith\stable-valued\sfunctions. +D 2015-10-11T19:46:59.677 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f0088ff0d2ac949fce6de7c00f13a99ac5bdb663 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -343,7 +343,7 @@ F src/printf.c 0c4bcdd1c2e2521024f0a69cb5eb334f86b3652a F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 1954a0f01bf65d78d7d559aea3d5c67f33376d91 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c 36c05502d5aef2a14aff3f7faba95b0c7d87ca9a +F src/select.c 0e0899675f9f2bdf93f5779906709f5c3009d465 F src/shell.c d25df04168d6ba5a4fa05bdbf859df667f9eb621 F src/sqlite.h.in 4b76d74d69af48c534c58fb723137dc6944bdedc F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad @@ -1048,7 +1048,7 @@ F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test d2fdaad713f103ac611fe7ef9b724c7b69f8149c F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 -F test/tabfunc01.test 83e63be7b6e3f67b6a03519c9c61bc68efb25f31 +F test/tabfunc01.test 03c4ad422c6ab596cff6dcaf86dd061a9f039525 F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 @@ -1390,7 +1390,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 35e6248abb4435a8b26d270092b856beff867406 -R 9fd826df5b8be6e5111bab176d72d145 +P fab6f09044d033dd09ed8a22e06bc6a7851bbabf +R 0a60e7c34e5485b5033bf31572e696c0 U drh -Z 61d2ace382c5fa85de2f4090e2c287bd +Z 976ba04014f2425afb980aca8bc89218 diff --git a/manifest.uuid b/manifest.uuid index ea0c817d83..a287e72de8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fab6f09044d033dd09ed8a22e06bc6a7851bbabf \ No newline at end of file +3d0bd95e977db50c314d33ec292f99e0539d7b4a \ No newline at end of file diff --git a/src/select.c b/src/select.c index c865472334..1010289aa5 100644 --- a/src/select.c +++ b/src/select.c @@ -3082,7 +3082,7 @@ static int multiSelectOrderBy( #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* Forward Declarations */ static void substExprList(sqlite3*, ExprList*, int, ExprList*); -static void substSelect(sqlite3*, Select *, int, ExprList *); +static void substSelect(sqlite3*, Select *, int, ExprList*, int); /* ** Scan through the expression pExpr. Replace every reference to @@ -3119,7 +3119,7 @@ static Expr *substExpr( pExpr->pLeft = substExpr(db, pExpr->pLeft, iTable, pEList); pExpr->pRight = substExpr(db, pExpr->pRight, iTable, pEList); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - substSelect(db, pExpr->x.pSelect, iTable, pEList); + substSelect(db, pExpr->x.pSelect, iTable, pEList, 1); }else{ substExprList(db, pExpr->x.pList, iTable, pEList); } @@ -3142,25 +3142,29 @@ static void substSelect( sqlite3 *db, /* Report malloc errors here */ Select *p, /* SELECT statement in which to make substitutions */ int iTable, /* Table to be replaced */ - ExprList *pEList /* Substitute values */ + ExprList *pEList, /* Substitute values */ + int doPrior /* Do substitutes on p->pPrior too */ ){ SrcList *pSrc; struct SrcList_item *pItem; int i; if( !p ) return; - substExprList(db, p->pEList, iTable, pEList); - substExprList(db, p->pGroupBy, iTable, pEList); - substExprList(db, p->pOrderBy, iTable, pEList); - p->pHaving = substExpr(db, p->pHaving, iTable, pEList); - p->pWhere = substExpr(db, p->pWhere, iTable, pEList); - substSelect(db, p->pPrior, iTable, pEList); - pSrc = p->pSrc; - assert( pSrc ); /* Even for (SELECT 1) we have: pSrc!=0 but pSrc->nSrc==0 */ - if( ALWAYS(pSrc) ){ - for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(db, pItem->pSelect, iTable, pEList); + do{ + substExprList(db, p->pEList, iTable, pEList); + substExprList(db, p->pGroupBy, iTable, pEList); + substExprList(db, p->pOrderBy, iTable, pEList); + p->pHaving = substExpr(db, p->pHaving, iTable, pEList); + p->pWhere = substExpr(db, p->pWhere, iTable, pEList); + pSrc = p->pSrc; + if( pSrc ){ + for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ + substSelect(db, pItem->pSelect, iTable, pEList, 1); + if( pItem->fg.isTabFunc ){ + substExprList(db, pItem->u1.pFuncArg, iTable, pEList); + } + } } - } + }while( doPrior && (p = p->pPrior)!=0 ); } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ @@ -3312,7 +3316,7 @@ static int flattenSubquery( int subqueryIsAgg /* True if the subquery uses aggregate functions */ ){ const char *zSavedAuthContext = pParse->zAuthContext; - Select *pParent; + Select *pParent; /* Current UNION ALL term of the other query */ Select *pSub; /* The inner query or "subquery" */ Select *pSub1; /* Pointer to the rightmost select in sub-query */ SrcList *pSrc; /* The FROM clause of the outer query */ @@ -3607,9 +3611,9 @@ static int flattenSubquery( ** ** The outer query has 3 slots in its FROM clause. One slot of the ** outer query (the middle slot) is used by the subquery. The next - ** block of code will expand the out query to 4 slots. The middle - ** slot is expanded to two slots in order to make space for the - ** two elements in the FROM clause of the subquery. + ** block of code will expand the outer query FROM clause to 4 slots. + ** The middle slot is expanded to two slots in order to make space + ** for the two elements in the FROM clause of the subquery. */ if( nSubSrc>1 ){ pParent->pSrc = pSrc = sqlite3SrcListEnlarge(db, pSrc, nSubSrc-1,iFrom+1); @@ -3648,11 +3652,6 @@ static int flattenSubquery( pList->a[i].zName = zName; } } - substExprList(db, pParent->pEList, iParent, pSub->pEList); - if( isAgg ){ - substExprList(db, pParent->pGroupBy, iParent, pSub->pEList); - pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); - } if( pSub->pOrderBy ){ /* At this point, any non-zero iOrderByCol values indicate that the ** ORDER BY column expression is identical to the iOrderByCol'th @@ -3672,27 +3671,20 @@ static int flattenSubquery( assert( pSub->pPrior==0 ); pParent->pOrderBy = pOrderBy; pSub->pOrderBy = 0; - }else if( pParent->pOrderBy ){ - substExprList(db, pParent->pOrderBy, iParent, pSub->pEList); - } - if( pSub->pWhere ){ - pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); - }else{ - pWhere = 0; } + pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); if( subqueryIsAgg ){ assert( pParent->pHaving==0 ); pParent->pHaving = pParent->pWhere; pParent->pWhere = pWhere; - pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); pParent->pHaving = sqlite3ExprAnd(db, pParent->pHaving, sqlite3ExprDup(db, pSub->pHaving, 0)); assert( pParent->pGroupBy==0 ); pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0); }else{ - pParent->pWhere = substExpr(db, pParent->pWhere, iParent, pSub->pEList); pParent->pWhere = sqlite3ExprAnd(db, pParent->pWhere, pWhere); } + substSelect(db, pParent, iParent, pSub->pEList, 0); /* The flattened query is distinct if either the inner or the ** outer query is distinct. diff --git a/test/tabfunc01.test b/test/tabfunc01.test index 5bd729b87d..47af39606e 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -75,8 +75,12 @@ do_execsql_test tabfunc01-2.1 { INSERT INTO t1(x) VALUES(2),(3); SELECT *, '|' FROM t1, generate_series(1,x) ORDER BY 1, 2 } {2 1 | 2 2 | 3 1 | 3 2 | 3 3 |} - do_execsql_test tabfunc01-2.2 { + SELECT *, '|' FROM (SELECT x FROM t1) AS y, generate_series(1,y.x) + ORDER BY 1, 2; +} {2 1 | 2 2 | 3 1 | 3 2 | 3 3 |} + +do_execsql_test tabfunc01-2.50 { SELECT * FROM generate_series() LIMIT 5; } {0 1 2 3 4} From 2906490b716daac524fbb3c1863c3ab9f4e2fc64 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 11 Oct 2015 20:08:31 +0000 Subject: [PATCH 2/7] Remove an unreachable branch in the query flattener substitution logic. FossilOrigin-Name: 46ee3453a31b5e793d5aaad471cec1b1a11ab2cf --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 11 +++++------ 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 2aad6581bf..a6ed3fb2d1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\ssubstitution\slogic\sin\sthe\squery\sflattener.\s\sSaves\scode\sspace,\sand\n(more\simportantly)\sworks\scorrectly\swith\stable-valued\sfunctions. -D 2015-10-11T19:46:59.677 +C Remove\san\sunreachable\sbranch\sin\sthe\squery\sflattener\ssubstitution\slogic. +D 2015-10-11T20:08:31.981 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f0088ff0d2ac949fce6de7c00f13a99ac5bdb663 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -343,7 +343,7 @@ F src/printf.c 0c4bcdd1c2e2521024f0a69cb5eb334f86b3652a F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 1954a0f01bf65d78d7d559aea3d5c67f33376d91 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c 0e0899675f9f2bdf93f5779906709f5c3009d465 +F src/select.c f70b36caa607655a8417198389f1802a56b1c8b6 F src/shell.c d25df04168d6ba5a4fa05bdbf859df667f9eb621 F src/sqlite.h.in 4b76d74d69af48c534c58fb723137dc6944bdedc F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad @@ -1390,7 +1390,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fab6f09044d033dd09ed8a22e06bc6a7851bbabf -R 0a60e7c34e5485b5033bf31572e696c0 +P 3d0bd95e977db50c314d33ec292f99e0539d7b4a +R 0631d5a91a7b127155b4e818909096dd U drh -Z 976ba04014f2425afb980aca8bc89218 +Z ec249e682dee8ecef8754c1790b6f485 diff --git a/manifest.uuid b/manifest.uuid index a287e72de8..8d1e749a05 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3d0bd95e977db50c314d33ec292f99e0539d7b4a \ No newline at end of file +46ee3453a31b5e793d5aaad471cec1b1a11ab2cf \ No newline at end of file diff --git a/src/select.c b/src/select.c index 1010289aa5..566cec44b8 100644 --- a/src/select.c +++ b/src/select.c @@ -3156,12 +3156,11 @@ static void substSelect( p->pHaving = substExpr(db, p->pHaving, iTable, pEList); p->pWhere = substExpr(db, p->pWhere, iTable, pEList); pSrc = p->pSrc; - if( pSrc ){ - for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(db, pItem->pSelect, iTable, pEList, 1); - if( pItem->fg.isTabFunc ){ - substExprList(db, pItem->u1.pFuncArg, iTable, pEList); - } + assert( pSrc!=0 ); + for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ + substSelect(db, pItem->pSelect, iTable, pEList, 1); + if( pItem->fg.isTabFunc ){ + substExprList(db, pItem->u1.pFuncArg, iTable, pEList); } } }while( doPrior && (p = p->pPrior)!=0 ); From bfad7be78a7080c49b56bb65428d3078972f18e5 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 11 Oct 2015 20:39:46 +0000 Subject: [PATCH 3/7] Detect and report circularly defined views even if the views have the columns defined in the CREATE VIEW statement. FossilOrigin-Name: 9ab9c8c6d747647f8ade58c2c4812fc69a813368 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 4 ++++ test/view.test | 7 +++++++ 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a6ed3fb2d1..5893122f2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunreachable\sbranch\sin\sthe\squery\sflattener\ssubstitution\slogic. -D 2015-10-11T20:08:31.981 +C Detect\sand\sreport\scircularly\sdefined\sviews\seven\sif\sthe\sviews\shave\sthe\ncolumns\sdefined\sin\sthe\sCREATE\sVIEW\sstatement. +D 2015-10-11T20:39:46.984 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f0088ff0d2ac949fce6de7c00f13a99ac5bdb663 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -343,7 +343,7 @@ F src/printf.c 0c4bcdd1c2e2521024f0a69cb5eb334f86b3652a F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 1954a0f01bf65d78d7d559aea3d5c67f33376d91 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c f70b36caa607655a8417198389f1802a56b1c8b6 +F src/select.c 0bc9cd2e1cacfdc9cdc9a83884cc100f166e80a4 F src/shell.c d25df04168d6ba5a4fa05bdbf859df667f9eb621 F src/sqlite.h.in 4b76d74d69af48c534c58fb723137dc6944bdedc F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad @@ -1253,7 +1253,7 @@ F test/vacuum3.test 77ecdd54592b45a0bcb133339f99f1ae0ae94d0d F test/vacuum4.test d3f8ecff345f166911568f397d2432c16d2867d9 F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 -F test/view.test 0d8935e64867ae01492347884f5cbb47f056f4c8 +F test/view.test f6c3a39e0c819891265e1d0754e99960d81ef6c9 F test/vtab1.test 6210e076997f176bedc300a87ad6404651b601dd F test/vtab2.test f8cd1bb9aba7143eba97812d9617880a36d247ad F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e @@ -1390,7 +1390,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3d0bd95e977db50c314d33ec292f99e0539d7b4a -R 0631d5a91a7b127155b4e818909096dd +P 46ee3453a31b5e793d5aaad471cec1b1a11ab2cf +R f02e8925b44dc59b1569a7e5ccbd6813 U drh -Z ec249e682dee8ecef8754c1790b6f485 +Z 2390f857ae51a8bc2278cbec6c80566a diff --git a/manifest.uuid b/manifest.uuid index 8d1e749a05..678dba625e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -46ee3453a31b5e793d5aaad471cec1b1a11ab2cf \ No newline at end of file +9ab9c8c6d747647f8ade58c2c4812fc69a813368 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 566cec44b8..fad46f0b47 100644 --- a/src/select.c +++ b/src/select.c @@ -4248,6 +4248,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pTab->nRef++; #if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) if( pTab->pSelect || IsVirtual(pTab) ){ + i16 nCol; if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; assert( pFrom->pSelect==0 ); if( pFrom->fg.isTabFunc && !IsVirtual(pTab) ){ @@ -4256,7 +4257,10 @@ static int selectExpander(Walker *pWalker, Select *p){ } pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); sqlite3SelectSetName(pFrom->pSelect, pTab->zName); + nCol = pTab->nCol; + pTab->nCol = -1; sqlite3WalkSelect(pWalker, pFrom->pSelect); + pTab->nCol = nCol; } #endif } diff --git a/test/view.test b/test/view.test index 0b8354a38e..235f8605ac 100644 --- a/test/view.test +++ b/test/view.test @@ -499,6 +499,13 @@ do_test view-14.1 { SELECT * FROM temp.t1; } } {1 {view t1 is circularly defined}} +do_test view-14.2 { + catchsql { + DROP VIEW IF EXISTS temp.t1; + CREATE TEMP VIEW t1(a,b) AS SELECT a,b FROM t1; + SELECT * FROM temp.t1; + } +} {1 {view t1 is circularly defined}} # Tickets #1688, #1709 # From 58a8a9236efe5e925854f00d715834a56e178b0d Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 12 Oct 2015 04:56:12 +0000 Subject: [PATCH 4/7] Change all references to 3.8.12 into 3.9.0. Comment changes only - no changes to code. FossilOrigin-Name: 6f2858f6817ca70c132f0437ac2f0f74deb273d2 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/loadext.c | 2 +- src/sqlite.h.in | 6 +++--- src/sqlite3ext.h | 4 ++-- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 5893122f2e..3e2bd9208d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Detect\sand\sreport\scircularly\sdefined\sviews\seven\sif\sthe\sviews\shave\sthe\ncolumns\sdefined\sin\sthe\sCREATE\sVIEW\sstatement. -D 2015-10-11T20:39:46.984 +C Change\sall\sreferences\sto\s3.8.12\sinto\s3.9.0.\s\sComment\schanges\sonly\s-\sno\nchanges\sto\scode. +D 2015-10-12T04:56:12.349 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f0088ff0d2ac949fce6de7c00f13a99ac5bdb663 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -307,7 +307,7 @@ F src/insert.c 3c522beb5bf50a2efee1fca1e80fd40942e5817c F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c d344a95d60c24e2f490ee59db9784b1b17439012 -F src/loadext.c b3f1a529245b14c881b270a9f29fae6c5727cde8 +F src/loadext.c c5916a158f1fb74d955847c84f7ebdb6f5de9d8c F src/main.c fec97668771438033a7559883401067b139729e1 F src/malloc.c 3a37ce6979a40f499d8cea9e9ab4e8517854d35d F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -345,9 +345,9 @@ F src/resolve.c 1954a0f01bf65d78d7d559aea3d5c67f33376d91 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 0bc9cd2e1cacfdc9cdc9a83884cc100f166e80a4 F src/shell.c d25df04168d6ba5a4fa05bdbf859df667f9eb621 -F src/sqlite.h.in 4b76d74d69af48c534c58fb723137dc6944bdedc +F src/sqlite.h.in dc35357824c6bb84a04acd4e99dee3e07299a8c6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad -F src/sqlite3ext.h 64350bf36833a56ad675e27392a913f417c5c308 +F src/sqlite3ext.h 4b66e3e3435da4b4c8c83696d0349f0c503b3924 F src/sqliteInt.h e21cc96bc24161df7373f6b24367cf580496889d F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 @@ -1390,7 +1390,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 46ee3453a31b5e793d5aaad471cec1b1a11ab2cf -R f02e8925b44dc59b1569a7e5ccbd6813 +P 9ab9c8c6d747647f8ade58c2c4812fc69a813368 +R 84edb3e7271e5a0e0aa256262a048c3a U drh -Z 2390f857ae51a8bc2278cbec6c80566a +Z 934ebb8a1ac47019300027e32a36b72c diff --git a/manifest.uuid b/manifest.uuid index 678dba625e..f2eda9b02d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ab9c8c6d747647f8ade58c2c4812fc69a813368 \ No newline at end of file +6f2858f6817ca70c132f0437ac2f0f74deb273d2 \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index d649616b73..1ae87d6b7e 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -408,7 +408,7 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_value_free, sqlite3_result_zeroblob64, sqlite3_bind_zeroblob64, - /* Version 3.8.12 and later */ + /* Version 3.9.0 and later */ sqlite3_value_subtype, sqlite3_result_subtype }; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 75e51f4b01..c3b2890f0b 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5656,9 +5656,9 @@ struct sqlite3_module { ** to included crashing the application). The estimatedRows field should ** therefore only be used if [sqlite3_libversion_number()] returns a ** value greater than or equal to 3008002. Similarly, the idxFlags field -** was added for version 3.8.12. It may therefore only be used if +** was added for version 3.9.0. It may therefore only be used if ** sqlite3_libversion_number() returns a value greater than or equal to -** 3008012. +** 3009000. */ struct sqlite3_index_info { /* Inputs */ @@ -5686,7 +5686,7 @@ struct sqlite3_index_info { double estimatedCost; /* Estimated cost of using this index */ /* Fields below are only available in SQLite 3.8.2 and later */ sqlite3_int64 estimatedRows; /* Estimated number of rows returned */ - /* Fields below are only available in SQLite 3.8.12 and later */ + /* Fields below are only available in SQLite 3.9.0 and later */ int idxFlags; /* Mask of SQLITE_INDEX_SCAN_* flags */ }; diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index 9b9f610e93..017ea308b1 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -272,7 +272,7 @@ struct sqlite3_api_routines { void (*value_free)(sqlite3_value*); int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64); int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64); - /* Version 3.8.12 and later */ + /* Version 3.9.0 and later */ unsigned int (*value_subtype)(sqlite3_value*); void (*result_subtype)(sqlite3_context*,unsigned int); }; @@ -511,7 +511,7 @@ struct sqlite3_api_routines { #define sqlite3_value_free sqlite3_api->value_free #define sqlite3_result_zeroblob64 sqlite3_api->result_zeroblob64 #define sqlite3_bind_zeroblob64 sqlite3_api->bind_zeroblob64 -/* Version 3.8.12 and later */ +/* Version 3.9.0 and later */ #define sqlite3_value_subtype sqlite3_api->value_subtype #define sqlite3_result_subtype sqlite3_api->result_subtype #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ From df705d8185df5691474314a3f07ec9cf18d3770d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 Oct 2015 19:12:29 +0000 Subject: [PATCH 5/7] Further tweaks to improve fts5 prefix query performance. FossilOrigin-Name: 1c20c1c28b56411f106cf2f6961b3ad4b4d6f6c8 --- ext/fts5/fts5Int.h | 4 +- ext/fts5/fts5_buffer.c | 10 +- ext/fts5/fts5_expr.c | 107 +++-------------- ext/fts5/fts5_index.c | 217 ++++++++++++++++++++++++---------- ext/fts5/fts5_main.c | 2 +- ext/fts5/fts5_vocab.c | 2 +- ext/fts5/test/fts5simple.test | 18 +++ manifest | 26 ++-- manifest.uuid | 2 +- 9 files changed, 207 insertions(+), 181 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index ce066f2aa8..3671da1f12 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -254,7 +254,6 @@ int sqlite3Fts5Get32(const u8*); typedef struct Fts5PoslistReader Fts5PoslistReader; struct Fts5PoslistReader { /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */ - int iCol; /* If (iCol>=0), this column only */ const u8 *a; /* Position list to iterate through */ int n; /* Size of buffer at a[] in bytes */ int i; /* Current offset in a[] */ @@ -266,7 +265,6 @@ struct Fts5PoslistReader { i64 iPos; /* (iCol<<32) + iPos */ }; int sqlite3Fts5PoslistReaderInit( - int iCol, /* If (iCol>=0), this column only */ const u8 *a, int n, /* Poslist buffer to iterate through */ Fts5PoslistReader *pIter /* Iterator object to initialize */ ); @@ -347,7 +345,7 @@ int sqlite3Fts5IterEof(Fts5IndexIter*); int sqlite3Fts5IterNext(Fts5IndexIter*); int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch); i64 sqlite3Fts5IterRowid(Fts5IndexIter*); -int sqlite3Fts5IterPoslist(Fts5IndexIter*, const u8 **pp, int *pn, i64 *pi); +int sqlite3Fts5IterPoslist(Fts5IndexIter*,Fts5Colset*, const u8**, int*, i64*); int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf); /* diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index e7e8d6d663..68f24bf18b 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -203,26 +203,20 @@ int sqlite3Fts5PoslistNext64( ** if the iterator reaches EOF, or false otherwise. */ int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader *pIter){ - if( sqlite3Fts5PoslistNext64(pIter->a, pIter->n, &pIter->i, &pIter->iPos) - || (pIter->iCol>=0 && (pIter->iPos >> 32) > pIter->iCol) - ){ + if( sqlite3Fts5PoslistNext64(pIter->a, pIter->n, &pIter->i, &pIter->iPos) ){ pIter->bEof = 1; } return pIter->bEof; } int sqlite3Fts5PoslistReaderInit( - int iCol, /* If (iCol>=0), this column only */ const u8 *a, int n, /* Poslist buffer to iterate through */ Fts5PoslistReader *pIter /* Iterator object to initialize */ ){ memset(pIter, 0, sizeof(*pIter)); pIter->a = a; pIter->n = n; - pIter->iCol = iCol; - do { - sqlite3Fts5PoslistReaderNext(pIter); - }while( pIter->bEof==0 && (pIter->iPos >> 32)bEof; } diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 02351478a0..a8a8363a4e 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -309,6 +309,7 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ */ static int fts5ExprSynonymPoslist( Fts5ExprTerm *pTerm, + Fts5Colset *pColset, i64 iRowid, int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */ u8 **pa, int *pn @@ -327,7 +328,7 @@ static int fts5ExprSynonymPoslist( const u8 *a; int n; i64 dummy; - rc = sqlite3Fts5IterPoslist(pIter, &a, &n, &dummy); + rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy); if( rc!=SQLITE_OK ) goto synonym_poslist_out; if( nIter==nAlloc ){ int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; @@ -341,7 +342,7 @@ static int fts5ExprSynonymPoslist( if( aIter!=aStatic ) sqlite3_free(aIter); aIter = aNew; } - sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter]); + sqlite3Fts5PoslistReaderInit(a, n, &aIter[nIter]); assert( aIter[nIter].bEof==0 ); nIter++; } @@ -409,13 +410,7 @@ static int fts5ExprPhraseIsMatch( Fts5PoslistReader *aIter = aStatic; int i; int rc = SQLITE_OK; - int iCol = -1; - if( pColset && pColset->nCol==1 ){ - iCol = pColset->aiCol[0]; - pColset = 0; - } - fts5BufferZero(&pPhrase->poslist); /* If the aStatic[] array is not large enough, allocate a large array @@ -435,12 +430,14 @@ static int fts5ExprPhraseIsMatch( int bFlag = 0; const u8 *a = 0; if( pTerm->pSynonym ){ - rc = fts5ExprSynonymPoslist(pTerm, pNode->iRowid, &bFlag, (u8**)&a, &n); + rc = fts5ExprSynonymPoslist( + pTerm, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n + ); }else{ - rc = sqlite3Fts5IterPoslist(pTerm->pIter, &a, &n, &dummy); + rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy); } if( rc!=SQLITE_OK ) goto ismatch_out; - sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]); + sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); aIter[i].bFlag = bFlag; if( aIter[i].bEof ) goto ismatch_out; } @@ -463,11 +460,9 @@ static int fts5ExprPhraseIsMatch( } }while( bMatch==0 ); - if( pColset==0 || fts5ExprColsetTest(pColset, FTS5_POS2COLUMN(iPos)) ){ - /* Append position iPos to the output */ - rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos); - if( rc!=SQLITE_OK ) goto ismatch_out; - } + /* Append position iPos to the output */ + rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos); + if( rc!=SQLITE_OK ) goto ismatch_out; for(i=0; inTerm; i++){ if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out; @@ -762,61 +757,6 @@ static int fts5ExprSynonymAdvanceto( return bEof; } -/* -** IN/OUT parameter (*pa) points to a position list n bytes in size. If -** the position list contains entries for column iCol, then (*pa) is set -** to point to the sub-position-list for that column and the number of -** bytes in it returned. Or, if the argument position list does not -** contain any entries for column iCol, return 0. -*/ -static int fts5ExprExtractCol( - const u8 **pa, /* IN/OUT: Pointer to poslist */ - int n, /* IN: Size of poslist in bytes */ - int iCol /* Column to extract from poslist */ -){ - int iCurrent = 0; - const u8 *p = *pa; - const u8 *pEnd = &p[n]; /* One byte past end of position list */ - u8 prev = 0; - - while( iCol!=iCurrent ){ - /* Advance pointer p until it points to pEnd or an 0x01 byte that is - ** not part of a varint */ - while( (prev & 0x80) || *p!=0x01 ){ - prev = *p++; - if( p==pEnd ) return 0; - } - *pa = p++; - p += fts5GetVarint32(p, iCurrent); - } - - /* Advance pointer p until it points to pEnd or an 0x01 byte that is - ** not part of a varint */ - assert( (prev & 0x80)==0 ); - while( pnCol; i++){ - const u8 *pSub = pPos; - int nSub = fts5ExprExtractCol(&pSub, nPos, pColset->aiCol[i]); - if( nSub ){ - fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); - } - } - return rc; -} static int fts5ExprNearTest( int *pRc, @@ -864,34 +804,15 @@ static int fts5ExprTokenTest( Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; Fts5Colset *pColset = pNear->pColset; - const u8 *pPos; - int nPos; int rc; assert( pNode->eType==FTS5_TERM ); assert( pNear->nPhrase==1 && pPhrase->nTerm==1 ); assert( pPhrase->aTerm[0].pSynonym==0 ); - rc = sqlite3Fts5IterPoslist(pIter, &pPos, &nPos, &pNode->iRowid); - - /* If the term may match any column, then this must be a match. - ** Return immediately in this case. Otherwise, try to find the - ** part of the poslist that corresponds to the required column. - ** If it can be found, return. If it cannot, the next iteration - ** of the loop will test the next rowid in the database for this - ** term. */ - if( pColset==0 ){ - assert( pPhrase->poslist.nSpace==0 ); - pPhrase->poslist.p = (u8*)pPos; - pPhrase->poslist.n = nPos; - }else if( pColset->nCol==1 ){ - assert( pPhrase->poslist.nSpace==0 ); - pPhrase->poslist.n = fts5ExprExtractCol(&pPos, nPos, pColset->aiCol[0]); - pPhrase->poslist.p = (u8*)pPos; - }else if( rc==SQLITE_OK ){ - rc = fts5ExprExtractColset(pColset, pPos, nPos, &pPhrase->poslist); - } - + rc = sqlite3Fts5IterPoslist(pIter, pColset, + (const u8**)&pPhrase->poslist.p, &pPhrase->poslist.n, &pNode->iRowid + ); pNode->bNomatch = (pPhrase->poslist.n==0); return rc; } diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index ec9ed3b921..6c731432fe 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -511,8 +511,9 @@ struct Fts5IndexIter { int nSeg; /* Size of aSeg[] array */ int bRev; /* True to iterate in reverse order */ - int bSkipEmpty; /* True to skip deleted entries */ - int bEof; /* True at EOF */ + u8 bSkipEmpty; /* True to skip deleted entries */ + u8 bEof; /* True at EOF */ + u8 bFiltered; /* True if column-filter already applied */ i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ Fts5CResult *aFirst; /* Current merge state (see above) */ @@ -1457,7 +1458,8 @@ static void fts5SegIterNextPage( */ static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){ int nSz; - int n = fts5GetVarint32(p, nSz); + int n = 0; + fts5FastGetVarint32(p, n, nSz); assert_nc( nSz>=0 ); *pnSz = nSz/2; *pbDel = nSz & 0x0001; @@ -1478,13 +1480,12 @@ static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ if( p->rc==SQLITE_OK ){ int iOff = pIter->iLeafOffset; /* Offset to read at */ + int nSz; ASSERT_SZLEAF_OK(pIter->pLeaf); - if( iOff>=pIter->pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; - }else{ - const u8 *a = &pIter->pLeaf->p[iOff]; - pIter->iLeafOffset += fts5GetPoslistSize(a, &pIter->nPos, &pIter->bDel); - } + fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); + pIter->bDel = (nSz & 0x0001); + pIter->nPos = nSz>>1; + pIter->iLeafOffset = iOff; } } @@ -2725,6 +2726,7 @@ static void fts5MultiIterNew2( if( pNew ){ Fts5SegIter *pIter = &pNew->aSeg[1]; + pNew->bFiltered = 1; pIter->flags = FTS5_SEGITER_ONETERM; if( pData->szLeaf>0 ){ pIter->pLeaf = pData; @@ -3940,7 +3942,7 @@ static void fts5PoslistCallback( ){ assert_nc( nChunk>=0 ); if( nChunk>0 ){ - fts5BufferAppendBlob(&p->rc, (Fts5Buffer*)pContext, nChunk, pChunk); + fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk); } } @@ -3980,7 +3982,7 @@ static void fts5PoslistFilterCallback( fts5FastGetVarint32(pChunk, i, iCol); if( fts5IndexColsetTest(pCtx->pColset, iCol) ){ pCtx->eState = 1; - fts5BufferAppendVarint(&p->rc, pCtx->pBuf, 1); + fts5BufferSafeAppendVarint(pCtx->pBuf, 1); }else{ pCtx->eState = 0; } @@ -3992,7 +3994,7 @@ static void fts5PoslistFilterCallback( i++; } if( pCtx->eState ){ - fts5BufferAppendBlob(&p->rc, pCtx->pBuf, i-iStart, &pChunk[iStart]); + fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); } if( ieState = fts5IndexColsetTest(pCtx->pColset, iCol); if( pCtx->eState ){ - fts5BufferAppendBlob(&p->rc, pCtx->pBuf, i-iStart, &pChunk[iStart]); + fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); iStart = i; } } @@ -4025,56 +4027,123 @@ static void fts5SegiterPoslist( Fts5Colset *pColset, Fts5Buffer *pBuf ){ - if( pColset==0 ){ - fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); - }else{ - PoslistCallbackCtx sCtx; - sCtx.pBuf = pBuf; - sCtx.pColset = pColset; - sCtx.eState = pColset ? fts5IndexColsetTest(pColset, 0) : 1; - assert( sCtx.eState==0 || sCtx.eState==1 ); - fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); + if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){ + if( pColset==0 ){ + fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); + }else{ + PoslistCallbackCtx sCtx; + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + sCtx.eState = pColset ? fts5IndexColsetTest(pColset, 0) : 1; + assert( sCtx.eState==0 || sCtx.eState==1 ); + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); + } } } /* -** Iterator pMulti currently points to a valid entry (not EOF). This -** function appends a copy of the position-list of the entry pMulti -** currently points to to buffer pBuf. -** -** If an error occurs, an error code is left in p->rc. It is assumed -** no error has already occurred when this function is called. +** IN/OUT parameter (*pa) points to a position list n bytes in size. If +** the position list contains entries for column iCol, then (*pa) is set +** to point to the sub-position-list for that column and the number of +** bytes in it returned. Or, if the argument position list does not +** contain any entries for column iCol, return 0. */ -static int fts5MultiIterPoslist( +static int fts5IndexExtractCol( + const u8 **pa, /* IN/OUT: Pointer to poslist */ + int n, /* IN: Size of poslist in bytes */ + int iCol /* Column to extract from poslist */ +){ + int iCurrent = 0; /* Anything before the first 0x01 is col 0 */ + const u8 *p = *pa; + const u8 *pEnd = &p[n]; /* One byte past end of position list */ + u8 prev = 0; + + while( iCol!=iCurrent ){ + /* Advance pointer p until it points to pEnd or an 0x01 byte that is + ** not part of a varint */ + while( (prev & 0x80) || *p!=0x01 ){ + prev = *p++; + if( p==pEnd ) return 0; + } + *pa = p++; + p += fts5GetVarint32(p, iCurrent); + } + + /* Advance pointer p until it points to pEnd or an 0x01 byte that is + ** not part of a varint */ + assert( (prev & 0x80)==0 ); + while( prc. +*/ +static int fts5AppendPoslist( Fts5Index *p, + i64 iDelta, Fts5IndexIter *pMulti, Fts5Colset *pColset, Fts5Buffer *pBuf ){ if( p->rc==SQLITE_OK ){ - int iSz; - int iData; - Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ]; assert( fts5MultiIterEof(p, pMulti)==0 ); + assert( pSeg->nPos>0 ); + if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){ + int iSv1; + int iSv2; + int iData; - /* WRITEPOSLISTSIZE */ - iSz = pBuf->n; - fts5BufferSafeAppendVarint(pBuf, pSeg->nPos*2); - iData = pBuf->n; + /* Append iDelta */ + iSv1 = pBuf->n; + fts5BufferSafeAppendVarint(pBuf, iDelta); - fts5SegiterPoslist(p, pSeg, pColset, pBuf); + /* WRITEPOSLISTSIZE */ + iSv2 = pBuf->n; + fts5BufferSafeAppendVarint(pBuf, pSeg->nPos*2); + iData = pBuf->n; - if( pColset ){ - int nActual = pBuf->n - iData; - if( nActual!=pSeg->nPos ){ - /* WRITEPOSLISTSIZE */ - if( nActual==0 ){ - return 1; + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf + && (pColset==0 || pColset->nCol==1) + ){ + const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + int nPos; + if( pColset ){ + nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]); }else{ - int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2)); - while( iSz<(iData-nReq) ){ pBuf->p[iSz++] = 0x80; } - sqlite3Fts5PutVarint(&pBuf->p[iSz], nActual*2); + nPos = pSeg->nPos; + } + fts5BufferSafeAppendBlob(pBuf, pPos, nPos); + }else{ + fts5SegiterPoslist(p, pSeg, pColset, pBuf); + } + + if( pColset ){ + int nActual = pBuf->n - iData; + if( nActual!=pSeg->nPos ){ + if( nActual==0 ){ + pBuf->n = iSv1; + return 1; + }else{ + int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2)); + while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; } + sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2); + } } } } @@ -4282,15 +4351,8 @@ static void fts5SetupPrefixIter( iLastRowid = 0; } - if( 0==sqlite3Fts5BufferGrow(&p->rc, &doclist, 9) ){ - int iSave = doclist.n; - assert( doclist.n!=0 || iLastRowid==0 ); - fts5BufferSafeAppendVarint(&doclist, iRowid - iLastRowid); - if( fts5MultiIterPoslist(p, p1, pColset, &doclist) ){ - doclist.n = iSave; - }else{ - iLastRowid = iRowid; - } + if( !fts5AppendPoslist(p, iRowid-iLastRowid, p1, pColset, &doclist) ){ + iLastRowid = iRowid; } } @@ -4648,6 +4710,26 @@ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIter, int *pn){ } +static int fts5IndexExtractColset ( + Fts5Colset *pColset, /* Colset to filter on */ + const u8 *pPos, int nPos, /* Position list */ + Fts5Buffer *pBuf /* Output buffer */ +){ + int rc = SQLITE_OK; + int i; + + fts5BufferZero(pBuf); + for(i=0; inCol; i++){ + const u8 *pSub = pPos; + int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); + if( nSub ){ + fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); + } + } + return rc; +} + + /* ** Return a pointer to a buffer containing a copy of the position list for ** the current entry. Output variable *pn is set to the size of the buffer @@ -4658,6 +4740,7 @@ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIter, int *pn){ */ int sqlite3Fts5IterPoslist( Fts5IndexIter *pIter, + Fts5Colset *pColset, /* Column filter (or NULL) */ const u8 **pp, /* OUT: Pointer to position-list data */ int *pn, /* OUT: Size of position-list in bytes */ i64 *piRowid /* OUT: Current rowid */ @@ -4665,13 +4748,25 @@ int sqlite3Fts5IterPoslist( Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; assert( pIter->pIndex->rc==SQLITE_OK ); *piRowid = pSeg->iRowid; - *pn = pSeg->nPos; - if( pSeg->iLeafOffset+pSeg->nPos <= pSeg->pLeaf->szLeaf ){ - *pp = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + if( pColset==0 || pIter->bFiltered ){ + *pn = pSeg->nPos; + *pp = pPos; + }else if( pColset->nCol==1 ){ + *pp = pPos; + *pn = fts5IndexExtractCol(pp, pSeg->nPos, pColset->aiCol[0]); + }else{ + fts5BufferZero(&pIter->poslist); + fts5IndexExtractColset(pColset, pPos, pSeg->nPos, &pIter->poslist); + *pp = pIter->poslist.p; + *pn = pIter->poslist.n; + } }else{ fts5BufferZero(&pIter->poslist); - fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); *pp = pIter->poslist.p; + *pn = pIter->poslist.n; } return fts5IndexReturn(pIter->pIndex); } @@ -4868,10 +4963,10 @@ static int fts5QueryCksum( const u8 *pPos; int nPos; i64 rowid = sqlite3Fts5IterRowid(pIdxIter); - rc = sqlite3Fts5IterPoslist(pIdxIter, &pPos, &nPos, &dummy); + rc = sqlite3Fts5IterPoslist(pIdxIter, 0, &pPos, &nPos, &dummy); if( rc==SQLITE_OK ){ Fts5PoslistReader sReader; - for(sqlite3Fts5PoslistReaderInit(-1, pPos, nPos, &sReader); + for(sqlite3Fts5PoslistReaderInit(pPos, nPos, &sReader); sReader.bEof==0; sqlite3Fts5PoslistReaderNext(&sReader) ){ diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index c2a37171ec..eba8a6cbce 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1643,7 +1643,7 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){ for(i=0; ipIter, &pPos, &nPos, &dummy); + rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy); if( rc==SQLITE_OK ){ if( pTab->eType==FTS5_VOCAB_ROW ){ while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test index 67fc494435..9175e420a2 100644 --- a/ext/fts5/test/fts5simple.test +++ b/ext/fts5/test/fts5simple.test @@ -251,6 +251,10 @@ do_execsql_test 10.1 { SELECT rowid FROM t3('c: c*'); } {2} +do_execsql_test 10.2 { + SELECT rowid FROM t3('b: acb'); +} {2} + #------------------------------------------------------------------------- # Test that character 0x1A is allowed in fts5 barewords. # @@ -281,6 +285,20 @@ do_test 11.5 { catchsql "SELECT rowid FROM t4('d\x19')" } {/fts5: syntax error/} +#------------------------------------------------------------------------- +# +do_test 12.1 { + execsql { + CREATE VIRTUAL TABLE xx USING fts5(x,y); + BEGIN; + INSERT INTO xx VALUES('1 2 3', 'a b c'); + } +} {} + +do_execsql_test 12.2 { + SELECT rowid FROM xx('x:a'); + COMMIT; +} {} finish_test diff --git a/manifest b/manifest index 3e2bd9208d..534a14bd88 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sall\sreferences\sto\s3.8.12\sinto\s3.9.0.\s\sComment\schanges\sonly\s-\sno\nchanges\sto\scode. -D 2015-10-12T04:56:12.349 +C Further\stweaks\sto\simprove\sfts5\sprefix\squery\sperformance. +D 2015-10-12T19:12:29.091 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f0088ff0d2ac949fce6de7c00f13a99ac5bdb663 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -106,21 +106,21 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 98f802fe41481f9d797fce496f0fefcad72c7782 -F ext/fts5/fts5Int.h ed6c05b803e0bacf85228a8d255853e89796f6f5 +F ext/fts5/fts5Int.h 38667e39859ff3f3bc91f47efe672023a145a118 F ext/fts5/fts5_aux.c b09aa27dcdaa3d50a30be433fddaa48a50aa827b -F ext/fts5/fts5_buffer.c b2fb69c1ee3378956c0d9ee964d61b59d296afaf +F ext/fts5/fts5_buffer.c e99224a316cc5b2c574ccccdc7f2344bca54784d F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695 -F ext/fts5/fts5_expr.c 1df899afed24c9c6195eea1780dcc56fcd1d1139 +F ext/fts5/fts5_expr.c 17a945210cbc0cd29f03a87fd30ab7bf994ed16c F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 -F ext/fts5/fts5_index.c e03217c37f344f79673be385de6b03f732291000 -F ext/fts5/fts5_main.c 36fa4fe8b80ba5d596fa6afb910d195f148fd9d2 +F ext/fts5/fts5_index.c f73968357818455039ecb79dcd4b082c3baaeaeb +F ext/fts5/fts5_main.c bf43550b8e9a68514abd179500f1917a2256cd7a F ext/fts5/fts5_storage.c df061a5caf9e50fbbd43113009b5b248362f4995 F ext/fts5/fts5_tcl.c 3bf445e66de32137d4693694ff7b1fd6074e32bd F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf F ext/fts5/fts5_tokenize.c f380f46f341af9c9a9908e1aade685ba1eaa157a F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 -F ext/fts5/fts5_vocab.c 17320c476a5296ee475ab616d95fd10515bacfec +F ext/fts5/fts5_vocab.c a05027ab6abb692ad27654c85137a4f1061a159e F ext/fts5/fts5parse.y e83dca6028e3309178d05b5bd920e372dc295d35 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl 51f7ef3af444b89c6f6ce3896a0ac349ff4e996d @@ -175,7 +175,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821 -F ext/fts5/test/fts5simple.test 85bbb268e01d2e3527d70a7fa511ddc3bba2ccc0 +F ext/fts5/test/fts5simple.test f8463172dc2d4bf9f74c78e9df9c83e942c63a94 F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 @@ -1390,7 +1390,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9ab9c8c6d747647f8ade58c2c4812fc69a813368 -R 84edb3e7271e5a0e0aa256262a048c3a -U drh -Z 934ebb8a1ac47019300027e32a36b72c +P 6f2858f6817ca70c132f0437ac2f0f74deb273d2 +R 648c7563956f80fd03a6301b24b597df +U dan +Z 3852831c6b0a26fcaca66c9a24d96d73 diff --git a/manifest.uuid b/manifest.uuid index f2eda9b02d..976558fcea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6f2858f6817ca70c132f0437ac2f0f74deb273d2 \ No newline at end of file +1c20c1c28b56411f106cf2f6961b3ad4b4d6f6c8 \ No newline at end of file From f2c26ed17b3e4f09a84d8574185b34fcaf1fe322 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 12 Oct 2015 22:20:29 +0000 Subject: [PATCH 6/7] Fix a couple harmless compiler warnings. FossilOrigin-Name: 7f896a971c5953d5370215ecd834d1fb711b4263 --- ext/misc/json1.c | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/fuzzcheck.c | 1 - 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index c968011eea..d5f019ff98 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -885,7 +885,7 @@ static int jsonParseFindParents(JsonParse *pParse){ ** Compare the OBJECT label at pNode against zKey,nKey. Return true on ** a match. */ -static int jsonLabelCompare(JsonNode *pNode, const char *zKey, int nKey){ +static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){ if( pNode->jnFlags & JNODE_RAW ){ if( pNode->n!=nKey ) return 0; return strncmp(pNode->u.zJContent, zKey, nKey)==0; diff --git a/manifest b/manifest index 534a14bd88..fc00ef7610 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\stweaks\sto\simprove\sfts5\sprefix\squery\sperformance. -D 2015-10-12T19:12:29.091 +C Fix\sa\scouple\sharmless\scompiler\swarnings. +D 2015-10-12T22:20:29.991 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f0088ff0d2ac949fce6de7c00f13a99ac5bdb663 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -198,7 +198,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c e827cb3148e7db283e4b6ca36a0c16480bf8e743 +F ext/misc/json1.c d31022098b0c4c7c2c1781cb328180a4da6fbdb3 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -753,7 +753,7 @@ F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1 F test/fuzz3.test 53fabcd5f0f430f8b221282f6c12c4d0903c21eb F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 -F test/fuzzcheck.c b8eb7ee40f6d28548a0b028e0676293522f3427f +F test/fuzzcheck.c 73d7c49980c1a380f1dff52df9c6d67c854d92ea F test/fuzzdata1.db 7ee3227bad0e7ccdeb08a9e6822916777073c664 F test/fuzzdata2.db f03a420d3b822cc82e4f894ca957618fbe9c4973 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1390,7 +1390,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6f2858f6817ca70c132f0437ac2f0f74deb273d2 -R 648c7563956f80fd03a6301b24b597df -U dan -Z 3852831c6b0a26fcaca66c9a24d96d73 +P 1c20c1c28b56411f106cf2f6961b3ad4b4d6f6c8 +R afbc852dad884213e6cdc034cff88444 +U mistachkin +Z 86c808002d54e8263a5056aea0515a65 diff --git a/manifest.uuid b/manifest.uuid index 976558fcea..1021eaf1e9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c20c1c28b56411f106cf2f6961b3ad4b4d6f6c8 \ No newline at end of file +7f896a971c5953d5370215ecd834d1fb711b4263 \ No newline at end of file diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index 4597891c3a..b1ba573d60 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -1051,7 +1051,6 @@ int main(int argc, char **argv){ /* Print the description, if there is one */ if( !quietFlag ){ - int i; zDbName = azSrcDb[iSrcDb]; i = strlen(zDbName) - 1; while( i>0 && zDbName[i-1]!='/' && zDbName[i-1]!='\\' ){ i--; } From cec7dec006cf656fa2f415ce2c73d5206d8eab46 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 12 Oct 2015 22:31:02 +0000 Subject: [PATCH 7/7] Remove the unused fts5ExprColsetTest() routine. FossilOrigin-Name: 9ecafc0c94102dca6da192f400278399464086fb --- ext/fts5/fts5_expr.c | 9 --------- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index a8a8363a4e..8800808a32 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -271,14 +271,6 @@ void sqlite3Fts5ExprFree(Fts5Expr *p){ } } -static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){ - int i; - for(i=0; inCol; i++){ - if( pColset->aiCol[i]==iCol ) return 1; - } - return 0; -} - /* ** Argument pTerm must be a synonym iterator. Return the current rowid ** that it points to. @@ -2242,4 +2234,3 @@ int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){ } return nRet; } - diff --git a/manifest b/manifest index fc00ef7610..2b8b0a9573 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\sharmless\scompiler\swarnings. -D 2015-10-12T22:20:29.991 +C Remove\sthe\sunused\sfts5ExprColsetTest()\sroutine. +D 2015-10-12T22:31:02.804 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f0088ff0d2ac949fce6de7c00f13a99ac5bdb663 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h 38667e39859ff3f3bc91f47efe672023a145a118 F ext/fts5/fts5_aux.c b09aa27dcdaa3d50a30be433fddaa48a50aa827b F ext/fts5/fts5_buffer.c e99224a316cc5b2c574ccccdc7f2344bca54784d F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695 -F ext/fts5/fts5_expr.c 17a945210cbc0cd29f03a87fd30ab7bf994ed16c +F ext/fts5/fts5_expr.c bc31478fd04de55150031f6e6a652939d3e335ac F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c f73968357818455039ecb79dcd4b082c3baaeaeb F ext/fts5/fts5_main.c bf43550b8e9a68514abd179500f1917a2256cd7a @@ -1390,7 +1390,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1c20c1c28b56411f106cf2f6961b3ad4b4d6f6c8 -R afbc852dad884213e6cdc034cff88444 -U mistachkin -Z 86c808002d54e8263a5056aea0515a65 +P 7f896a971c5953d5370215ecd834d1fb711b4263 +R 9caea6f3d0d85e53ac44439278fad807 +U drh +Z 910ebb884723e98753b60b18fc83ff4b diff --git a/manifest.uuid b/manifest.uuid index 1021eaf1e9..9f4eab788a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7f896a971c5953d5370215ecd834d1fb711b4263 \ No newline at end of file +9ecafc0c94102dca6da192f400278399464086fb \ No newline at end of file