From c3489bbf77cf206fdb7d6ffbffe2e3ce6b680c13 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Feb 2016 16:04:59 +0000 Subject: [PATCH 1/3] Take the LIMIT clause into account when estimating the cost of sorting. FossilOrigin-Name: d491745cab951e0de70f1f79b7640ea8aff6e8bb --- manifest | 23 +++++++++++++---------- manifest.uuid | 2 +- src/select.c | 35 +++++++++++++++++++++-------------- src/sqliteInt.h | 42 ++++++++++++++++++++++-------------------- src/treeview.c | 5 +++-- src/where.c | 44 ++++++++++++++++++++++++++++++++------------ src/whereInt.h | 1 + 7 files changed, 93 insertions(+), 59 deletions(-) diff --git a/manifest b/manifest index ed34efd918..d1fca82070 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\scommand-line\sshell:\s\sWhen\sthe\s".import"\scommand\sis\screating\sa\snew\ntable\susing\scolumn\snames\sfrom\sthe\sfirst\srow\sof\sCSV\sinput,\smake\ssure\s\ndouble-quotes\sin\sthe\sname\sare\sproperly\sescaped. -D 2016-02-25T13:33:02.701 +C Take\sthe\sLIMIT\sclause\sinto\saccount\swhen\sestimating\sthe\scost\sof\ssorting. +D 2016-02-25T16:04:59.636 F Makefile.in 4e90dc1521879022aa9479268a4cd141d1771142 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 28fc4ee02333996d31b3602b39eeb8e609a89ce4 @@ -349,12 +349,12 @@ F src/printf.c 63e6fb12bbe702dd664dc3703776c090383a5a26 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c b8f7174e5f8c33c44ded3a25a973d0bb89228c20 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e -F src/select.c 1a7e23a3bb2edb9cdc46ab0cf7c1500109cf2531 +F src/select.c dbc73a63bee99bf7cfac6052477ffc21e899bb08 F src/shell.c 89b73e894e737cc2f21e4bce0feb3ea21cc61124 F src/sqlite.h.in a7caa805a345d6a14136c11ccfa6f4c1a6ef6e7b F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d -F src/sqliteInt.h 5200682fc1050f00fba07128c0facc37462c4411 +F src/sqliteInt.h 5a87bccd05996f4c3facd8f831cf92a05b818b6f F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 @@ -408,7 +408,7 @@ F src/test_windirent.h b12055cab6227f7be10f5c19296f67c60cc5e2a5 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 3d338cdd00d916ce8a05c397001d64ed58e6fe1c -F src/treeview.c dc39ccf04e9331237388b9cb73289c9d87ea050b +F src/treeview.c c525282442111b3f61eb176784567cd6654db5dc F src/trigger.c e14840ee0c3e549e758ec9bf3e4146e166002280 F src/update.c a7eeeaffad59c6506f01303a071dac11de8269ca F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c @@ -428,8 +428,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 10deb6b43887662691e5f53d10b3c171c401169b F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354 -F src/where.c 5b67fb8035ae4697cf721db095f800ef8dff5f56 -F src/whereInt.h 78b6b4de94db84aecbdc07fe3e38f648eb391e9a +F src/where.c ba8f2cd97e5125d2423b3dffad9f77b888b9b1a8 +F src/whereInt.h 93297d56edd137b7ea004490690fb6e2ce028a34 F src/wherecode.c 39c1ef4598bedf1d66249334c74efd23ddd182ac F src/whereexpr.c fb87944b1254234e5bba671aaf6dee476241506a F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1429,7 +1429,10 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh a98af506df552f3b3c0d904f94e4cdc4e1a6d598 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e721975faa0925be4029330550ff2a9666041ff7 -R c8d6b90bca6c11dea1a586d1afdb957c +P 2e67a1c823c7003e7d2619c7d8b6db6ce046c527 +R 537d9f6aec4a86824de8d982730a8e8f +T *branch * planner-improvements +T *sym-planner-improvements * +T -sym-trunk * U drh -Z 8dad8d5c86c3c0b275d53a26a0713023 +Z 54aed3c7f721cc8ad2041b056394372f diff --git a/manifest.uuid b/manifest.uuid index 6f30a6f925..c9fb7a8620 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2e67a1c823c7003e7d2619c7d8b6db6ce046c527 \ No newline at end of file +d491745cab951e0de70f1f79b7640ea8aff6e8bb \ No newline at end of file diff --git a/src/select.c b/src/select.c index ff4d9d85c1..bde278b43c 100644 --- a/src/select.c +++ b/src/select.c @@ -105,7 +105,7 @@ Select *sqlite3SelectNew( ExprList *pGroupBy, /* the GROUP BY clause */ Expr *pHaving, /* the HAVING clause */ ExprList *pOrderBy, /* the ORDER BY clause */ - u16 selFlags, /* Flag parameters, such as SF_Distinct */ + u32 selFlags, /* Flag parameters, such as SF_Distinct */ Expr *pLimit, /* LIMIT value. NULL means not used */ Expr *pOffset /* OFFSET value. NULL means no offset */ ){ @@ -1845,8 +1845,9 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ VdbeComment((v, "LIMIT counter")); if( n==0 ){ sqlite3VdbeGoto(v, iBreak); - }else if( n>=0 && p->nSelectRow>(u64)n ){ - p->nSelectRow = n; + }else if( n>=0 && p->nSelectRow>sqlite3LogEst((u64)n) ){ + p->nSelectRow = sqlite3LogEst((u64)n); + p->selFlags |= SF_FixedLimit; } }else{ sqlite3ExprCode(pParse, p->pLimit, iLimit); @@ -2287,12 +2288,12 @@ static int multiSelect( testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; p->pPrior = pPrior; - p->nSelectRow += pPrior->nSelectRow; + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); if( pPrior->pLimit && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit) - && nLimit>0 && p->nSelectRow > (u64)nLimit + && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) ){ - p->nSelectRow = nLimit; + p->nSelectRow = sqlite3LogEst((u64)nLimit); } if( addr ){ sqlite3VdbeJumpHere(v, addr); @@ -2364,7 +2365,9 @@ static int multiSelect( pDelete = p->pPrior; p->pPrior = pPrior; p->pOrderBy = 0; - if( p->op==TK_UNION ) p->nSelectRow += pPrior->nSelectRow; + if( p->op==TK_UNION ){ + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + } sqlite3ExprDelete(db, p->pLimit); p->pLimit = pLimit; p->pOffset = pOffset; @@ -3001,7 +3004,7 @@ static int multiSelectOrderBy( addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); VdbeCoverage(v); sqlite3VdbeGoto(v, addrEofA); - p->nSelectRow += pPrior->nSelectRow; + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); } /* Generate a subroutine to run when the results from select B @@ -4987,7 +4990,7 @@ int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); + pItem->pTab->nRowLogEst = pSub->nSelectRow; pItem->fg.viaCoroutine = 1; pItem->regResult = dest.iSdst; sqlite3VdbeEndCoroutine(v, pItem->regReturn); @@ -5018,7 +5021,7 @@ int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); + pItem->pTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); VdbeComment((v, "end %s", pItem->pTab->zName)); @@ -5101,7 +5104,7 @@ int sqlite3Select( /* Set the limiter. */ iEnd = sqlite3VdbeMakeLabel(v); - p->nSelectRow = LARGEST_INT64; + p->nSelectRow = 320; /* 4 billion rows */ computeLimitRegisters(pParse, p, iEnd); if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen); @@ -5125,10 +5128,12 @@ int sqlite3Select( if( !isAgg && pGroupBy==0 ){ /* No aggregate functions and no GROUP BY clause */ u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); + assert( WHERE_USE_LIMIT==SF_FixedLimit ); + wctrlFlags |= p->selFlags & SF_FixedLimit; /* Begin the database scan. */ pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, - p->pEList, wctrlFlags, 0); + p->pEList, wctrlFlags, p->nSelectRow); if( pWInfo==0 ) goto select_end; if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); @@ -5188,9 +5193,11 @@ int sqlite3Select( for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){ pItem->u.x.iAlias = 0; } - if( p->nSelectRow>100 ) p->nSelectRow = 100; + assert( 66==sqlite3LogEst(100) ); + if( p->nSelectRow>66 ) p->nSelectRow = 66; }else{ - p->nSelectRow = 1; + assert( 0==sqlite3LogEst(1) ); + p->nSelectRow = 0; } /* If there is both a GROUP BY and an ORDER BY clause and they are diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 2ea5f57910..7924ed9dc8 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2442,6 +2442,7 @@ struct SrcList { #define WHERE_SORTBYGROUP 0x0800 /* Support sqlite3WhereIsSorted() */ #define WHERE_REOPEN_IDX 0x1000 /* Try to use OP_ReopenIdx */ #define WHERE_ONEPASS_MULTIROW 0x2000 /* ONEPASS is ok with multiple rows */ +#define WHERE_USE_LIMIT 0x4000 /* There is a constant LIMIT clause */ /* Allowed return values from sqlite3WhereIsDistinct() */ @@ -2520,13 +2521,13 @@ struct NameContext { struct Select { ExprList *pEList; /* The fields of the result */ u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ - u16 selFlags; /* Various SF_* values */ + LogEst nSelectRow; /* Estimated number of result rows */ + u32 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ #if SELECTTRACE_ENABLED char zSelName[12]; /* Symbolic name of this SELECT use for debugging */ #endif int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ - u64 nSelectRow; /* Estimated number of result rows */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ ExprList *pGroupBy; /* The GROUP BY clause */ @@ -2543,22 +2544,23 @@ struct Select { ** Allowed values for Select.selFlags. The "SF" prefix stands for ** "Select Flag". */ -#define SF_Distinct 0x0001 /* Output should be DISTINCT */ -#define SF_All 0x0002 /* Includes the ALL keyword */ -#define SF_Resolved 0x0004 /* Identifiers have been resolved */ -#define SF_Aggregate 0x0008 /* Contains aggregate functions */ -#define SF_UsesEphemeral 0x0010 /* Uses the OpenEphemeral opcode */ -#define SF_Expanded 0x0020 /* sqlite3SelectExpand() called on this */ -#define SF_HasTypeInfo 0x0040 /* FROM subqueries have Table metadata */ -#define SF_Compound 0x0080 /* Part of a compound query */ -#define SF_Values 0x0100 /* Synthesized from VALUES clause */ -#define SF_MultiValue 0x0200 /* Single VALUES term with multiple rows */ -#define SF_NestedFrom 0x0400 /* Part of a parenthesized FROM clause */ -#define SF_MaybeConvert 0x0800 /* Need convertCompoundSelectToSubquery() */ -#define SF_MinMaxAgg 0x1000 /* Aggregate containing min() or max() */ -#define SF_Recursive 0x2000 /* The recursive part of a recursive CTE */ -#define SF_Converted 0x4000 /* By convertCompoundSelectToSubquery() */ -#define SF_IncludeHidden 0x8000 /* Include hidden columns in output */ +#define SF_Distinct 0x00001 /* Output should be DISTINCT */ +#define SF_All 0x00002 /* Includes the ALL keyword */ +#define SF_Resolved 0x00004 /* Identifiers have been resolved */ +#define SF_Aggregate 0x00008 /* Contains aggregate functions */ +#define SF_UsesEphemeral 0x00010 /* Uses the OpenEphemeral opcode */ +#define SF_Expanded 0x00020 /* sqlite3SelectExpand() called on this */ +#define SF_HasTypeInfo 0x00040 /* FROM subqueries have Table metadata */ +#define SF_Compound 0x00080 /* Part of a compound query */ +#define SF_Values 0x00100 /* Synthesized from VALUES clause */ +#define SF_MultiValue 0x00200 /* Single VALUES term with multiple rows */ +#define SF_NestedFrom 0x00400 /* Part of a parenthesized FROM clause */ +#define SF_MaybeConvert 0x00800 /* Need convertCompoundSelectToSubquery() */ +#define SF_MinMaxAgg 0x01000 /* Aggregate containing min() or max() */ +#define SF_Recursive 0x02000 /* The recursive part of a recursive CTE */ +#define SF_FixedLimit 0x04000 /* nSelectRow set by a constant LIMIT */ +#define SF_Converted 0x08000 /* By convertCompoundSelectToSubquery() */ +#define SF_IncludeHidden 0x10000 /* Include hidden columns in output */ /* @@ -3489,7 +3491,7 @@ Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*, void sqlite3DropIndex(Parse*, SrcList*, int); int sqlite3Select(Parse*, Select*, SelectDest*); Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, - Expr*,ExprList*,u16,Expr*,Expr*); + Expr*,ExprList*,u32,Expr*,Expr*); void sqlite3SelectDelete(sqlite3*, Select*); Table *sqlite3SrcListLookup(Parse*, SrcList*); int sqlite3IsReadOnly(Parse*, Table*, int); @@ -3501,7 +3503,7 @@ void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int); void sqlite3WhereEnd(WhereInfo*); -u64 sqlite3WhereOutputRowCount(WhereInfo*); +LogEst sqlite3WhereOutputRowCount(WhereInfo*); int sqlite3WhereIsDistinct(WhereInfo*); int sqlite3WhereIsOrdered(WhereInfo*); int sqlite3WhereIsSorted(WhereInfo*); diff --git a/src/treeview.c b/src/treeview.c index 2985804314..ff3b4be5aa 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -132,9 +132,10 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ sqlite3TreeViewPush(pView, 1); } do{ - sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x", + sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x nSelectRow=%d", ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), - ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p, p->selFlags + ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p, p->selFlags, + (int)p->nSelectRow ); if( cnt++ ) sqlite3TreeViewPop(pView); if( p->pPrior ){ diff --git a/src/where.c b/src/where.c index ca9e4a6f9d..addd50674d 100644 --- a/src/where.c +++ b/src/where.c @@ -31,8 +31,8 @@ static int whereLoopResize(sqlite3*, WhereLoop*, int); /* ** Return the estimated number of output rows from a WHERE clause */ -u64 sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ - return sqlite3LogEstToInt(pWInfo->nRowOut); +LogEst sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ + return pWInfo->nRowOut; } /* @@ -3438,6 +3438,7 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ ** order. */ static LogEst whereSortingCost( + WhereInfo *pWInfo, LogEst nRow, int nOrderBy, int nSorted @@ -3458,7 +3459,15 @@ static LogEst whereSortingCost( LogEst rScale, rSortCost; assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; - rSortCost = nRow + estLog(nRow) + rScale + 16; + rSortCost = nRow + rScale + 16; + + /* Multiple by log(M) where M is the number of output rows. + ** Use the LIMIT for M if it is smaller */ + if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 ){ + LogEst m = sqlite3LogEst(pWInfo->iLimit); + if( m=0 && isOrdereddb; memset(&sWLB, 0, sizeof(sWLB)); @@ -4065,6 +4079,7 @@ WhereInfo *sqlite3WhereBegin( pWInfo->pResultSet = pResultSet; pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); pWInfo->wctrlFlags = wctrlFlags; + pWInfo->iLimit = iAuxArg; pWInfo->savedNQueryLoop = pParse->nQueryLoop; assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */ pMaskSet = &pWInfo->sMaskSet; @@ -4145,9 +4160,14 @@ WhereInfo *sqlite3WhereBegin( } /* Construct the WhereLoop objects */ - WHERETRACE(0xffff,("*** Optimizer Start *** (wctrlFlags: 0x%x)\n", - wctrlFlags)); #if defined(WHERETRACE_ENABLED) + if( sqlite3WhereTrace & 0xffff ){ + sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags); + if( wctrlFlags & WHERE_USE_LIMIT ){ + sqlite3DebugPrintf(", limit: %d", iAuxArg); + } + sqlite3DebugPrintf(")\n"); + } if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ int i; for(i=0; inTerm; i++){ @@ -4330,8 +4350,8 @@ WhereInfo *sqlite3WhereBegin( Index *pIx = pLoop->u.btree.pIndex; int iIndexCur; int op = OP_OpenRead; - /* iIdxCur is always set if to a positive value if ONEPASS is possible */ - assert( iIdxCur!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 ); + /* iAuxArg is always set if to a positive value if ONEPASS is possible */ + assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 ); if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx) && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){ @@ -4341,7 +4361,7 @@ WhereInfo *sqlite3WhereBegin( op = 0; }else if( pWInfo->eOnePass!=ONEPASS_OFF ){ Index *pJ = pTabItem->pTab->pIndex; - iIndexCur = iIdxCur; + iIndexCur = iAuxArg; assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); while( ALWAYS(pJ) && pJ!=pIx ){ iIndexCur++; @@ -4349,8 +4369,8 @@ WhereInfo *sqlite3WhereBegin( } op = OP_OpenWrite; pWInfo->aiCurOnePass[1] = iIndexCur; - }else if( iIdxCur && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){ - iIndexCur = iIdxCur; + }else if( iAuxArg && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){ + iIndexCur = iAuxArg; if( wctrlFlags & WHERE_REOPEN_IDX ) op = OP_ReopenIdx; }else{ iIndexCur = pParse->nTab++; diff --git a/src/whereInt.h b/src/whereInt.h index 1a189980ef..eb6ca326f5 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -412,6 +412,7 @@ struct WhereInfo { WhereLoop *pLoops; /* List of all WhereLoop objects */ Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ LogEst nRowOut; /* Estimated number of output rows */ + LogEst iLimit; /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */ u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ u8 sorted; /* True if really sorted (not just grouped) */ From 87eb919d05ba5487e3df2db6d0431d7b7b9c28a2 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Feb 2016 18:03:38 +0000 Subject: [PATCH 2/3] Do not use an automatic index on an outer loop that only runs once. FossilOrigin-Name: 5957e793414ff80ed01a7a67e70c3fd096a3f6e0 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/where.c | 8 ++++++++ test/autoindex2.test | 7 ++++++- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index d1fca82070..d9b4a87ec1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Take\sthe\sLIMIT\sclause\sinto\saccount\swhen\sestimating\sthe\scost\sof\ssorting. -D 2016-02-25T16:04:59.636 +C Do\snot\suse\san\sautomatic\sindex\son\san\souter\sloop\sthat\sonly\sruns\sonce. +D 2016-02-25T18:03:38.680 F Makefile.in 4e90dc1521879022aa9479268a4cd141d1771142 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 28fc4ee02333996d31b3602b39eeb8e609a89ce4 @@ -428,7 +428,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 10deb6b43887662691e5f53d10b3c171c401169b F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354 -F src/where.c ba8f2cd97e5125d2423b3dffad9f77b888b9b1a8 +F src/where.c 07bf0330592acd2610d131aabe175cbada3e9710 F src/whereInt.h 93297d56edd137b7ea004490690fb6e2ce028a34 F src/wherecode.c 39c1ef4598bedf1d66249334c74efd23ddd182ac F src/whereexpr.c fb87944b1254234e5bba671aaf6dee476241506a @@ -475,7 +475,7 @@ F test/auth2.test 264c6af53cad9aba5218c68bbe18036e39007bfa F test/auth3.test 5cfa94ed90c6617c42b7ba4b133fd79678b251c7 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 F test/autoindex1.test 14b63a9f1e405fe6d5bfc8c8d00249c2ebaf13ea -F test/autoindex2.test af7e595c6864cc6ef5fc38d5db579a3e34940cb8 +F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df F test/autoindex3.test a3be0d1a53a7d2edff208a5e442312957047e972 F test/autoindex4.test 49d3cd791a9baa16fb461d7ea3de80d019a819cf F test/autoindex5.test 96f084a5e6024ea07cace5888df3223f3ea86990 @@ -1429,10 +1429,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh a98af506df552f3b3c0d904f94e4cdc4e1a6d598 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2e67a1c823c7003e7d2619c7d8b6db6ce046c527 -R 537d9f6aec4a86824de8d982730a8e8f -T *branch * planner-improvements -T *sym-planner-improvements * -T -sym-trunk * +P d491745cab951e0de70f1f79b7640ea8aff6e8bb +R fb78c9128e17cc39ff6d4163eb8b91e3 U drh -Z 54aed3c7f721cc8ad2041b056394372f +Z b2631495aa69546c0305d417243ff503 diff --git a/manifest.uuid b/manifest.uuid index c9fb7a8620..b4a5bbf637 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d491745cab951e0de70f1f79b7640ea8aff6e8bb \ No newline at end of file +5957e793414ff80ed01a7a67e70c3fd096a3f6e0 \ No newline at end of file diff --git a/src/where.c b/src/where.c index addd50674d..3dc3c87a09 100644 --- a/src/where.c +++ b/src/where.c @@ -3585,6 +3585,14 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; + if( iLoop==0 + && (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 + && pParse->nQueryLoop==0 + ){ + /* Never put an automatic index in the outer loop if the query + ** is only being run once. */ + continue; + } /* At this point, pWLoop is a candidate to be the next loop. ** Compute its cost */ rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); diff --git a/test/autoindex2.test b/test/autoindex2.test index c8abdf410a..afd4a666b7 100644 --- a/test/autoindex2.test +++ b/test/autoindex2.test @@ -218,10 +218,15 @@ do_execsql_test autoindex2-120 { AND t1.did = t2.did AND t2.uid = t3.uid ORDER BY t1.ptime desc LIMIT 500; -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1x1 (ptime>?)} 0 1 1 {SEARCH TABLE t2 USING INDEX t2x0 (did=?)} 0 2 2 {SEARCH TABLE t3 USING INDEX t3x0 (uid=?)}} +} {~/AUTO/} # # ^^^--- Before being fixed, the above was using an automatic covering # on t3 and reordering the tables so that t3 was in the outer loop and # implementing the ORDER BY clause using a B-Tree. +# +# This test is sanitized data received from a user. The original unsanitized +# data and STAT4 data is found in the th3private test repository. See one of +# the th3private check-ins on 2016-02-25. The test is much more accurate when +# STAT4 data is used. finish_test From 5a6f5ede8175ff253b85925e2728ea76f8c86c9f Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Feb 2016 18:22:09 +0000 Subject: [PATCH 3/3] Improvement on the previous check-in: disallow automatic indexes for any loop that is expected to run less than twice. FossilOrigin-Name: aab53a21894ba51d325fd8f8f4bb4163ece74391 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 10 ++++------ 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index d9b4a87ec1..cb1fddd357 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\san\sautomatic\sindex\son\san\souter\sloop\sthat\sonly\sruns\sonce. -D 2016-02-25T18:03:38.680 +C Improvement\son\sthe\sprevious\scheck-in:\s\sdisallow\sautomatic\sindexes\sfor\sany\nloop\sthat\sis\sexpected\sto\srun\sless\sthan\stwice. +D 2016-02-25T18:22:09.962 F Makefile.in 4e90dc1521879022aa9479268a4cd141d1771142 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 28fc4ee02333996d31b3602b39eeb8e609a89ce4 @@ -428,7 +428,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 10deb6b43887662691e5f53d10b3c171c401169b F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354 -F src/where.c 07bf0330592acd2610d131aabe175cbada3e9710 +F src/where.c b21a2b3cee42e1db4f14c8fc5124f607e0c668c0 F src/whereInt.h 93297d56edd137b7ea004490690fb6e2ce028a34 F src/wherecode.c 39c1ef4598bedf1d66249334c74efd23ddd182ac F src/whereexpr.c fb87944b1254234e5bba671aaf6dee476241506a @@ -1429,7 +1429,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh a98af506df552f3b3c0d904f94e4cdc4e1a6d598 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d491745cab951e0de70f1f79b7640ea8aff6e8bb -R fb78c9128e17cc39ff6d4163eb8b91e3 +P 5957e793414ff80ed01a7a67e70c3fd096a3f6e0 +R 8a8e5a80c02a4dfa694df7d4a40dc042 U drh -Z b2631495aa69546c0305d417243ff503 +Z 5c10952be751c4b6279994eaa94bd734 diff --git a/manifest.uuid b/manifest.uuid index b4a5bbf637..27e8814012 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5957e793414ff80ed01a7a67e70c3fd096a3f6e0 \ No newline at end of file +aab53a21894ba51d325fd8f8f4bb4163ece74391 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 3dc3c87a09..a68dcf5bfc 100644 --- a/src/where.c +++ b/src/where.c @@ -3585,12 +3585,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; - if( iLoop==0 - && (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 - && pParse->nQueryLoop==0 - ){ - /* Never put an automatic index in the outer loop if the query - ** is only being run once. */ + if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<10 ){ + /* Do not use an automatic index if the this loop is expected + ** to run less than 2 times. */ + assert( 10==sqlite3LogEst(2) ); continue; } /* At this point, pWLoop is a candidate to be the next loop.