From 56f1b99d4dd96241b1ba7101bbc1cfacd80c1eec Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Sep 2012 14:29:39 +0000 Subject: [PATCH 1/9] Pass information around between the major routines of the query planner using a single pointer to a structure rather than a long list of parameters. FossilOrigin-Name: 1104d42e104d561ce60d05d158acfe499ee9fd50 --- manifest | 15 +- manifest.uuid | 2 +- src/where.c | 372 +++++++++++++++++++++++++------------------------- 3 files changed, 195 insertions(+), 194 deletions(-) diff --git a/manifest b/manifest index 2e6bf42149..9b64595abd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunused\ssubfunction\sparameter\sand\san\sobsolete\scomment\sfrom\sthe\nquery\splanner\slogic\sin\swhere.c. -D 2012-09-24T19:50:00.842 +C Pass\sinformation\saround\sbetween\sthe\smajor\sroutines\sof\sthe\squery\splanner\nusing\sa\ssingle\spointer\sto\sa\sstructure\srather\sthan\sa\slong\slist\sof\sparameters. +D 2012-09-25T14:29:39.134 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c 5acb3e7bbd31f10ba39acad9ce6b399055337a9d F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 40708330a0e9bf79c0ab97109b8014fa04cce858 +F src/where.c 82be1d2f8f27012de0ed9d0977753ed24312b791 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1016,7 +1016,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 22989f3588531efd555cc29d6c576e7a34b7edc4 -R af274415395e47bec73279dd70f3872e +P 349a55cd8ba9ce65ebfd987ecfebd1204f7d0a85 +R b0986bef8aa73322a60d8a15dd219873 +T *branch * qp-enhancements +T *sym-qp-enhancements * +T -sym-trunk * U drh -Z 2c59ca6ec247c019d5fe7043b4d83355 +Z 6cfa0793222da70bc8fbd646a62c8a2a diff --git a/manifest.uuid b/manifest.uuid index 2d6a191e78..4dfae10d21 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -349a55cd8ba9ce65ebfd987ecfebd1204f7d0a85 \ No newline at end of file +1104d42e104d561ce60d05d158acfe499ee9fd50 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 85fb1ed4f0..66fe0516a6 100644 --- a/src/where.c +++ b/src/where.c @@ -267,6 +267,26 @@ struct WhereCost { #define WHERE_DISTINCT 0x40000000 /* Correct order for DISTINCT */ #define WHERE_COVER_SCAN 0x80000000 /* Full scan of a covering index */ +/* +** This module contains many separate subroutines that work together to +** find the best indices to use for accessing a particular table in a query. +** An instance of the following structure holds context information about the +** index search so that it can be more easily passed between the various +** routines. +*/ +typedef struct WhereBestIdx WhereBestIdx; +struct WhereBestIdx { + Parse *pParse; /* Parser context */ + WhereClause *pWC; /* The WHERE clause */ + struct SrcList_item *pSrc; /* The FROM clause term to search */ + Bitmask notReady; /* Mask of cursors not available */ + Bitmask notValid; /* Cursors not available for any purpose */ + ExprList *pOrderBy; /* The ORDER BY clause */ + ExprList *pDistinct; /* The select-list if query is DISTINCT */ + sqlite3_index_info **ppIdxInfo; /* Index information passed to xBestIndex */ + WhereCost cost; /* Lowest cost query plan */ +}; + /* ** Initialize a preallocated WhereClause structure. */ @@ -1811,9 +1831,7 @@ static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ /* ** Required because bestIndex() is called by bestOrClauseIndex() */ -static void bestIndex( - Parse*, WhereClause*, struct SrcList_item*, - Bitmask, Bitmask, WhereCost*); +static void bestIndex(WhereBestIdx*); /* ** This routine attempts to find an scanning strategy that can be used @@ -1822,20 +1840,14 @@ static void bestIndex( ** The table associated with FROM clause term pSrc may be either a ** regular B-Tree table or a virtual table. */ -static void bestOrClauseIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - WhereCost *pCost /* Lowest cost query plan */ -){ +static void bestOrClauseIndex(WhereBestIdx *p){ #ifndef SQLITE_OMIT_OR_OPTIMIZATION - const int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ + WhereClause *pWC = p->pWC; /* The WHERE clause */ + struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */ + const int iCur = pSrc->iCursor; /* The cursor of the table */ const Bitmask maskSrc = getMask(pWC->pMaskSet, iCur); /* Bitmask for pSrc */ WhereTerm * const pWCEnd = &pWC->a[pWC->nTerm]; /* End of pWC->a[] */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ /* The OR-clause optimization is disallowed if the INDEXED BY or ** NOT INDEXED clauses are used or if the WHERE_AND_ONLY bit is set. */ @@ -1849,7 +1861,7 @@ static void bestOrClauseIndex( /* Search the WHERE clause terms for a usable WO_OR term. */ for(pTerm=pWC->a; pTermeOperator==WO_OR - && ((pTerm->prereqAll & ~maskSrc) & notReady)==0 + && ((pTerm->prereqAll & ~maskSrc) & p->notReady)==0 && (pTerm->u.pOrInfo->indexable & maskSrc)!=0 ){ WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; @@ -1859,15 +1871,19 @@ static void bestOrClauseIndex( double rTotal = 0; double nRow = 0; Bitmask used = 0; + WhereBestIdx sBOI; + sBOI = *p; + sBOI.pOrderBy = 0; + sBOI.pDistinct = 0; + sBOI.ppIdxInfo = 0; for(pOrTerm=pOrWC->a; pOrTerma), (pTerm - pWC->a) )); if( pOrTerm->eOperator==WO_AND ){ - WhereClause *pAndWC = &pOrTerm->u.pAndInfo->wc; - bestIndex(pParse, pAndWC, pSrc, notReady, notValid, &sTermCost); + sBOI.pWC = &pOrTerm->u.pAndInfo->wc; + bestIndex(&sBOI); }else if( pOrTerm->leftCursor==iCur ){ WhereClause tempWC; tempWC.pParse = pWC->pParse; @@ -1877,19 +1893,20 @@ static void bestOrClauseIndex( tempWC.a = pOrTerm; tempWC.wctrlFlags = 0; tempWC.nTerm = 1; - bestIndex(pParse, &tempWC, pSrc, notReady, notValid, &sTermCost); + sBOI.pWC = &tempWC; + bestIndex(&sBOI); }else{ continue; } - rTotal += sTermCost.rCost; - nRow += sTermCost.plan.nRow; - used |= sTermCost.used; - if( rTotal>=pCost->rCost ) break; + rTotal += sBOI.cost.rCost; + nRow += sBOI.cost.plan.nRow; + used |= sBOI.cost.used; + if( rTotal>=p->cost.rCost ) break; } /* If there is an ORDER BY clause, increase the scan cost to account ** for the cost of the sort. */ - if( pOrderBy!=0 ){ + if( p->pOrderBy!=0 ){ WHERETRACE(("... sorting increases OR cost %.9g to %.9g\n", rTotal, rTotal+nRow*estLog(nRow))); rTotal += nRow*estLog(nRow); @@ -1899,12 +1916,12 @@ static void bestOrClauseIndex( ** less than the current cost stored in pCost, replace the contents ** of pCost. */ WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow)); - if( rTotalrCost ){ - pCost->rCost = rTotal; - pCost->used = used; - pCost->plan.nRow = nRow; - pCost->plan.wsFlags = flags; - pCost->plan.u.pTerm = pTerm; + if( rTotalcost.rCost ){ + p->cost.rCost = rTotal; + p->cost.used = used; + p->cost.plan.nRow = nRow; + p->cost.plan.wsFlags = flags; + p->cost.plan.u.pTerm = pTerm; } } } @@ -1941,15 +1958,12 @@ static int termCanDriveIndex( ** is taken into account, then alter the query plan to use the ** transient index. */ -static void bestAutomaticIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ - WhereCost *pCost /* Lowest cost query plan */ -){ - double nTableRow; /* Rows in the input table */ - double logN; /* log(nTableRow) */ +static void bestAutomaticIndex(WhereBestIdx *p){ + Parse *pParse = p->pParse; /* The parsing context */ + WhereClause *pWC = p->pWC; /* The WHERE clause */ + struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */ + double nTableRow; /* Rows in the input table */ + double logN; /* log(nTableRow) */ double costTempIdx; /* per-query cost of the transient index */ WhereTerm *pTerm; /* A single term of the WHERE clause */ WhereTerm *pWCEnd; /* End of pWC->a[] */ @@ -1963,7 +1977,7 @@ static void bestAutomaticIndex( /* Automatic indices are disabled at run-time */ return; } - if( (pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 ){ + if( (p->cost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 ){ /* We already have some kind of index in use for this query. */ return; } @@ -1981,7 +1995,7 @@ static void bestAutomaticIndex( nTableRow = pTable->nRowEst; logN = estLog(nTableRow); costTempIdx = 2*logN*(nTableRow/pParse->nQueryLoop + 1); - if( costTempIdx>=pCost->rCost ){ + if( costTempIdx>=p->cost.rCost ){ /* The cost of creating the transient table would be greater than ** doing the full table scan */ return; @@ -1990,19 +2004,19 @@ static void bestAutomaticIndex( /* Search for any equality comparison term */ pWCEnd = &pWC->a[pWC->nTerm]; for(pTerm=pWC->a; pTermnotReady) ){ WHERETRACE(("auto-index reduces cost from %.1f to %.1f\n", - pCost->rCost, costTempIdx)); - pCost->rCost = costTempIdx; - pCost->plan.nRow = logN + 1; - pCost->plan.wsFlags = WHERE_TEMP_INDEX; - pCost->used = pTerm->prereqRight; + p->cost.rCost, costTempIdx)); + p->cost.rCost = costTempIdx; + p->cost.plan.nRow = logN + 1; + p->cost.plan.wsFlags = WHERE_TEMP_INDEX; + p->cost.used = pTerm->prereqRight; break; } } } #else -# define bestAutomaticIndex(A,B,C,D,E) /* no-op */ +# define bestAutomaticIndex(A) /* no-op */ #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ @@ -2163,12 +2177,11 @@ static void constructAutomaticIndex( ** responsibility of the caller to eventually release the structure ** by passing the pointer returned by this function to sqlite3_free(). */ -static sqlite3_index_info *allocateIndexInfo( - Parse *pParse, - WhereClause *pWC, - struct SrcList_item *pSrc, - ExprList *pOrderBy -){ +static sqlite3_index_info *allocateIndexInfo(WhereBestIdx *p){ + Parse *pParse = p->pParse; + WhereClause *pWC = p->pWC; + struct SrcList_item *pSrc = p->pSrc; + ExprList *pOrderBy = p->pOrderBy; int i, j; int nTerm; struct sqlite3_index_constraint *pIdxCons; @@ -2198,12 +2211,13 @@ static sqlite3_index_info *allocateIndexInfo( */ nOrderBy = 0; if( pOrderBy ){ - for(i=0; inExpr; i++){ + int n = pOrderBy->nExpr; + for(i=0; ia[i].pExpr; if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; } - if( i==pOrderBy->nExpr ){ - nOrderBy = pOrderBy->nExpr; + if( i==n){ + nOrderBy = n; } } @@ -2327,16 +2341,10 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ ** routine takes care of freeing the sqlite3_index_info structure after ** everybody has finished with it. */ -static void bestVirtualIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for index */ - Bitmask notValid, /* Cursors not valid for any purpose */ - ExprList *pOrderBy, /* The order by clause */ - WhereCost *pCost, /* Lowest cost query plan */ - sqlite3_index_info **ppIdxInfo /* Index information passed to xBestIndex */ -){ +static void bestVirtualIndex(WhereBestIdx *p){ + Parse *pParse = p->pParse; /* The parsing context */ + WhereClause *pWC = p->pWC; /* The WHERE clause */ + struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */ Table *pTab = pSrc->pTab; sqlite3_index_info *pIdxInfo; struct sqlite3_index_constraint *pIdxCons; @@ -2350,15 +2358,15 @@ static void bestVirtualIndex( ** malloc in allocateIndexInfo() fails and this function returns leaving ** wsFlags in an uninitialized state, the caller may behave unpredictably. */ - memset(pCost, 0, sizeof(*pCost)); - pCost->plan.wsFlags = WHERE_VIRTUALTABLE; + memset(&p->cost, 0, sizeof(p->cost)); + p->cost.plan.wsFlags = WHERE_VIRTUALTABLE; /* If the sqlite3_index_info structure has not been previously ** allocated and initialized, then allocate and initialize it now. */ - pIdxInfo = *ppIdxInfo; + pIdxInfo = *p->ppIdxInfo; if( pIdxInfo==0 ){ - *ppIdxInfo = pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pOrderBy); + *p->ppIdxInfo = pIdxInfo = allocateIndexInfo(p); } if( pIdxInfo==0 ){ return; @@ -2403,7 +2411,7 @@ static void bestVirtualIndex( for(i=0; inConstraint; i++, pIdxCons++){ j = pIdxCons->iTermOffset; pTerm = &pWC->a[j]; - pIdxCons->usable = (pTerm->prereqRight¬Ready) ? 0 : 1; + pIdxCons->usable = (pTerm->prereqRight&p->notReady) ? 0 : 1; } memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); if( pIdxInfo->needToFreeIdxStr ){ @@ -2416,7 +2424,7 @@ static void bestVirtualIndex( /* ((double)2) In case of SQLITE_OMIT_FLOATING_POINT... */ pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((double)2); nOrderBy = pIdxInfo->nOrderBy; - if( !pOrderBy ){ + if( !p->pOrderBy ){ pIdxInfo->nOrderBy = 0; } @@ -2427,7 +2435,7 @@ static void bestVirtualIndex( pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; for(i=0; inConstraint; i++){ if( pUsage[i].argvIndex>0 ){ - pCost->used |= pWC->a[pIdxCons[i].iTermOffset].prereqRight; + p->cost.used |= pWC->a[pIdxCons[i].iTermOffset].prereqRight; } } @@ -2436,7 +2444,7 @@ static void bestVirtualIndex( ** matches the processing for non-virtual tables in bestBtreeIndex(). */ rCost = pIdxInfo->estimatedCost; - if( pOrderBy && pIdxInfo->orderByConsumed==0 ){ + if( p->pOrderBy && pIdxInfo->orderByConsumed==0 ){ rCost += estLog(rCost)*rCost; } @@ -2448,21 +2456,21 @@ static void bestVirtualIndex( ** is defined. */ if( (SQLITE_BIG_DBL/((double)2))rCost = (SQLITE_BIG_DBL/((double)2)); + p->cost.rCost = (SQLITE_BIG_DBL/((double)2)); }else{ - pCost->rCost = rCost; + p->cost.rCost = rCost; } - pCost->plan.u.pVtabIdx = pIdxInfo; + p->cost.plan.u.pVtabIdx = pIdxInfo; if( pIdxInfo->orderByConsumed ){ - pCost->plan.wsFlags |= WHERE_ORDERBY; + p->cost.plan.wsFlags |= WHERE_ORDERBY; } - pCost->plan.nEq = 0; + p->cost.plan.nEq = 0; pIdxInfo->nOrderBy = nOrderBy; /* Try to find a more efficient access pattern by using multiple indexes ** to optimize an OR expression within the WHERE clause. */ - bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); + bestOrClauseIndex(p); } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -2864,8 +2872,7 @@ static int whereInScanEst( /* ** Find the best query plan for accessing a particular table. Write the -** best query plan and its cost into the WhereCost object supplied as the -** last parameter. +** best query plan and its cost into the p->cost. ** ** The lowest cost plan wins. The cost is an estimate of the amount of ** CPU and disk I/O needed to process the requested result. @@ -2890,16 +2897,10 @@ static int whereInScanEst( ** selected plan may still take advantage of the built-in rowid primary key ** index. */ -static void bestBtreeIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - ExprList *pDistinct, /* The select-list if query is DISTINCT */ - WhereCost *pCost /* Lowest cost query plan */ -){ +static void bestBtreeIndex(WhereBestIdx *p){ + Parse *pParse = p->pParse; /* The parsing context */ + WhereClause *pWC = p->pWC; /* The WHERE clause */ + struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */ int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ Index *pProbe; /* An index we are evaluating */ Index *pIdx; /* Copy of pProbe, or zero for IPK index */ @@ -2908,11 +2909,11 @@ static void bestBtreeIndex( Index sPk; /* A fake index object for the primary key */ tRowcnt aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ int aiColumnPk = -1; /* The aColumn[] value for the sPk index */ - int wsFlagMask; /* Allowed flags in pCost->plan.wsFlag */ + int wsFlagMask; /* Allowed flags in p->cost.plan.wsFlag */ /* Initialize the cost to a worst-case value */ - memset(pCost, 0, sizeof(*pCost)); - pCost->rCost = SQLITE_BIG_DBL; + memset(&p->cost, 0, sizeof(p->cost)); + p->cost.rCost = SQLITE_BIG_DBL; /* If the pSrc table is the right table of a LEFT JOIN then we may not ** use an index to satisfy IS NULL constraints on that table. This is @@ -3036,8 +3037,8 @@ static void bestBtreeIndex( int nInMul = 1; /* Number of distinct equalities to lookup */ double rangeDiv = (double)1; /* Estimated reduction in search space */ int nBound = 0; /* Number of range constraints seen */ - int bSort = !!pOrderBy; /* True if external sort required */ - int bDist = !!pDistinct; /* True if index cannot help with DISTINCT */ + int bSort = !!p->pOrderBy; /* True if external sort required */ + int bDist = !!p->pDistinct; /* True if index cannot help with DISTINCT */ int bLookup = 0; /* True if not a covering index */ WhereTerm *pTerm; /* A single term of the WHERE clause */ #ifdef SQLITE_ENABLE_STAT3 @@ -3047,7 +3048,7 @@ static void bestBtreeIndex( /* Determine the values of nEq and nInMul */ for(nEq=0; nEqnColumn; nEq++){ int j = pProbe->aiColumn[nEq]; - pTerm = findTerm(pWC, iCur, j, notReady, eqTermMask, pIdx); + pTerm = findTerm(pWC, iCur, j, p->notReady, eqTermMask, pIdx); if( pTerm==0 ) break; wsFlags |= (WHERE_COLUMN_EQ|WHERE_ROWID_EQ); testcase( pTerm->pWC!=pWC ); @@ -3088,9 +3089,10 @@ static void bestBtreeIndex( } }else if( pProbe->bUnordered==0 ){ int j = (nEq==pProbe->nColumn ? -1 : pProbe->aiColumn[nEq]); - if( findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){ - WhereTerm *pTop = findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE, pIdx); - WhereTerm *pBtm = findTerm(pWC, iCur, j, notReady, WO_GT|WO_GE, pIdx); + if( findTerm(pWC, iCur, j, p->notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){ + WhereTerm *pTop, *pBtm; + pTop = findTerm(pWC, iCur, j, p->notReady, WO_LT|WO_LE, pIdx); + pBtm = findTerm(pWC, iCur, j, p->notReady, WO_GT|WO_GE, pIdx); whereRangeScanEst(pParse, pProbe, nEq, pBtm, pTop, &rangeDiv); if( pTop ){ nBound = 1; @@ -3113,7 +3115,7 @@ static void bestBtreeIndex( ** in wsFlags. Otherwise, if there is an ORDER BY clause but the index ** will scan rows in a different order, set the bSort variable. */ if( isSortingIndex( - pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, nEq, wsFlags, &rev) + pParse, pWC->pMaskSet, pProbe, iCur, p->pOrderBy, nEq, wsFlags, &rev) ){ bSort = 0; wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; @@ -3123,7 +3125,7 @@ static void bestBtreeIndex( /* If there is a DISTINCT qualifier and this index will scan rows in ** order of the DISTINCT expressions, clear bDist and set the appropriate ** flags in wsFlags. */ - if( isDistinctIndex(pParse, pWC, pProbe, iCur, pDistinct, nEq) + if( isDistinctIndex(pParse, pWC, pProbe, iCur, p->pDistinct, nEq) && (wsFlags & WHERE_COLUMN_IN)==0 ){ bDist = 0; @@ -3283,7 +3285,7 @@ static void bestBtreeIndex( ** might be selected even when there exists an optimal index that has ** no such dependency. */ - if( nRow>2 && cost<=pCost->rCost ){ + if( nRow>2 && cost<=p->cost.rCost ){ int k; /* Loop counter */ int nSkipEq = nEq; /* Number of == constraints to skip */ int nSkipRange = nBound; /* Number of < constraints to skip */ @@ -3292,7 +3294,7 @@ static void bestBtreeIndex( thisTab = getMask(pWC->pMaskSet, iCur); for(pTerm=pWC->a, k=pWC->nTerm; nRow>2 && k; k--, pTerm++){ if( pTerm->wtFlags & TERM_VIRTUAL ) continue; - if( (pTerm->prereqAll & notValid)!=thisTab ) continue; + if( (pTerm->prereqAll & p->notValid)!=thisTab ) continue; if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){ if( nSkipEq ){ /* Ignore the first nEq equality matches since the index @@ -3331,21 +3333,21 @@ static void bestBtreeIndex( " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f used=0x%llx\n", pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk"), nEq, nInMul, (int)rangeDiv, bSort, bLookup, wsFlags, - notReady, log10N, nRow, cost, used + p->notReady, log10N, nRow, cost, used )); /* If this index is the best we have seen so far, then record this ** index and its cost in the pCost structure. */ if( (!pIdx || wsFlags) - && (costrCost || (cost<=pCost->rCost && nRowplan.nRow)) + && (costcost.rCost || (cost<=p->cost.rCost && nRowcost.plan.nRow)) ){ - pCost->rCost = cost; - pCost->used = used; - pCost->plan.nRow = nRow; - pCost->plan.wsFlags = (wsFlags&wsFlagMask); - pCost->plan.nEq = nEq; - pCost->plan.u.pIdx = pIdx; + p->cost.rCost = cost; + p->cost.used = used; + p->cost.plan.nRow = nRow; + p->cost.plan.wsFlags = (wsFlags&wsFlagMask); + p->cost.plan.nEq = nEq; + p->cost.plan.u.pIdx = pIdx; } /* If there was an INDEXED BY clause, then only that one index is @@ -3362,25 +3364,25 @@ static void bestBtreeIndex( ** in. This is used for application testing, to help find cases ** where application behaviour depends on the (undefined) order that ** SQLite outputs rows in in the absence of an ORDER BY clause. */ - if( !pOrderBy && pParse->db->flags & SQLITE_ReverseOrder ){ - pCost->plan.wsFlags |= WHERE_REVERSE; + if( !p->pOrderBy && pParse->db->flags & SQLITE_ReverseOrder ){ + p->cost.plan.wsFlags |= WHERE_REVERSE; } - assert( pOrderBy || (pCost->plan.wsFlags&WHERE_ORDERBY)==0 ); - assert( pCost->plan.u.pIdx==0 || (pCost->plan.wsFlags&WHERE_ROWID_EQ)==0 ); + assert( p->pOrderBy || (p->cost.plan.wsFlags&WHERE_ORDERBY)==0 ); + assert( p->cost.plan.u.pIdx==0 || (p->cost.plan.wsFlags&WHERE_ROWID_EQ)==0 ); assert( pSrc->pIndex==0 - || pCost->plan.u.pIdx==0 - || pCost->plan.u.pIdx==pSrc->pIndex + || p->cost.plan.u.pIdx==0 + || p->cost.plan.u.pIdx==pSrc->pIndex ); WHERETRACE(("best index is: %s\n", - ((pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ? "none" : - pCost->plan.u.pIdx ? pCost->plan.u.pIdx->zName : "ipk") + ((p->cost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ? "none" : + p->cost.plan.u.pIdx ? p->cost.plan.u.pIdx->zName : "ipk") )); - bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); - bestAutomaticIndex(pParse, pWC, pSrc, notReady, pCost); - pCost->plan.wsFlags |= eqTermMask; + bestOrClauseIndex(p); + bestAutomaticIndex(p); + p->cost.plan.wsFlags |= eqTermMask; } /* @@ -3395,26 +3397,20 @@ static void bestBtreeIndex( ** details will be reconsidered later if the optimization is found to be ** applicable. */ -static void bestIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - WhereCost *pCost /* Lowest cost query plan */ -){ +static void bestIndex(WhereBestIdx *p){ #ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pSrc->pTab) ){ - sqlite3_index_info *p = 0; - bestVirtualIndex(pParse, pWC, pSrc, notReady, notValid, 0, pCost, &p); - if( p->needToFreeIdxStr ){ - sqlite3_free(p->idxStr); + if( IsVirtual(p->pSrc->pTab) ){ + sqlite3_index_info *pIdxInfo = 0; + p->ppIdxInfo = &pIdxInfo; + bestVirtualIndex(p); + if( pIdxInfo->needToFreeIdxStr ){ + sqlite3_free(pIdxInfo->idxStr); } - sqlite3DbFree(pParse->db, p); + sqlite3DbFree(p->pParse->db, pIdxInfo); }else #endif { - bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, 0, 0, pCost); + bestBtreeIndex(p); } } @@ -4700,14 +4696,18 @@ WhereInfo *sqlite3WhereBegin( WhereInfo *pWInfo; /* Will become the return value of this function */ Vdbe *v = pParse->pVdbe; /* The virtual database engine */ Bitmask notReady; /* Cursors that are not yet positioned */ + WhereBestIdx sWBI; /* Best index search context */ WhereMaskSet *pMaskSet; /* The expression mask set */ - WhereClause *pWC; /* Decomposition of the WHERE clause */ - struct SrcList_item *pTabItem; /* A single entry from pTabList */ - WhereLevel *pLevel; /* A single level in pWInfo->a[] */ - int iFrom; /* First unused FROM clause element */ + WhereLevel *pLevel; /* A single level in pWInfo->a[] */ + int iFrom; /* First unused FROM clause element */ int andFlags; /* AND-ed combination of all pWC->a[].wtFlags */ sqlite3 *db; /* Database connection */ + + /* Variable initialization */ + memset(&sWBI, 0, sizeof(sWBI)); + sWBI.pParse = pParse; + /* The number of tables in the FROM clause is limited by the number of ** bits in a Bitmask */ @@ -4747,10 +4747,10 @@ WhereInfo *sqlite3WhereBegin( pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; pWInfo->iBreak = sqlite3VdbeMakeLabel(v); - pWInfo->pWC = pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo]; + pWInfo->pWC = sWBI.pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo]; pWInfo->wctrlFlags = wctrlFlags; pWInfo->savedNQueryLoop = pParse->nQueryLoop; - pMaskSet = (WhereMaskSet*)&pWC[1]; + pMaskSet = (WhereMaskSet*)&sWBI.pWC[1]; /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ @@ -4760,9 +4760,9 @@ WhereInfo *sqlite3WhereBegin( ** subexpression is separated by an AND operator. */ initMaskSet(pMaskSet); - whereClauseInit(pWC, pParse, pMaskSet, wctrlFlags); + whereClauseInit(sWBI.pWC, pParse, pMaskSet, wctrlFlags); sqlite3ExprCodeConstants(pParse, pWhere); - whereSplit(pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ + whereSplit(sWBI.pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ /* Special case: a WHERE clause that is constant. Evaluate the ** expression and either jump over all of the code or fall thru. @@ -4793,12 +4793,12 @@ WhereInfo *sqlite3WhereBegin( ** equal to pTabList->nSrc but might be shortened to 1 if the ** WHERE_ONETABLE_ONLY flag is set. */ - assert( pWC->vmask==0 && pMaskSet->n==0 ); + assert( sWBI.pWC->vmask==0 && pMaskSet->n==0 ); for(i=0; inSrc; i++){ createMask(pMaskSet, pTabList->a[i].iCursor); #ifndef SQLITE_OMIT_VIRTUALTABLE if( ALWAYS(pTabList->a[i].pTab) && IsVirtual(pTabList->a[i].pTab) ){ - pWC->vmask |= ((Bitmask)1 << i); + sWBI.pWC->vmask |= ((Bitmask)1 << i); } #endif } @@ -4818,7 +4818,7 @@ WhereInfo *sqlite3WhereBegin( ** want to analyze these virtual terms, so start analyzing at the end ** and work forward so that the added virtual terms are never processed. */ - exprAnalyzeAll(pTabList, pWC); + exprAnalyzeAll(pTabList, sWBI.pWC); if( db->mallocFailed ){ goto whereBeginError; } @@ -4827,7 +4827,7 @@ WhereInfo *sqlite3WhereBegin( ** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to ** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT. */ - if( pDistinct && isDistinctRedundant(pParse, pTabList, pWC, pDistinct) ){ + if( pDistinct && isDistinctRedundant(pParse, pTabList, sWBI.pWC, pDistinct) ){ pDistinct = 0; pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } @@ -4910,48 +4910,43 @@ WhereInfo *sqlite3WhereBegin( nUnconstrained = 0; notIndexed = 0; for(isOptimal=(iFrom=0 && bestJ<0; isOptimal--){ - Bitmask mask; /* Mask of tables not yet ready */ - for(j=iFrom, pTabItem=&pTabList->a[j]; ja[j]; jjointype & (JT_LEFT|JT_CROSS))!=0; + doNotReorder = (sWBI.pSrc->jointype & (JT_LEFT|JT_CROSS))!=0; if( j!=iFrom && doNotReorder ) break; - m = getMask(pMaskSet, pTabItem->iCursor); + m = getMask(pMaskSet, sWBI.pSrc->iCursor); if( (m & notReady)==0 ){ if( j==iFrom ) iFrom++; continue; } - mask = (isOptimal ? m : notReady); - pOB = (i==0) ? pOrderBy : 0; - pDist = (i==0 ? pDistinct : 0); - if( pTabItem->pIndex==0 ) nUnconstrained++; + sWBI.notValid = notReady; + sWBI.notReady = (isOptimal ? m : notReady); + sWBI.pOrderBy = (i==0) ? pOrderBy : 0; + sWBI.pDistinct = (i==0 ? pDistinct : 0); + if( sWBI.pSrc->pIndex==0 ) nUnconstrained++; WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", j, isOptimal)); - assert( pTabItem->pTab ); + assert( sWBI.pSrc->pTab ); #ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pTabItem->pTab) ){ - sqlite3_index_info **pp = &pWInfo->a[j].pIdxInfo; - bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOB, - &sCost, pp); + if( IsVirtual(sWBI.pSrc->pTab) ){ + sWBI.ppIdxInfo = &pWInfo->a[j].pIdxInfo; + bestVirtualIndex(&sWBI); }else #endif { - bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOB, - pDist, &sCost); + bestBtreeIndex(&sWBI); } - assert( isOptimal || (sCost.used¬Ready)==0 ); + assert( isOptimal || (sWBI.cost.used¬Ready)==0 ); /* If an INDEXED BY clause is present, then the plan must use that ** index if it uses any index at all */ - assert( pTabItem->pIndex==0 - || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 - || sCost.plan.u.pIdx==pTabItem->pIndex ); + assert( sWBI.pSrc->pIndex==0 + || (sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 + || sWBI.cost.plan.u.pIdx==sWBI.pSrc->pIndex ); - if( isOptimal && (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ + if( isOptimal && (sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ notIndexed |= m; } @@ -4975,20 +4970,20 @@ WhereInfo *sqlite3WhereBegin( ** (4) The plan cost must be lower than prior plans or else the ** cost must be the same and the number of rows must be lower. */ - if( (sCost.used¬Ready)==0 /* (1) */ - && (bestJ<0 || (notIndexed&m)!=0 /* (2) */ + if( (sWBI.cost.used¬Ready)==0 /* (1) */ + && (bestJ<0 || (notIndexed&m)!=0 /* (2) */ || (bestPlan.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 - || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0) - && (nUnconstrained==0 || pTabItem->pIndex==0 /* (3) */ - || NEVER((sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) - && (bestJ<0 || sCost.rCostpIndex==0 /* (3) */ + || NEVER((sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) + && (bestJ<0 || sWBI.cost.rCosta; ia[pLevel->iFrom]; pTab = pTabItem->pTab; @@ -5112,7 +5108,7 @@ WhereInfo *sqlite3WhereBegin( } #ifndef SQLITE_OMIT_AUTOMATIC_INDEX if( (pLevel->plan.wsFlags & WHERE_TEMP_INDEX)!=0 ){ - constructAutomaticIndex(pParse, pWC, pTabItem, notReady, pLevel); + constructAutomaticIndex(pParse, sWBI.pWC, pTabItem, notReady, pLevel); }else #endif if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ @@ -5126,7 +5122,7 @@ WhereInfo *sqlite3WhereBegin( VdbeComment((v, "%s", pIx->zName)); } sqlite3CodeVerifySchema(pParse, iDb); - notReady &= ~getMask(pWC->pMaskSet, pTabItem->iCursor); + notReady &= ~getMask(sWBI.pWC->pMaskSet, pTabItem->iCursor); } pWInfo->iTop = sqlite3VdbeCurrentAddr(v); if( db->mallocFailed ) goto whereBeginError; @@ -5154,6 +5150,8 @@ WhereInfo *sqlite3WhereBegin( char *z; int n; int w; + struct SrcList_item *pTabItem; + pLevel = &pWInfo->a[i]; w = pLevel->plan.wsFlags; pTabItem = &pTabList->a[pLevel->iFrom]; From 9cd1c99fe1f786e32623c4614d9b0b0c086ff441 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Sep 2012 20:43:35 +0000 Subject: [PATCH 2/9] Augment the WhereBestIdx structure to pass down into the query planner information that might be used to better detect ORDER BY and DISTINCT optimizations spanning multiple tables of a join. FossilOrigin-Name: 4226e51ff837f0ffe16355491a655d919d13488e --- manifest | 15 ++++------ manifest.uuid | 2 +- src/where.c | 79 ++++++++++++++++++++++++++++++--------------------- 3 files changed, 53 insertions(+), 43 deletions(-) diff --git a/manifest b/manifest index 9b64595abd..28323b288a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Pass\sinformation\saround\sbetween\sthe\smajor\sroutines\sof\sthe\squery\splanner\nusing\sa\ssingle\spointer\sto\sa\sstructure\srather\sthan\sa\slong\slist\sof\sparameters. -D 2012-09-25T14:29:39.134 +C Augment\sthe\sWhereBestIdx\sstructure\sto\spass\sdown\sinto\sthe\squery\splanner\s\ninformation\sthat\smight\sbe\sused\sto\sbetter\sdetect\sORDER\sBY\sand\sDISTINCT\noptimizations\sspanning\smultiple\stables\sof\sa\sjoin. +D 2012-09-25T20:43:35.989 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c 5acb3e7bbd31f10ba39acad9ce6b399055337a9d F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 82be1d2f8f27012de0ed9d0977753ed24312b791 +F src/where.c e75e67f0631182be4ff0d325c5a1e4c4ec3d0962 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1016,10 +1016,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 349a55cd8ba9ce65ebfd987ecfebd1204f7d0a85 -R b0986bef8aa73322a60d8a15dd219873 -T *branch * qp-enhancements -T *sym-qp-enhancements * -T -sym-trunk * +P 1104d42e104d561ce60d05d158acfe499ee9fd50 +R 660ef52f8cc671ae827c548793cfd963 U drh -Z 6cfa0793222da70bc8fbd646a62c8a2a +Z 10b5de991bc34ec524f2e3adc126c5c7 diff --git a/manifest.uuid b/manifest.uuid index 4dfae10d21..1edaad7ba3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1104d42e104d561ce60d05d158acfe499ee9fd50 \ No newline at end of file +4226e51ff837f0ffe16355491a655d919d13488e \ No newline at end of file diff --git a/src/where.c b/src/where.c index 66fe0516a6..63823f277d 100644 --- a/src/where.c +++ b/src/where.c @@ -284,6 +284,8 @@ struct WhereBestIdx { ExprList *pOrderBy; /* The ORDER BY clause */ ExprList *pDistinct; /* The select-list if query is DISTINCT */ sqlite3_index_info **ppIdxInfo; /* Index information passed to xBestIndex */ + int i, n; /* Which loop is being coded; # of loops */ + WhereLevel *a; /* Info about outer loops */ WhereCost cost; /* Lowest cost query plan */ }; @@ -3037,14 +3039,22 @@ static void bestBtreeIndex(WhereBestIdx *p){ int nInMul = 1; /* Number of distinct equalities to lookup */ double rangeDiv = (double)1; /* Estimated reduction in search space */ int nBound = 0; /* Number of range constraints seen */ - int bSort = !!p->pOrderBy; /* True if external sort required */ - int bDist = !!p->pDistinct; /* True if index cannot help with DISTINCT */ + int bSort; /* True if external sort required */ + int bDist; /* True if index cannot help with DISTINCT */ int bLookup = 0; /* True if not a covering index */ WhereTerm *pTerm; /* A single term of the WHERE clause */ #ifdef SQLITE_ENABLE_STAT3 WhereTerm *pFirstTerm = 0; /* First term matching the index */ #endif + if( (p->i) > 0 ){ + bSort = 0; + bDist = 0; + }else{ + bSort = p->pOrderBy!=0; + bDist = p->pDistinct!=0; + } + /* Determine the values of nEq and nInMul */ for(nEq=0; nEqnColumn; nEq++){ int j = pProbe->aiColumn[nEq]; @@ -3114,7 +3124,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** naturally scan rows in the required order, set the appropriate flags ** in wsFlags. Otherwise, if there is an ORDER BY clause but the index ** will scan rows in a different order, set the bSort variable. */ - if( isSortingIndex( + if( bSort && isSortingIndex( pParse, pWC->pMaskSet, pProbe, iCur, p->pOrderBy, nEq, wsFlags, &rev) ){ bSort = 0; @@ -3125,7 +3135,8 @@ static void bestBtreeIndex(WhereBestIdx *p){ /* If there is a DISTINCT qualifier and this index will scan rows in ** order of the DISTINCT expressions, clear bDist and set the appropriate ** flags in wsFlags. */ - if( isDistinctIndex(pParse, pWC, pProbe, iCur, p->pDistinct, nEq) + if( bDist + && isDistinctIndex(pParse, pWC, pProbe, iCur, p->pDistinct, nEq) && (wsFlags & WHERE_COLUMN_IN)==0 ){ bDist = 0; @@ -4690,7 +4701,6 @@ WhereInfo *sqlite3WhereBegin( u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ int iIdxCur /* If WHERE_ONETABLE_ONLY is set, index cursor number */ ){ - int i; /* Loop counter */ int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ int nTabList; /* Number of elements in pTabList */ WhereInfo *pWInfo; /* Will become the return value of this function */ @@ -4701,6 +4711,7 @@ WhereInfo *sqlite3WhereBegin( WhereLevel *pLevel; /* A single level in pWInfo->a[] */ int iFrom; /* First unused FROM clause element */ int andFlags; /* AND-ed combination of all pWC->a[].wtFlags */ + int ii; /* Loop counter */ sqlite3 *db; /* Database connection */ @@ -4751,6 +4762,7 @@ WhereInfo *sqlite3WhereBegin( pWInfo->wctrlFlags = wctrlFlags; pWInfo->savedNQueryLoop = pParse->nQueryLoop; pMaskSet = (WhereMaskSet*)&sWBI.pWC[1]; + sWBI.a = pWInfo->a; /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ @@ -4794,19 +4806,19 @@ WhereInfo *sqlite3WhereBegin( ** WHERE_ONETABLE_ONLY flag is set. */ assert( sWBI.pWC->vmask==0 && pMaskSet->n==0 ); - for(i=0; inSrc; i++){ - createMask(pMaskSet, pTabList->a[i].iCursor); + for(ii=0; iinSrc; ii++){ + createMask(pMaskSet, pTabList->a[ii].iCursor); #ifndef SQLITE_OMIT_VIRTUALTABLE - if( ALWAYS(pTabList->a[i].pTab) && IsVirtual(pTabList->a[i].pTab) ){ - sWBI.pWC->vmask |= ((Bitmask)1 << i); + if( ALWAYS(pTabList->a[ii].pTab) && IsVirtual(pTabList->a[ii].pTab) ){ + sWBI.pWC->vmask |= ((Bitmask)1 << ii); } #endif } #ifndef NDEBUG { Bitmask toTheLeft = 0; - for(i=0; inSrc; i++){ - Bitmask m = getMask(pMaskSet, pTabList->a[i].iCursor); + for(ii=0; iinSrc; ii++){ + Bitmask m = getMask(pMaskSet, pTabList->a[ii].iCursor); assert( (m-1)==toTheLeft ); toTheLeft |= m; } @@ -4847,10 +4859,13 @@ WhereInfo *sqlite3WhereBegin( ** This loop also figures out the nesting order of tables in the FROM ** clause. */ - notReady = ~(Bitmask)0; + sWBI.notValid = ~(Bitmask)0; + sWBI.pOrderBy = pOrderBy; + sWBI.n = nTabList; + sWBI.pDistinct = pDistinct; andFlags = ~0; WHERETRACE(("*** Optimizer Start ***\n")); - for(i=iFrom=0, pLevel=pWInfo->a; ia; sWBI.ijointype & (JT_LEFT|JT_CROSS))!=0; if( j!=iFrom && doNotReorder ) break; m = getMask(pMaskSet, sWBI.pSrc->iCursor); - if( (m & notReady)==0 ){ + if( (m & sWBI.notValid)==0 ){ if( j==iFrom ) iFrom++; continue; } - sWBI.notValid = notReady; - sWBI.notReady = (isOptimal ? m : notReady); - sWBI.pOrderBy = (i==0) ? pOrderBy : 0; - sWBI.pDistinct = (i==0 ? pDistinct : 0); + sWBI.notReady = (isOptimal ? m : sWBI.notValid); if( sWBI.pSrc->pIndex==0 ) nUnconstrained++; WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", @@ -4938,7 +4950,7 @@ WhereInfo *sqlite3WhereBegin( { bestBtreeIndex(&sWBI); } - assert( isOptimal || (sWBI.cost.used¬Ready)==0 ); + assert( isOptimal || (sWBI.cost.used&sWBI.notValid)==0 ); /* If an INDEXED BY clause is present, then the plan must use that ** index if it uses any index at all */ @@ -4953,7 +4965,8 @@ WhereInfo *sqlite3WhereBegin( /* Conditions under which this table becomes the best so far: ** ** (1) The table must not depend on other tables that have not - ** yet run. + ** yet run. (In other words, it must not depend on tables + ** in inner loops.) ** ** (2) A full-table-scan plan cannot supercede indexed plan unless ** the full-table-scan is an "optimal" plan as defined above. @@ -4970,7 +4983,7 @@ WhereInfo *sqlite3WhereBegin( ** (4) The plan cost must be lower than prior plans or else the ** cost must be the same and the number of rows must be lower. */ - if( (sWBI.cost.used¬Ready)==0 /* (1) */ + if( (sWBI.cost.used&sWBI.notValid)==0 /* (1) */ && (bestJ<0 || (notIndexed&m)!=0 /* (2) */ || (bestPlan.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 || (sWBI.cost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0) @@ -4990,7 +5003,7 @@ WhereInfo *sqlite3WhereBegin( } } assert( bestJ>=0 ); - assert( notReady & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); + assert( sWBI.notValid & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); WHERETRACE(("*** Optimizer selects table %d for loop %d" " with cost=%g and nRow=%g\n", bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow)); @@ -5016,7 +5029,7 @@ WhereInfo *sqlite3WhereBegin( }else{ pLevel->iIdxCur = -1; } - notReady &= ~getMask(pMaskSet, pTabList->a[bestJ].iCursor); + sWBI.notValid &= ~getMask(pMaskSet, pTabList->a[bestJ].iCursor); pLevel->iFrom = (u8)bestJ; if( bestPlan.plan.nRow>=(double)1 ){ pParse->nQueryLoop *= bestPlan.plan.nRow; @@ -5069,7 +5082,7 @@ WhereInfo *sqlite3WhereBegin( sqlite3CodeVerifySchema(pParse, -1); /* Insert the cookie verifier Goto */ notReady = ~(Bitmask)0; pWInfo->nRowOut = (double)1; - for(i=0, pLevel=pWInfo->a; ia; iia[i]; - explainOneScan(pParse, pTabList, pLevel, i, pLevel->iFrom, wctrlFlags); - notReady = codeOneLoopStart(pWInfo, i, wctrlFlags, notReady); + for(ii=0; iia[ii]; + explainOneScan(pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags); + notReady = codeOneLoopStart(pWInfo, ii, wctrlFlags, notReady); pWInfo->iContinue = pLevel->addrCont; } @@ -5146,13 +5159,13 @@ WhereInfo *sqlite3WhereBegin( ** the index is listed as "{}". If the primary key is used the ** index name is '*'. */ - for(i=0; ia[i]; + pLevel = &pWInfo->a[ii]; w = pLevel->plan.wsFlags; pTabItem = &pTabList->a[pLevel->iFrom]; z = pTabItem->zAlias; From 46c35f9b208e7a46718e16fb2551fc940f26abd7 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 26 Sep 2012 23:17:01 +0000 Subject: [PATCH 3/9] Further refactoring of the ORDER BY related query-planning logic in order to make it easier to extend to support optimizing out ORDER BY on joins. No actual behavior changes, yet. FossilOrigin-Name: 96496ddae12a239b30a1fc997fbea43e3a75bfe7 --- manifest | 16 ++-- manifest.uuid | 2 +- src/select.c | 1 + src/sqliteInt.h | 3 +- src/where.c | 216 +++++++++++++++++++++++++++--------------------- 5 files changed, 135 insertions(+), 103 deletions(-) diff --git a/manifest b/manifest index 28323b288a..b047d69df4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Augment\sthe\sWhereBestIdx\sstructure\sto\spass\sdown\sinto\sthe\squery\splanner\s\ninformation\sthat\smight\sbe\sused\sto\sbetter\sdetect\sORDER\sBY\sand\sDISTINCT\noptimizations\sspanning\smultiple\stables\sof\sa\sjoin. -D 2012-09-25T20:43:35.989 +C Further\srefactoring\sof\sthe\sORDER\sBY\srelated\squery-planning\slogic\sin\sorder\nto\smake\sit\seasier\sto\sextend\sto\ssupport\soptimizing\sout\sORDER\sBY\son\sjoins.\nNo\sactual\sbehavior\schanges,\syet. +D 2012-09-26T23:17:01.276 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -174,12 +174,12 @@ F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 9e28280ec98035f31900fdd1db01f86f68ca6c32 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 1fad66b73a69c4004c9969a95b46d1f03390677d +F src/select.c a91b651652b43a8baaeec0ad13a6a6b290a4d0af F src/shell.c 8ee5a3cb502e2d574f97b43972e6c1e275e7bec7 F src/sqlite.h.in cbe846facaba903654b4136c97e7f57b3ac0bac7 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h 6b17114bb5c6ae30d360aeeef30ab40587afad4e +F src/sqliteInt.h cbcd2dd649338598a1a773a9adaf9f62e7256852 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c 5acb3e7bbd31f10ba39acad9ce6b399055337a9d F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c e75e67f0631182be4ff0d325c5a1e4c4ec3d0962 +F src/where.c 59b852d51d86fd3bf5935067994114bf88affbef F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1016,7 +1016,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 1104d42e104d561ce60d05d158acfe499ee9fd50 -R 660ef52f8cc671ae827c548793cfd963 +P 4226e51ff837f0ffe16355491a655d919d13488e +R 82e96539fb8ddcfab6615c786a34a1ea U drh -Z 10b5de991bc34ec524f2e3adc126c5c7 +Z e46432508017f3912d4e7cd1bd8daf63 diff --git a/manifest.uuid b/manifest.uuid index 1edaad7ba3..52ca9470ee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4226e51ff837f0ffe16355491a655d919d13488e \ No newline at end of file +96496ddae12a239b30a1fc997fbea43e3a75bfe7 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 01fe27694f..0f2320f59c 100644 --- a/src/select.c +++ b/src/select.c @@ -4506,6 +4506,7 @@ int sqlite3Select( goto select_end; } updateAccumulator(pParse, &sAggInfo); + assert( pMinMax==0 || pMinMax->nExpr==1 ); if( pWInfo->nOBSat>0 ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iBreak); VdbeComment((v, "%s() by index", diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7b9894254a..cd38dc8707 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1906,7 +1906,8 @@ struct SrcList { */ struct WherePlan { u32 wsFlags; /* WHERE_* flags that describe the strategy */ - u32 nEq; /* Number of == constraints */ + u16 nEq; /* Number of == constraints */ + u16 nOBSat; /* Number of ORDER BY terms satisfied */ double nRow; /* Estimated number of rows (for EQP) */ union { Index *pIdx; /* Index when WHERE_INDEXED is true */ diff --git a/src/where.c b/src/where.c index 63823f277d..05f5bcc414 100644 --- a/src/where.c +++ b/src/where.c @@ -285,7 +285,7 @@ struct WhereBestIdx { ExprList *pDistinct; /* The select-list if query is DISTINCT */ sqlite3_index_info **ppIdxInfo; /* Index information passed to xBestIndex */ int i, n; /* Which loop is being coded; # of loops */ - WhereLevel *a; /* Info about outer loops */ + WhereLevel *aLevel; /* Info about outer loops */ WhereCost cost; /* Lowest cost query plan */ }; @@ -1431,22 +1431,18 @@ static void exprAnalyze( } /* -** Return TRUE if any of the expressions in pList->a[iFirst...] contain -** a reference to any table other than the iBase table. +** Return TRUE if the given index is UNIQUE and all columns past the +** first nSkip columns are NOT NULL. */ -static int referencesOtherTables( - ExprList *pList, /* Search expressions in ths list */ - WhereMaskSet *pMaskSet, /* Mapping from tables to bitmaps */ - int iFirst, /* Be searching with the iFirst-th expression */ - int iBase /* Ignore references to this table */ -){ - Bitmask allowed = ~getMask(pMaskSet, iBase); - while( iFirstnExpr ){ - if( (exprTableUsage(pMaskSet, pList->a[iFirst++].pExpr)&allowed)!=0 ){ - return 1; - } +static int indexIsUniqueNotNull(Index *pIdx, int nSkip){ + Table *pTab = pIdx->pTable; + int i; + if( pIdx->onError==OE_None ) return 0; + for(i=nSkip; inColumn; i++){ + int j = pIdx->aiColumn[i]; + if( j>=0 && pTab->aCol[j].notNull==0 ) return 0; } - return 0; + return 1; } /* @@ -1612,43 +1608,54 @@ static int isDistinctRedundant( /* ** This routine decides if pIdx can be used to satisfy the ORDER BY -** clause. If it can, it returns 1. If pIdx cannot satisfy the -** ORDER BY clause, this routine returns 0. +** clause, either in whole or in part. The return value is the +** cumulative number of terms in the ORDER BY clause that are satisfied +** by the index pIdx and other indices in outer loops. ** -** pOrderBy is an ORDER BY clause from a SELECT statement. pTab is the -** left-most table in the FROM clause of that same SELECT statement and -** the table has a cursor number of "base". pIdx is an index on pTab. +** The table being queried has a cursor number of "base". pIdx is the +** index that is postulated for use to access the table. ** ** nEqCol is the number of columns of pIdx that are used as equality -** constraints. Any of these columns may be missing from the ORDER BY -** clause and the match can still be a success. +** constraints and where the other side of the == is an ordered column +** or constant. An "order column" in the previous sentence means a column +** in table from an outer loop whose values will always appear in the +** correct order due to othre index, or because the outer loop generates +** a unique result. Any of the first nEqCol columns of pIdx may be missing +** from the ORDER BY clause and the match can still be a success. ** -** All terms of the ORDER BY that match against the index must be either -** ASC or DESC. (Terms of the ORDER BY clause past the end of a UNIQUE -** index do not need to satisfy this constraint.) The *pbRev value is -** set to 1 if the ORDER BY clause is all DESC and it is set to 0 if -** the ORDER BY clause is all ASC. +** The *pbRev value is set to 0 order 1 depending on whether or not +** pIdx should be run in the forward order or in reverse order. */ static int isSortingIndex( - Parse *pParse, /* Parsing context */ - WhereMaskSet *pMaskSet, /* Mapping from table cursor numbers to bitmaps */ - Index *pIdx, /* The index we are testing */ - int base, /* Cursor number for the table to be sorted */ - ExprList *pOrderBy, /* The ORDER BY clause */ - int nEqCol, /* Number of index columns with == constraints */ - int wsFlags, /* Index usages flags */ - int *pbRev /* Set to 1 if ORDER BY is DESC */ + WhereBestIdx *p, /* Best index search context */ + Index *pIdx, /* The index we are testing */ + int base, /* Cursor number for the table to be sorted */ + int nEqCol, /* Number of index columns with ordered == constraints */ + int wsFlags, /* Index usages flags */ + int bOuterRev, /* True if outer loops scan in reverse order */ + int *pbRev /* Set to 1 for reverse-order scan of pIdx */ ){ - int i, j; /* Loop counters */ - int sortOrder = 0; /* XOR of index and ORDER BY sort direction */ - int nTerm; /* Number of ORDER BY terms */ - struct ExprList_item *pTerm; /* A term of the ORDER BY clause */ - sqlite3 *db = pParse->db; - - if( !pOrderBy ) return 0; - if( wsFlags & WHERE_COLUMN_IN ) return 0; - if( pIdx->bUnordered ) return 0; + int i; /* Number of pIdx terms used */ + int j; /* Number of ORDER BY terms satisfied */ + int sortOrder = 0; /* XOR of index and ORDER BY sort direction */ + int nTerm; /* Number of ORDER BY terms */ + struct ExprList_item *pTerm; /* A term of the ORDER BY clause */ + ExprList *pOrderBy; /* The ORDER BY clause */ + Parse *pParse = p->pParse; /* Parser context */ + sqlite3 *db = pParse->db; /* Database connection */ + int nPriorSat; /* ORDER BY terms satisfied by outer loops */ + int seenRowid = 0; /* True if an ORDER BY rowid term is seen */ + if( p->i==0 ){ + nPriorSat = 0; + }else{ + nPriorSat = p->aLevel[p->i-1].plan.nOBSat; + } + if( p->i>0 && nEqCol==0 /*&& !allOuterLoopsUnique(p)*/ ) return nPriorSat; + pOrderBy = p->pOrderBy; + if( !pOrderBy ) return nPriorSat; + if( wsFlags & WHERE_COLUMN_IN ) return nPriorSat; + if( pIdx->bUnordered ) return nPriorSat; nTerm = pOrderBy->nExpr; assert( nTerm>0 ); @@ -1665,7 +1672,7 @@ static int isSortingIndex( ** of the index is also allowed to match against the ORDER BY ** clause. */ - for(i=j=0, pTerm=pOrderBy->a; jnColumn; i++){ + for(i=0,j=nPriorSat,pTerm=&pOrderBy->a[j]; jnColumn; i++){ Expr *pExpr; /* The expression of the ORDER BY pTerm */ CollSeq *pColl; /* The collating sequence of pExpr */ int termSortOrder; /* Sort order for this term */ @@ -1709,7 +1716,7 @@ static int isSortingIndex( /* If an index column fails to match and is not constrained by == ** then the index cannot satisfy the ORDER BY constraint. */ - return 0; + return nPriorSat; } } assert( pIdx->aSortOrder!=0 || iColumn==-1 ); @@ -1720,53 +1727,38 @@ static int isSortingIndex( if( termSortOrder!=sortOrder ){ /* Indices can only be used if all ORDER BY terms past the ** equality constraints are all either DESC or ASC. */ - return 0; + break; } }else{ sortOrder = termSortOrder; } j++; pTerm++; - if( iColumn<0 && !referencesOtherTables(pOrderBy, pMaskSet, j, base) ){ - /* If the indexed column is the primary key and everything matches - ** so far and none of the ORDER BY terms to the right reference other - ** tables in the join, then we are assured that the index can be used - ** to sort because the primary key is unique and so none of the other - ** columns will make any difference - */ - j = nTerm; + if( iColumn<0 ){ + seenRowid = 1; + break; } } + *pbRev = bOuterRev ^ sortOrder; - *pbRev = sortOrder!=0; - if( j>=nTerm ){ - /* All terms of the ORDER BY clause are covered by this index so - ** this index can be used for sorting. */ - return 1; - } - if( pIdx->onError!=OE_None && i==pIdx->nColumn - && (wsFlags & WHERE_COLUMN_NULL)==0 - && !referencesOtherTables(pOrderBy, pMaskSet, j, base) + /* If there was an "ORDER BY rowid" term that matched, or it is only + ** possible for a single row from this table to match, then skip over + ** any additional ORDER BY terms dealing with this table. + */ + if( seenRowid || + ( (wsFlags & WHERE_COLUMN_NULL)==0 + && i>=pIdx->nColumn + && indexIsUniqueNotNull(pIdx, nEqCol) + ) ){ - Column *aCol = pIdx->pTable->aCol; - - /* All terms of this index match some prefix of the ORDER BY clause, - ** the index is UNIQUE, and no terms on the tail of the ORDER BY - ** refer to other tables in a join. So, assuming that the index entries - ** visited contain no NULL values, then this index delivers rows in - ** the required order. - ** - ** It is not possible for any of the first nEqCol index fields to be - ** NULL (since the corresponding "=" operator in the WHERE clause would - ** not be true). So if all remaining index columns have NOT NULL - ** constaints attached to them, we can be confident that the visited - ** index entries are free of NULLs. */ - for(i=nEqCol; inColumn; i++){ - if( aCol[pIdx->aiColumn[i]].notNull==0 ) break; + /* Advance j over additional ORDER BY terms associated with base */ + WhereMaskSet *pMS = p->pWC->pMaskSet; + Bitmask m = ~getMask(pMS, base); + while( ja[j].pExpr)&m)==0 ){ + j++; } - return (i==pIdx->nColumn); } - return 0; + return j; } /* @@ -2871,6 +2863,22 @@ static int whereInScanEst( } #endif /* defined(SQLITE_ENABLE_STAT3) */ +/* +** pTerm is an == constraint. Check to see if the other side of +** the == is a constant or a value that is guaranteed to be ordered +** by outer loops. Return 1 if pTerm is ordered, and 0 if not. +*/ +static int isOrderedTerm(WhereBestIdx *p, WhereTerm *pTerm, int *pbRev){ + if( p->i==0 ){ + return 1; /* All == are ordered in the outer loop */ + } + + + + /* If we cannot prove that the constraint is ordered, assume it is not */ + return 0; +} + /* ** Find the best query plan for accessing a particular table. Write the @@ -2968,7 +2976,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ double cost; /* Cost of using pProbe */ double nRow; /* Estimated number of rows in result set */ double log10N = (double)1; /* base-10 logarithm of nRow (inexact) */ - int rev; /* True to scan in reverse order */ + int bRev = 2; /* 0=forward scan. 1=reverse. 2=undecided */ int wsFlags = 0; Bitmask used = 0; @@ -3001,6 +3009,10 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** the sub-select is assumed to return 25 rows for the purposes of ** determining nInMul. ** + ** nOrdered: + ** The number of equality terms that are constrainted by outer loop + ** variables that are well-ordered. + ** ** bInEst: ** Set to true if there was at least one "x IN (SELECT ...)" term used ** in determining the value of nInMul. Note that the RHS of the @@ -3019,6 +3031,10 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** external sort (i.e. scanning the index being evaluated will not ** correctly order records). ** + ** bDistinct: + ** Boolean. True if there is a DISTINCT clause that will require an + ** external btree. + ** ** bLookup: ** Boolean. True if a table lookup is required for each index entry ** visited. In other words, true if this is not a covering index. @@ -3035,6 +3051,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** SELECT a, b, c FROM tbl WHERE a = 1; */ int nEq; /* Number of == or IN terms matching index */ + int nOrdered; /* Number of ordered terms matching index */ int bInEst = 0; /* True if "x IN (SELECT...)" seen */ int nInMul = 1; /* Number of distinct equalities to lookup */ double rangeDiv = (double)1; /* Estimated reduction in search space */ @@ -3042,11 +3059,14 @@ static void bestBtreeIndex(WhereBestIdx *p){ int bSort; /* True if external sort required */ int bDist; /* True if index cannot help with DISTINCT */ int bLookup = 0; /* True if not a covering index */ + int nOBSat = 0; /* Number of ORDER BY terms satisfied */ + int nOrderBy; /* Number of ORDER BY terms */ WhereTerm *pTerm; /* A single term of the WHERE clause */ #ifdef SQLITE_ENABLE_STAT3 WhereTerm *pFirstTerm = 0; /* First term matching the index */ #endif + nOrderBy = p->pOrderBy ? p->pOrderBy->nExpr : 0; if( (p->i) > 0 ){ bSort = 0; bDist = 0; @@ -3056,7 +3076,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ } /* Determine the values of nEq and nInMul */ - for(nEq=0; nEqnColumn; nEq++){ + for(nEq=nOrdered=0; nEqnColumn; nEq++){ int j = pProbe->aiColumn[nEq]; pTerm = findTerm(pWC, iCur, j, p->notReady, eqTermMask, pIdx); if( pTerm==0 ) break; @@ -3075,6 +3095,8 @@ static void bestBtreeIndex(WhereBestIdx *p){ } }else if( pTerm->eOperator & WO_ISNULL ){ wsFlags |= WHERE_COLUMN_NULL; + }else if( bSort && nEq==nOrdered && isOrderedTerm(p, pTerm, &bRev) ){ + nOrdered++; } #ifdef SQLITE_ENABLE_STAT3 if( nEq==0 && pProbe->aSample ) pFirstTerm = pTerm; @@ -3124,12 +3146,18 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** naturally scan rows in the required order, set the appropriate flags ** in wsFlags. Otherwise, if there is an ORDER BY clause but the index ** will scan rows in a different order, set the bSort variable. */ - if( bSort && isSortingIndex( - pParse, pWC->pMaskSet, pProbe, iCur, p->pOrderBy, nEq, wsFlags, &rev) - ){ - bSort = 0; - wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; - wsFlags |= (rev ? WHERE_REVERSE : 0); + assert( bRev>=0 && bRev<=2 ); + if( bSort ){ + testcase( bRev==0 ); + testcase( bRev==1 ); + testcase( bRev==2 ); + nOBSat = isSortingIndex(p, pProbe, iCur, nOrdered, + wsFlags, bRev&1, &bRev); + if( nOrderBy==nOBSat ){ + bSort = 0; + wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; + wsFlags |= (bRev==1 ? WHERE_REVERSE : 0); + } } /* If there is a DISTINCT qualifier and this index will scan rows in @@ -3272,7 +3300,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** difference and select C of 3.0. */ if( bSort ){ - cost += nRow*estLog(nRow)*3; + cost += nRow*estLog(nRow*(nOrderBy - nOBSat)/nOrderBy)*3; } if( bDist ){ cost += nRow*estLog(nRow)*3; @@ -3341,10 +3369,11 @@ static void bestBtreeIndex(WhereBestIdx *p){ WHERETRACE(( "%s(%s): nEq=%d nInMul=%d rangeDiv=%d bSort=%d bLookup=%d wsFlags=0x%x\n" - " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f used=0x%llx\n", + " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f used=0x%llx\n" + " nOrdered=%d nOBSat=%d\n", pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk"), nEq, nInMul, (int)rangeDiv, bSort, bLookup, wsFlags, - p->notReady, log10N, nRow, cost, used + p->notReady, log10N, nRow, cost, used, nOrdered, nOBSat )); /* If this index is the best we have seen so far, then record this @@ -3358,6 +3387,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ p->cost.plan.nRow = nRow; p->cost.plan.wsFlags = (wsFlags&wsFlagMask); p->cost.plan.nEq = nEq; + p->cost.plan.nOBSat = nOBSat; p->cost.plan.u.pIdx = pIdx; } @@ -4762,7 +4792,7 @@ WhereInfo *sqlite3WhereBegin( pWInfo->wctrlFlags = wctrlFlags; pWInfo->savedNQueryLoop = pParse->nQueryLoop; pMaskSet = (WhereMaskSet*)&sWBI.pWC[1]; - sWBI.a = pWInfo->a; + sWBI.aLevel = pWInfo->a; /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ From ff2d52117bd54c88229d4e46471d9dd2b257ad0e Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Sep 2012 12:05:09 +0000 Subject: [PATCH 4/9] Further tweaks to the query planner logic in preparation for adding ORDER BY opt-out for joins. FossilOrigin-Name: 53efc10af990d3f293551f3cd8ef2f8be2d9d716 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 23 ++++++++++------------- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index b047d69df4..5f591e9ff6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\srefactoring\sof\sthe\sORDER\sBY\srelated\squery-planning\slogic\sin\sorder\nto\smake\sit\seasier\sto\sextend\sto\ssupport\soptimizing\sout\sORDER\sBY\son\sjoins.\nNo\sactual\sbehavior\schanges,\syet. -D 2012-09-26T23:17:01.276 +C Further\stweaks\sto\sthe\squery\splanner\slogic\sin\spreparation\sfor\sadding\nORDER\sBY\sopt-out\sfor\sjoins. +D 2012-09-27T12:05:09.522 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c 5acb3e7bbd31f10ba39acad9ce6b399055337a9d F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 59b852d51d86fd3bf5935067994114bf88affbef +F src/where.c 67438c6192c5f777139e7abb642a9186a21a718d F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1016,7 +1016,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 4226e51ff837f0ffe16355491a655d919d13488e -R 82e96539fb8ddcfab6615c786a34a1ea +P 96496ddae12a239b30a1fc997fbea43e3a75bfe7 +R 57ad7fba1d5db0bb9191afb65acee747 U drh -Z e46432508017f3912d4e7cd1bd8daf63 +Z 58fdbd300662a0c1ec2665b77df5f795 diff --git a/manifest.uuid b/manifest.uuid index 52ca9470ee..2e377696fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96496ddae12a239b30a1fc997fbea43e3a75bfe7 \ No newline at end of file +53efc10af990d3f293551f3cd8ef2f8be2d9d716 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 05f5bcc414..0405860c9a 100644 --- a/src/where.c +++ b/src/where.c @@ -3067,13 +3067,8 @@ static void bestBtreeIndex(WhereBestIdx *p){ #endif nOrderBy = p->pOrderBy ? p->pOrderBy->nExpr : 0; - if( (p->i) > 0 ){ - bSort = 0; - bDist = 0; - }else{ - bSort = p->pOrderBy!=0; - bDist = p->pDistinct!=0; - } + bSort = nOrderBy>0 && (p->i==0 || p->aLevel[p->i-1].plan.nOBSati==0 && p->pDistinct!=0; /* Determine the values of nEq and nInMul */ for(nEq=nOrdered=0; nEqnColumn; nEq++){ @@ -3369,8 +3364,8 @@ static void bestBtreeIndex(WhereBestIdx *p){ WHERETRACE(( "%s(%s): nEq=%d nInMul=%d rangeDiv=%d bSort=%d bLookup=%d wsFlags=0x%x\n" - " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f used=0x%llx\n" - " nOrdered=%d nOBSat=%d\n", + " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f\n" + " used=0x%llx nOrdered=%d nOBSat=%d\n", pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk"), nEq, nInMul, (int)rangeDiv, bSort, bLookup, wsFlags, p->notReady, log10N, nRow, cost, used, nOrdered, nOBSat @@ -5024,8 +5019,9 @@ WhereInfo *sqlite3WhereBegin( && sWBI.cost.plan.nRow=0 ); assert( sWBI.notValid & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); WHERETRACE(("*** Optimizer selects table %d for loop %d" - " with cost=%g and nRow=%g\n", - bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow)); + " with cost=%.1f, nRow=%.1f, nOBSat=%d\n", + bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow, + bestPlan.plan.nOBSat)); if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){ pWInfo->nOBSat = pOrderBy->nExpr; } From 32634d270ce9de65ac8a1cec14df671a477ce91c Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Sep 2012 14:11:36 +0000 Subject: [PATCH 5/9] Enable ORDER BY clauses that span joins to be optimized out. FossilOrigin-Name: c29538f9b1ee4d4869999570604c9618ca0d08ac --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5f591e9ff6..a66358269a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\stweaks\sto\sthe\squery\splanner\slogic\sin\spreparation\sfor\sadding\nORDER\sBY\sopt-out\sfor\sjoins. -D 2012-09-27T12:05:09.522 +C Enable\sORDER\sBY\sclauses\sthat\sspan\sjoins\sto\sbe\soptimized\sout. +D 2012-09-27T14:11:36.514 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c 5acb3e7bbd31f10ba39acad9ce6b399055337a9d F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 67438c6192c5f777139e7abb642a9186a21a718d +F src/where.c 36af33a92ccbf8b5be3bc7eae08a7f022afb60d8 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1016,7 +1016,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 96496ddae12a239b30a1fc997fbea43e3a75bfe7 -R 57ad7fba1d5db0bb9191afb65acee747 +P 53efc10af990d3f293551f3cd8ef2f8be2d9d716 +R d9d746ad3fd7e5c77ffbaf4f5251f6f8 U drh -Z 58fdbd300662a0c1ec2665b77df5f795 +Z 769b41d4cd58dfa3626d0858103449b9 diff --git a/manifest.uuid b/manifest.uuid index 2e377696fc..c7339ef5c5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -53efc10af990d3f293551f3cd8ef2f8be2d9d716 \ No newline at end of file +c29538f9b1ee4d4869999570604c9618ca0d08ac \ No newline at end of file diff --git a/src/where.c b/src/where.c index 0405860c9a..ce5437a9f5 100644 --- a/src/where.c +++ b/src/where.c @@ -2863,17 +2863,67 @@ static int whereInScanEst( } #endif /* defined(SQLITE_ENABLE_STAT3) */ +/* +** Check to see if column iCol of the table with cursor iTab will appear +** in sorted order according to the current query plan. Return true if +** it will and false if not. +** +** If *pbRev is initially 2 (meaning "unknown") then set *pbRev to the +** sort order of iTab.iCol. If *pbRev is 0 or 1 but does not match +** the sort order of iTab.iCol, then consider the column to be unordered. +*/ +static int isOrderedColumn(WhereBestIdx *p, int iTab, int iCol, int *pbRev){ + int i, j; + WhereLevel *pLevel = &p->aLevel[p->i-1]; + Index *pIdx; + u8 sortOrder; + for(i=p->i-1; i>=0; i--, pLevel--){ + if( pLevel->iTabCur!=iTab ) continue; + if( (pLevel->plan.wsFlags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE))!=0 ){ + if( iCol!=(-1) ) return 0; + sortOrder = 0; + }else if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ + pIdx = pLevel->plan.u.pIdx; + for(j=0; jnColumn; j++){ + if( iCol==pIdx->aiColumn[j] ) break; + } + if( j>=pIdx->nColumn ) return 0; + sortOrder = pIdx->aSortOrder[j]; + }else{ + if( iCol!=(-1) ) return 0; + sortOrder = 0; + } + if( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ) sortOrder = 1 - sortOrder; + if( *pbRev==2 ){ + *pbRev = sortOrder; + return 1; + } + return (*pbRev==sortOrder); + } + return 0; +} + /* ** pTerm is an == constraint. Check to see if the other side of ** the == is a constant or a value that is guaranteed to be ordered ** by outer loops. Return 1 if pTerm is ordered, and 0 if not. */ static int isOrderedTerm(WhereBestIdx *p, WhereTerm *pTerm, int *pbRev){ + Expr *pExpr = pTerm->pExpr; + assert( pExpr->op==TK_EQ ); + assert( pExpr->pLeft!=0 && pExpr->pLeft->op==TK_COLUMN ); + assert( pExpr->pRight!=0 ); if( p->i==0 ){ return 1; /* All == are ordered in the outer loop */ } - - + if( pTerm->prereqRight==0 ){ + return 1; /* RHS of the == is a constant */ + } + if( pExpr->pRight->op==TK_COLUMN + && isOrderedColumn(p, pExpr->pRight->iTable, pExpr->pRight->iColumn, pbRev) + ){ + return 1; + } /* If we cannot prove that the constraint is ordered, assume it is not */ return 0; From 7e5418e4a487f7579c1249857bf69b7a6985bf32 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Sep 2012 15:05:54 +0000 Subject: [PATCH 6/9] Add more bits to the bit vector that is used to disable optimizations for built-in test. Add specific bit patterns to disable ORDER BY using an index in general and for joins. Use macros to test for bits in the disabled-optimization bit vector, in order to make the code clearer. FossilOrigin-Name: d2fcba1e143beca8c45724d2108870657c269e17 --- manifest | 24 +++++++-------- manifest.uuid | 2 +- src/delete.c | 4 ++- src/expr.c | 4 +-- src/main.c | 3 +- src/select.c | 4 +-- src/sqliteInt.h | 82 +++++++++++++++++++++++++++++-------------------- src/test1.c | 4 ++- src/where.c | 13 +++----- 9 files changed, 77 insertions(+), 63 deletions(-) diff --git a/manifest b/manifest index a66358269a..1bbd906b74 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\sORDER\sBY\sclauses\sthat\sspan\sjoins\sto\sbe\soptimized\sout. -D 2012-09-27T14:11:36.514 +C Add\smore\sbits\sto\sthe\sbit\svector\sthat\sis\sused\sto\sdisable\soptimizations\sfor\nbuilt-in\stest.\s\sAdd\sspecific\sbit\spatterns\sto\sdisable\sORDER\sBY\susing\san\nindex\sin\sgeneral\sand\sfor\sjoins.\s\sUse\smacros\sto\stest\sfor\sbits\sin\sthe\s\ndisabled-optimization\sbit\svector,\sin\sorder\sto\smake\sthe\scode\sclearer. +D 2012-09-27T15:05:54.735 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -129,8 +129,8 @@ F src/callback.c 0cb4228cdcd827dcc5def98fb099edcc9142dbcd F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 72a70dcfda75d3a1f81041ce4573e7afddcd8e4e F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 -F src/delete.c 335f36750dc6ac88d580aa36a6487459be9889de -F src/expr.c 4d1cef0fae6f3cf3c754773fd413f3e221021003 +F src/delete.c 4f7d773ec44b7db22b30ec9144f58a69154e09b7 +F src/expr.c 4de967b85f577ba00a7cdcb53d22070def6198db F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c c82a04e7a92bb728f9ab972b76590403283be2af F src/func.c cbb90dc84b22eea25caf39528d342279e61b8898 @@ -143,7 +143,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 0ee69fca0be54cd93939df98d2aca4ca46f44416 F src/loadext.c f20382fbaeec832438a1ba7797bee3d3c8a6d51d -F src/main.c 97d13e749ae84fe62238a5940c5b46b2b22cd369 +F src/main.c c5fec9814efaa64fa1de83497553f5904aa1bbec F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa @@ -174,17 +174,17 @@ F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 9e28280ec98035f31900fdd1db01f86f68ca6c32 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c a91b651652b43a8baaeec0ad13a6a6b290a4d0af +F src/select.c 75c5e37cc882c468383c9d9e07496b9a16cfae3e F src/shell.c 8ee5a3cb502e2d574f97b43972e6c1e275e7bec7 F src/sqlite.h.in cbe846facaba903654b4136c97e7f57b3ac0bac7 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h cbcd2dd649338598a1a773a9adaf9f62e7256852 +F src/sqliteInt.h 6ad55f4efbd1bb209711032e40b7c81c8855d634 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c e4de2458b3ef38fdd0498bc4e5ea5367a241b0f3 -F src/test1.c bec5295347a7bc38a53ca955f01cfcaf116fdb88 +F src/test1.c 9000293bfdcd67de05bf91d257343392c4b194f4 F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c 5acb3e7bbd31f10ba39acad9ce6b399055337a9d F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 36af33a92ccbf8b5be3bc7eae08a7f022afb60d8 +F src/where.c 4b837884c4747b022cf36bd00d9eddbb6b451241 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -1016,7 +1016,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 53efc10af990d3f293551f3cd8ef2f8be2d9d716 -R d9d746ad3fd7e5c77ffbaf4f5251f6f8 +P c29538f9b1ee4d4869999570604c9618ca0d08ac +R 112951e028390501419dede06d50cf2d U drh -Z 769b41d4cd58dfa3626d0858103449b9 +Z 58d07d9f2619813e6ef0c2cb6bcd8ae9 diff --git a/manifest.uuid b/manifest.uuid index c7339ef5c5..17a2e618d2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c29538f9b1ee4d4869999570604c9618ca0d08ac \ No newline at end of file +d2fcba1e143beca8c45724d2108870657c269e17 \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index 44e5995a69..f7f52865e1 100644 --- a/src/delete.c +++ b/src/delete.c @@ -638,7 +638,9 @@ int sqlite3GenerateIndexKey( } if( doMakeRec ){ const char *zAff; - if( pTab->pSelect || (pParse->db->flags & SQLITE_IdxRealAsInt)!=0 ){ + if( pTab->pSelect + || OptimizationDisabled(pParse->db, SQLITE_IdxRealAsInt) + ){ zAff = 0; }else{ zAff = sqlite3IndexAffinityStr(v, pIdx); diff --git a/src/expr.c b/src/expr.c index 3d63738813..2d1fb38ede 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2066,7 +2066,7 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ ** for testing only - to verify that SQLite always gets the same answer ** with and without the column cache. */ - if( pParse->db->flags & SQLITE_ColumnCache ) return; + if( OptimizationDisabled(pParse->db, SQLITE_ColumnCache) ) return; /* First replace any existing entry. ** @@ -3382,7 +3382,7 @@ static int evalConstExpr(Walker *pWalker, Expr *pExpr){ void sqlite3ExprCodeConstants(Parse *pParse, Expr *pExpr){ Walker w; if( pParse->cookieGoto ) return; - if( (pParse->db->flags & SQLITE_FactorOutConst)!=0 ) return; + if( OptimizationDisabled(pParse->db, SQLITE_FactorOutConst) ) return; w.xExprCallback = evalConstExpr; w.xSelectCallback = 0; w.pParse = pParse; diff --git a/src/main.c b/src/main.c index 466dee5516..8713c790c9 100644 --- a/src/main.c +++ b/src/main.c @@ -3018,8 +3018,7 @@ int sqlite3_test_control(int op, ...){ */ case SQLITE_TESTCTRL_OPTIMIZATIONS: { sqlite3 *db = va_arg(ap, sqlite3*); - int x = va_arg(ap,int); - db->flags = (x & SQLITE_OptMask) | (db->flags & ~SQLITE_OptMask); + db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); break; } diff --git a/src/select.c b/src/select.c index 0f2320f59c..2da14d93ed 100644 --- a/src/select.c +++ b/src/select.c @@ -2809,7 +2809,7 @@ static int flattenSubquery( */ assert( p!=0 ); assert( p->pPrior==0 ); /* Unable to flatten compound queries */ - if( db->flags & SQLITE_QueryFlattener ) return 0; + if( OptimizationDisabled(db, SQLITE_QueryFlattener) ) return 0; pSrc = p->pSrc; assert( pSrc && iFrom>=0 && iFromnSrc ); pSubitem = &pSrc->a[iFrom]; @@ -4012,7 +4012,7 @@ int sqlite3Select( ** to disable this optimization for testing purposes. */ if( sqlite3ExprListCompare(p->pGroupBy, pOrderBy)==0 - && (db->flags & SQLITE_GroupByOrder)==0 ){ + && OptimizationEnabled(db, SQLITE_GroupByOrder) ){ pOrderBy = 0; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index cd38dc8707..4501c737f3 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -827,6 +827,7 @@ struct sqlite3 { unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ + u16 dbOptFlags; /* Flags to enable/disable optimizations */ u8 autoCommit; /* The auto-commit flag. */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ @@ -931,46 +932,59 @@ struct sqlite3 { /* ** Possible values for the sqlite3.flags. */ -#define SQLITE_VdbeTrace 0x00000100 /* True to trace VDBE execution */ -#define SQLITE_InternChanges 0x00000200 /* Uncommitted Hash table changes */ -#define SQLITE_FullColNames 0x00000400 /* Show full column names on SELECT */ -#define SQLITE_ShortColNames 0x00000800 /* Show short columns names */ -#define SQLITE_CountRows 0x00001000 /* Count rows changed by INSERT, */ +#define SQLITE_VdbeTrace 0x00000001 /* True to trace VDBE execution */ +#define SQLITE_InternChanges 0x00000002 /* Uncommitted Hash table changes */ +#define SQLITE_FullColNames 0x00000004 /* Show full column names on SELECT */ +#define SQLITE_ShortColNames 0x00000008 /* Show short columns names */ +#define SQLITE_CountRows 0x00000010 /* Count rows changed by INSERT, */ /* DELETE, or UPDATE and return */ /* the count using a callback. */ -#define SQLITE_NullCallback 0x00002000 /* Invoke the callback once if the */ +#define SQLITE_NullCallback 0x00000020 /* Invoke the callback once if the */ /* result set is empty */ -#define SQLITE_SqlTrace 0x00004000 /* Debug print SQL as it executes */ -#define SQLITE_VdbeListing 0x00008000 /* Debug listings of VDBE programs */ -#define SQLITE_WriteSchema 0x00010000 /* OK to update SQLITE_MASTER */ - /* 0x00020000 Unused */ -#define SQLITE_IgnoreChecks 0x00040000 /* Do not enforce check constraints */ -#define SQLITE_ReadUncommitted 0x0080000 /* For shared-cache mode */ -#define SQLITE_LegacyFileFmt 0x00100000 /* Create new databases in format 1 */ -#define SQLITE_FullFSync 0x00200000 /* Use full fsync on the backend */ -#define SQLITE_CkptFullFSync 0x00400000 /* Use full fsync for checkpoint */ -#define SQLITE_RecoveryMode 0x00800000 /* Ignore schema errors */ -#define SQLITE_ReverseOrder 0x01000000 /* Reverse unordered SELECTs */ -#define SQLITE_RecTriggers 0x02000000 /* Enable recursive triggers */ -#define SQLITE_ForeignKeys 0x04000000 /* Enforce foreign key constraints */ -#define SQLITE_AutoIndex 0x08000000 /* Enable automatic indexes */ -#define SQLITE_PreferBuiltin 0x10000000 /* Preference to built-in funcs */ -#define SQLITE_LoadExtension 0x20000000 /* Enable load_extension */ -#define SQLITE_EnableTrigger 0x40000000 /* True to enable triggers */ +#define SQLITE_SqlTrace 0x00000040 /* Debug print SQL as it executes */ +#define SQLITE_VdbeListing 0x00000080 /* Debug listings of VDBE programs */ +#define SQLITE_WriteSchema 0x00000100 /* OK to update SQLITE_MASTER */ + /* 0x00000200 Unused */ +#define SQLITE_IgnoreChecks 0x00000400 /* Do not enforce check constraints */ +#define SQLITE_ReadUncommitted 0x0000800 /* For shared-cache mode */ +#define SQLITE_LegacyFileFmt 0x00001000 /* Create new databases in format 1 */ +#define SQLITE_FullFSync 0x00002000 /* Use full fsync on the backend */ +#define SQLITE_CkptFullFSync 0x00004000 /* Use full fsync for checkpoint */ +#define SQLITE_RecoveryMode 0x00008000 /* Ignore schema errors */ +#define SQLITE_ReverseOrder 0x00010000 /* Reverse unordered SELECTs */ +#define SQLITE_RecTriggers 0x00020000 /* Enable recursive triggers */ +#define SQLITE_ForeignKeys 0x00040000 /* Enforce foreign key constraints */ +#define SQLITE_AutoIndex 0x00080000 /* Enable automatic indexes */ +#define SQLITE_PreferBuiltin 0x00100000 /* Preference to built-in funcs */ +#define SQLITE_LoadExtension 0x00200000 /* Enable load_extension */ +#define SQLITE_EnableTrigger 0x00400000 /* True to enable triggers */ /* -** Bits of the sqlite3.flags field that are used by the -** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface. -** These must be the low-order bits of the flags field. +** Bits of the sqlite3.dbOptFlags field that are used by the +** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to +** selectively disable various optimizations. */ -#define SQLITE_QueryFlattener 0x01 /* Disable query flattening */ -#define SQLITE_ColumnCache 0x02 /* Disable the column cache */ -#define SQLITE_GroupByOrder 0x04 /* Disable GROUPBY cover of ORDERBY */ -#define SQLITE_FactorOutConst 0x08 /* Disable factoring out constants */ -#define SQLITE_IdxRealAsInt 0x10 /* Store REAL as INT in indices */ -#define SQLITE_DistinctOpt 0x20 /* DISTINCT using indexes */ -#define SQLITE_CoverIdxScan 0x40 /* Disable covering index scans */ -#define SQLITE_OptMask 0xff /* Mask of all disablable opts */ +#define SQLITE_QueryFlattener 0x0001 /* Query flattening */ +#define SQLITE_ColumnCache 0x0002 /* Column cache */ +#define SQLITE_GroupByOrder 0x0004 /* GROUPBY cover of ORDERBY */ +#define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ +#define SQLITE_IdxRealAsInt 0x0010 /* Store REAL as INT in indices */ +#define SQLITE_DistinctOpt 0x0020 /* DISTINCT using indexes */ +#define SQLITE_CoverIdxScan 0x0040 /* Covering index scans */ +#define SQLITE_OrderByIdx 0x0180 /* ORDER BY using indices */ +#define SQLITE_OrderByIdxJoin 0x0100 /* ORDER BY of joins via index */ +#define SQLITE_AllOpts 0x01ff /* All optimizations */ + +/* +** Macros for testing whether or not optimizations are enabled or disabled. +*/ +#ifndef SQLITE_OMIT_BUILTIN_TEST +#define OptimizationDisabled(db, mask) (((db)->dbOptFlags&(mask))!=0) +#define OptimizationEnabled(db, mask) (((db)->dbOptFlags&(mask))==0) +#else +#define OptimizationDisabled(db, mask) 0 +#define OptimizationEnabled(db, mask) 1 +#endif /* ** Possible values for the sqlite.magic field. diff --git a/src/test1.c b/src/test1.c index 0b9b812e8f..45ca124bb2 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5933,7 +5933,7 @@ static int optimization_control( const char *zOptName; int mask; } aOpt[] = { - { "all", SQLITE_OptMask }, + { "all", SQLITE_AllOpts }, { "query-flattener", SQLITE_QueryFlattener }, { "column-cache", SQLITE_ColumnCache }, { "groupby-order", SQLITE_GroupByOrder }, @@ -5941,6 +5941,8 @@ static int optimization_control( { "real-as-int", SQLITE_IdxRealAsInt }, { "distinct-opt", SQLITE_DistinctOpt }, { "cover-idx-scan", SQLITE_CoverIdxScan }, + { "order-by-idx", SQLITE_OrderByIdx }, + { "order-by-idx-join",SQLITE_OrderByIdxJoin }, }; if( objc!=4 ){ diff --git a/src/where.c b/src/where.c index ce5437a9f5..0395496572 100644 --- a/src/where.c +++ b/src/where.c @@ -1646,9 +1646,11 @@ static int isSortingIndex( int nPriorSat; /* ORDER BY terms satisfied by outer loops */ int seenRowid = 0; /* True if an ORDER BY rowid term is seen */ + if( OptimizationDisabled(db, SQLITE_OrderByIdx) ) return 0; if( p->i==0 ){ nPriorSat = 0; }else{ + if( OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; nPriorSat = p->aLevel[p->i-1].plan.nOBSat; } if( p->i>0 && nEqCol==0 /*&& !allOuterLoopsUnique(p)*/ ) return nPriorSat; @@ -2879,10 +2881,7 @@ static int isOrderedColumn(WhereBestIdx *p, int iTab, int iCol, int *pbRev){ u8 sortOrder; for(i=p->i-1; i>=0; i--, pLevel--){ if( pLevel->iTabCur!=iTab ) continue; - if( (pLevel->plan.wsFlags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE))!=0 ){ - if( iCol!=(-1) ) return 0; - sortOrder = 0; - }else if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ + if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ pIdx = pLevel->plan.u.pIdx; for(j=0; jnColumn; j++){ if( iCol==pIdx->aiColumn[j] ) break; @@ -3289,9 +3288,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ if( wsFlags==WHERE_IDX_ONLY && (pWC->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 && sqlite3GlobalConfig.bUseCis -#ifndef SQLITE_OMIT_BUILTIN_TEST - && (pParse->db->flags & SQLITE_CoverIdxScan)==0 -#endif + && OptimizationEnabled(pParse->db, SQLITE_CoverIdxScan) ){ /* This index is not useful for indexing, but it is a covering index. ** A full-scan of the index might be a little faster than a full-scan @@ -4841,7 +4838,7 @@ WhereInfo *sqlite3WhereBegin( /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ - if( db->flags & SQLITE_DistinctOpt ) pDistinct = 0; + if( OptimizationDisabled(db, SQLITE_DistinctOpt) ) pDistinct = 0; /* Split the WHERE clause into separate subexpressions where each ** subexpression is separated by an AND operator. From 6b10a6a7edaa09b0e9151fe5b0185d33dce0f06f Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Sep 2012 17:31:32 +0000 Subject: [PATCH 7/9] Test cases and bug fixes applied to the ORDER BY optimization for joins. Some test cases fail, but except for the new orderby1.test failures, all failures appear to be issues with the tests, not with the core code. FossilOrigin-Name: 75cda864ededb6dc0f84bd52ed3311753a58e351 --- manifest | 17 ++--- manifest.uuid | 2 +- src/where.c | 23 +++++-- test/collate4.test | 2 +- test/collate5.test | 4 +- test/orderby1.test | 155 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 185 insertions(+), 18 deletions(-) create mode 100644 test/orderby1.test diff --git a/manifest b/manifest index 1bbd906b74..e8dbbe14e0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smore\sbits\sto\sthe\sbit\svector\sthat\sis\sused\sto\sdisable\soptimizations\sfor\nbuilt-in\stest.\s\sAdd\sspecific\sbit\spatterns\sto\sdisable\sORDER\sBY\susing\san\nindex\sin\sgeneral\sand\sfor\sjoins.\s\sUse\smacros\sto\stest\sfor\sbits\sin\sthe\s\ndisabled-optimization\sbit\svector,\sin\sorder\sto\smake\sthe\scode\sclearer. -D 2012-09-27T15:05:54.735 +C Test\scases\sand\sbug\sfixes\sapplied\sto\sthe\sORDER\sBY\soptimization\sfor\sjoins.\nSome\stest\scases\sfail,\sbut\sexcept\sfor\sthe\snew\sorderby1.test\sfailures,\sall\nfailures\sappear\sto\sbe\sissues\swith\sthe\stests,\snot\swith\sthe\score\scode. +D 2012-09-27T17:31:32.268 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c 5acb3e7bbd31f10ba39acad9ce6b399055337a9d F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 4b837884c4747b022cf36bd00d9eddbb6b451241 +F src/where.c a537824b2eec986c203a6a370090520b955f258b F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -324,8 +324,8 @@ F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 F test/collate1.test e3eaa48c21e150814be1a7b852d2a8af24458d04 F test/collate2.test 04cebe4a033be319d6ddbb3bbc69464e01700b49 F test/collate3.test d28d2cfab2c3a3d4628ae4b2b7afc9965daa3b4c -F test/collate4.test d37682293d3c32223dec2e6afdeaf9de18415248 -F test/collate5.test 67f1d3e848e230ff4802815a79acb0a8b5e69bd7 +F test/collate4.test 27adc324b58ec367bdd0bc6baab628af5129e9f1 +F test/collate5.test 65d928034d30d2d263a80f6359f7549ee1598ec6 F test/collate6.test 8be65a182abaac8011a622131486dafb8076e907 F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 F test/collate8.test df26649cfcbddf109c04122b340301616d3a88f6 @@ -634,6 +634,7 @@ F test/notify3.test a86259abbfb923aa27d30f0fc038c88e5251488a F test/notnull.test cc7c78340328e6112a13c3e311a9ab3127114347 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 +F test/orderby1.test 868a4f88c4f4565684bcc18e8ed61a990ece431b F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 2163c6ef119f497a71a84137c957c63763e640ab F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 @@ -1016,7 +1017,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P c29538f9b1ee4d4869999570604c9618ca0d08ac -R 112951e028390501419dede06d50cf2d +P d2fcba1e143beca8c45724d2108870657c269e17 +R 388197fbdd69fccc34c9d73c7bc9f9e7 U drh -Z 58d07d9f2619813e6ef0c2cb6bcd8ae9 +Z f07796285dcd213d3100411a37b9d38f diff --git a/manifest.uuid b/manifest.uuid index 17a2e618d2..38f15f501f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2fcba1e143beca8c45724d2108870657c269e17 \ No newline at end of file +75cda864ededb6dc0f84bd52ed3311753a58e351 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 0395496572..01155c14d3 100644 --- a/src/where.c +++ b/src/where.c @@ -2883,16 +2883,27 @@ static int isOrderedColumn(WhereBestIdx *p, int iTab, int iCol, int *pbRev){ if( pLevel->iTabCur!=iTab ) continue; if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ pIdx = pLevel->plan.u.pIdx; - for(j=0; jnColumn; j++){ - if( iCol==pIdx->aiColumn[j] ) break; + if( iCol<0 ){ + sortOrder = 0; + testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ); + }else{ + for(j=0; jnColumn; j++){ + if( iCol==pIdx->aiColumn[j] ) break; + } + if( j>=pIdx->nColumn ) return 0; + sortOrder = pIdx->aSortOrder[j]; + testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ); } - if( j>=pIdx->nColumn ) return 0; - sortOrder = pIdx->aSortOrder[j]; }else{ if( iCol!=(-1) ) return 0; sortOrder = 0; + testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ); + } + if( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ){ + assert( sortOrder==0 || sortOrder==1 ); + testcase( sortOrder==1 ); + sortOrder = 1 - sortOrder; } - if( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ) sortOrder = 1 - sortOrder; if( *pbRev==2 ){ *pbRev = sortOrder; return 1; @@ -3200,8 +3211,8 @@ static void bestBtreeIndex(WhereBestIdx *p){ if( nOrderBy==nOBSat ){ bSort = 0; wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; - wsFlags |= (bRev==1 ? WHERE_REVERSE : 0); } + if( bRev ) wsFlags |= WHERE_REVERSE; } /* If there is a DISTINCT qualifier and this index will scan rows in diff --git a/test/collate4.test b/test/collate4.test index 6b3a1c7aee..1cf8caf04c 100644 --- a/test/collate4.test +++ b/test/collate4.test @@ -389,7 +389,7 @@ ifcapable subquery { SELECT a FROM collate4t1 WHERE a IN (SELECT * FROM collate4t2) ORDER BY rowid } - } {a A 6} + } {a A 5} do_test collate4-2.1.8 { count { SELECT a FROM collate4t1 WHERE a IN ('z', 'a'); diff --git a/test/collate5.test b/test/collate5.test index e5bea7a5a5..5f8697ea51 100644 --- a/test/collate5.test +++ b/test/collate5.test @@ -221,7 +221,7 @@ do_test collate5-3.0 { execsql { SELECT a FROM collate5t1 UNION ALL SELECT a FROM collate5t2 ORDER BY 1; } -} {a A a A b B b B n N} +} {/[aA] [aA] [aA] [aA] [bB] [bB] [bB] [bB] [nN] [nN]/} do_test collate5-3.1 { execsql { SELECT a FROM collate5t2 UNION ALL SELECT a FROM collate5t1 ORDER BY 1; @@ -282,7 +282,7 @@ do_test collate5-4.2 { execsql { SELECT a, b, count(*) FROM collate5t1 GROUP BY a, b ORDER BY a, b; } -} {A 1.0 2 b 2 1 B 3 1} +} {/[aA] 1(.0)? 2 [bB] 2 1 [bB] 3 1/} do_test collate5-4.3 { execsql { DROP TABLE collate5t1; diff --git a/test/orderby1.test b/test/orderby1.test new file mode 100644 index 0000000000..2837dfcab3 --- /dev/null +++ b/test/orderby1.test @@ -0,0 +1,155 @@ +# 2012 Sept 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing that the optimizations that disable +# ORDER BY clauses when the natural order of a query is correct. +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix orderby1 + +# Generate test data for a join. Verify that the join gets the +# correct answer. +# +do_test 1.0 { + db eval { + BEGIN; + CREATE TABLE album( + aid INTEGER PRIMARY KEY, + title TEXT UNIQUE NOT NULL + ); + CREATE TABLE track( + tid INTEGER PRIMARY KEY, + aid INTEGER NOT NULL REFERENCES album, + tn INTEGER NOT NULL, + name TEXT, + UNIQUE(aid, tn) + ); + INSERT INTO album VALUES(1, '1-one'), (2, '2-two'), (3, '3-three'); + INSERT INTO track VALUES + (NULL, 1, 1, 'one-a'), + (NULL, 2, 2, 'two-b'), + (NULL, 3, 3, 'three-c'), + (NULL, 1, 3, 'one-c'), + (NULL, 2, 1, 'two-a'), + (NULL, 3, 1, 'three-a'); + COMMIT; + } +} {} +do_test 1.1a { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn + } +} {one-a one-c two-a two-b three-a three-c} + +# Verify that the ORDER BY clause is optimized out +# +do_test 1.1b { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn + } +} {~/ORDER BY/} ;# ORDER BY optimized out + +# The same query with ORDER BY clause optimization disabled via + operators +# should give exactly the same answer. +# +do_test 1.2a { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn + } +} {one-a one-c two-a two-b three-a three-c} + +# The output is sorted manually in this case. +# +do_test 1.2b { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn + } +} {/ORDER BY/} ;# separate sorting pass due to "+" on ORDER BY terms + +# The same query with ORDER BY optimizations turned off via built-in test. +# +do_test 1.3a { + optimization_control db order-by-idx-join 0 + db cache flush + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn + } +} {one-a one-c two-a two-b three-a three-c} +do_test 1.3b { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn + } +} {/ORDER BY/} ;# separate sorting pass due to disabled optimization +optimization_control db all 1 +db cache flush + +# Reverse order sorts +# +do_test 1.4a { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn + } +} {three-a three-c two-a two-b one-a one-c} +do_test 1.4b { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY +title DESC, +tn + } +} {three-a three-c two-a two-b one-a one-c} ;# verify same order after sorting +do_test 1.4c { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn + } +} {~/ORDER BY/} ;# ORDER BY optimized-out + + +do_test 1.5a { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn DESC + } +} {one-c one-a two-b two-a three-c three-a} +do_test 1.5b { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn DESC + } +} {one-c one-a two-b two-a three-c three-a} ;# verify same order after sorting +do_test 1.5c { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn + } +} {~/ORDER BY/} ;# ORDER BY optimized-out + +do_test 1.6a { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn DESC + } +} {three-c three-a two-b two-a one-c one-a} +do_test 1.6b { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY +title DESC, +tn DESC + } +} {three-c three-a two-b two-a one-c one-a} ;# verify same order after sorting +do_test 1.6c { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn DESC + } +} {~/ORDER BY/} ;# ORDER BY optimized-out + + +finish_test From 5343b2d4a8f778672cffe9ea2d2dd2d0bce7d6ac Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Sep 2012 19:53:38 +0000 Subject: [PATCH 8/9] More test cases an bug fixes for the ORDER BY optimization of joins. All veryquick tests now pass. FossilOrigin-Name: 0d573320057b0903a5589cabfb1b1ece1c57958e --- manifest | 22 ++-- manifest.uuid | 2 +- src/where.c | 23 ++-- test/e_select.test | 16 +-- test/orderby1.test | 277 ++++++++++++++++++++++++++++++++++++++- test/tester.tcl | 4 +- test/tkt-cbd054fa6b.test | 4 +- test/where.test | 6 +- 8 files changed, 315 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index e8dbbe14e0..1b05b5bd9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\scases\sand\sbug\sfixes\sapplied\sto\sthe\sORDER\sBY\soptimization\sfor\sjoins.\nSome\stest\scases\sfail,\sbut\sexcept\sfor\sthe\snew\sorderby1.test\sfailures,\sall\nfailures\sappear\sto\sbe\sissues\swith\sthe\stests,\snot\swith\sthe\score\scode. -D 2012-09-27T17:31:32.268 +C More\stest\scases\san\sbug\sfixes\sfor\sthe\sORDER\sBY\soptimization\sof\sjoins.\s\sAll\nveryquick\stests\snow\spass. +D 2012-09-27T19:53:38.142 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c 5acb3e7bbd31f10ba39acad9ce6b399055337a9d F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c a537824b2eec986c203a6a370090520b955f258b +F src/where.c cd047c1223f6364f869d44b7d3941a687de7fcb6 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -386,7 +386,7 @@ F test/e_fts3.test 5c02288842e4f941896fd44afdef564dd5fc1459 F test/e_insert.test c6ac239a97cb16dfbd0c16496f8cd871b4068c0c F test/e_reindex.test dfedfc32c5a282b0596c6537cbcd4217fbb1a216 F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 -F test/e_select.test 69013a64b469458820abb7f3281a7eaa6c1fda76 +F test/e_select.test 07e8d81268ba1ffcaf1dc4bec48956af150c42f9 F test/e_select2.test 5c3d3da19c7b3e90ae444579db2b70098599ab92 F test/e_update.test 161d5dc6a3ed9dd08f5264d13e20735d7a89f00c F test/e_uri.test 9e190ca799d9190eec6e43f2aadf1d10c06a57a3 @@ -634,7 +634,7 @@ F test/notify3.test a86259abbfb923aa27d30f0fc038c88e5251488a F test/notnull.test cc7c78340328e6112a13c3e311a9ab3127114347 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 -F test/orderby1.test 868a4f88c4f4565684bcc18e8ed61a990ece431b +F test/orderby1.test 31c9865626046666e81cd22ecf8e1c24a4ea41b6 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 2163c6ef119f497a71a84137c957c63763e640ab F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 @@ -740,7 +740,7 @@ F test/tclsqlite.test a3d2df21ee98957f5de4f9dc1db0eab68047ab5d F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test 51edd31c65ed1560dd600b1796e8325df96318e2 F test/temptrigger.test 26670ed7a39cf2296a7f0a9e0a1d7bdb7abe936d -F test/tester.tcl ed47103d30a1a4b3c1d8de207606a407e55cc5c2 +F test/tester.tcl 2f383e811010b05a83c0f00fc168cae1dd63a6d9 F test/thread001.test 7cc2ce08f9cde95964736d11e91f9ab610f82f91 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -784,7 +784,7 @@ F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3 F test/tkt-bd484a090c.test 60460bf946f79a79712b71f202eda501ca99b898 F test/tkt-bdc6bbbb38.test fc38bb09bdd440e3513a1f5f98fc60a075182d7d F test/tkt-c48d99d690.test bed446e3513ae10eec1b86fdd186ef750226c408 -F test/tkt-cbd054fa6b.test bd9fb546f63bc0c79d1776978d059fa51c5b1c63 +F test/tkt-cbd054fa6b.test 9c27ed07b333eed458e5d4543f91ecdcf05aeb19 F test/tkt-d11f09d36e.test fb44f7961aa6d4b632fb7b9768239832210b5fc7 F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 @@ -955,7 +955,7 @@ F test/walro.test a31deb621033442a76c3a61e44929250d06f81b1 F test/walshared.test 6dda2293880c300baf5d791c307f653094585761 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e -F test/where.test 59cf231e6edaf0f20b106a26d4294847e7c6eb25 +F test/where.test ea9659ff6e31681d0cdaf964747139cb10808200 F test/where2.test 43d4becaf5a5df854e6c21d624a1cb84c6904554 F test/where3.test 667e75642102c97a00bf9b23d3cb267db321d006 F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2 @@ -1017,7 +1017,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P d2fcba1e143beca8c45724d2108870657c269e17 -R 388197fbdd69fccc34c9d73c7bc9f9e7 +P 75cda864ededb6dc0f84bd52ed3311753a58e351 +R 2ac4d1c44b199e10f1d07ecdc3868045 U drh -Z f07796285dcd213d3100411a37b9d38f +Z 9df007fa9fa34b8089a8009c6ca9320b diff --git a/manifest.uuid b/manifest.uuid index 38f15f501f..f52b6aaffd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75cda864ededb6dc0f84bd52ed3311753a58e351 \ No newline at end of file +0d573320057b0903a5589cabfb1b1ece1c57958e \ No newline at end of file diff --git a/src/where.c b/src/where.c index 01155c14d3..9bdcd536c4 100644 --- a/src/where.c +++ b/src/where.c @@ -1645,13 +1645,17 @@ static int isSortingIndex( sqlite3 *db = pParse->db; /* Database connection */ int nPriorSat; /* ORDER BY terms satisfied by outer loops */ int seenRowid = 0; /* True if an ORDER BY rowid term is seen */ + int nEqOneRow; /* Idx columns that ref unique values */ if( OptimizationDisabled(db, SQLITE_OrderByIdx) ) return 0; if( p->i==0 ){ nPriorSat = 0; + nEqOneRow = nEqCol; }else{ if( OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; nPriorSat = p->aLevel[p->i-1].plan.nOBSat; + sortOrder = bOuterRev; + nEqOneRow = 0; } if( p->i>0 && nEqCol==0 /*&& !allOuterLoopsUnique(p)*/ ) return nPriorSat; pOrderBy = p->pOrderBy; @@ -1725,7 +1729,7 @@ static int isSortingIndex( assert( pTerm->sortOrder==0 || pTerm->sortOrder==1 ); assert( iSortOrder==0 || iSortOrder==1 ); termSortOrder = iSortOrder ^ pTerm->sortOrder; - if( i>nEqCol ){ + if( i>nEqOneRow ){ if( termSortOrder!=sortOrder ){ /* Indices can only be used if all ORDER BY terms past the ** equality constraints are all either DESC or ASC. */ @@ -1741,7 +1745,7 @@ static int isSortingIndex( break; } } - *pbRev = bOuterRev ^ sortOrder; + *pbRev = sortOrder; /* If there was an "ORDER BY rowid" term that matched, or it is only ** possible for a single row from this table to match, then skip over @@ -3296,7 +3300,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** So this computation assumes table records are about twice as big ** as index records */ - if( wsFlags==WHERE_IDX_ONLY + if( (wsFlags&~WHERE_REVERSE)==WHERE_IDX_ONLY && (pWC->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 && sqlite3GlobalConfig.bUseCis && OptimizationEnabled(pParse->db, SQLITE_CoverIdxScan) @@ -3421,9 +3425,10 @@ static void bestBtreeIndex(WhereBestIdx *p){ WHERETRACE(( - "%s(%s): nEq=%d nInMul=%d rangeDiv=%d bSort=%d bLookup=%d wsFlags=0x%x\n" - " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f\n" - " used=0x%llx nOrdered=%d nOBSat=%d\n", + "%s(%s):\n" + " nEq=%d nInMul=%d rangeDiv=%d bSort=%d bLookup=%d wsFlags=0x%08x\n" + " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f\n" + " used=0x%llx nOrdered=%d nOBSat=%d\n", pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk"), nEq, nInMul, (int)rangeDiv, bSort, bLookup, wsFlags, p->notReady, log10N, nRow, cost, used, nOrdered, nOBSat @@ -5088,10 +5093,10 @@ WhereInfo *sqlite3WhereBegin( } assert( bestJ>=0 ); assert( sWBI.notValid & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); - WHERETRACE(("*** Optimizer selects table %d for loop %d" - " with cost=%.1f, nRow=%.1f, nOBSat=%d\n", + WHERETRACE(("*** Optimizer selects table %d for loop %d with:\n" + " cost=%.1f, nRow=%.1f, nOBSat=%d wsFlags=0x%08x\n", bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow, - bestPlan.plan.nOBSat)); + bestPlan.plan.nOBSat, bestPlan.plan.wsFlags)); if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){ pWInfo->nOBSat = pOrderBy->nExpr; } diff --git a/test/e_select.test b/test/e_select.test index 9f26f6fcd3..fb63d051de 100644 --- a/test/e_select.test +++ b/test/e_select.test @@ -1023,7 +1023,7 @@ do_execsql_test e_select-4.9.0 { # do_select_tests e_select-4.9 { 1 "SELECT group_concat(one), two FROM b1 GROUP BY two" { - 4,5 f 1 o 7,6 s 3,2 t + /#,# f 1 o #,# s #,# t/ } 2 "SELECT group_concat(one), sum(one) FROM b1 GROUP BY (one>4)" { 1,2,3,4 10 5,6,7 18 @@ -1040,7 +1040,7 @@ do_select_tests e_select-4.9 { # values are considered equal. # do_select_tests e_select-4.10 { - 1 "SELECT group_concat(y) FROM b2 GROUP BY x" {0,1 3 2,4} + 1 "SELECT group_concat(y) FROM b2 GROUP BY x" {/#,# 3 #,#/} 2 "SELECT count(*) FROM b2 GROUP BY CASE WHEN y<4 THEN NULL ELSE 0 END" {4 1} } @@ -1745,12 +1745,12 @@ do_select_tests e_select-8.4 { 1 2 7 1 2 8 1 4 93 1 5 -1 } 8 "SELECT z, x FROM d1 ORDER BY 2" { - 3 1 8 1 7 1 -20 1 - 93 1 -1 1 -1 2 93 2 + /# 1 # 1 # 1 # 1 + # 1 # 1 # 2 # 2/ } 9 "SELECT z, x FROM d1 ORDER BY 1" { - -20 1 -1 2 -1 1 3 1 - 7 1 8 1 93 2 93 1 + /-20 1 -1 # -1 # 3 1 + 7 1 8 1 93 # 93 #/ } } @@ -1766,10 +1766,10 @@ do_select_tests e_select-8.5 { 94 94 9 8 4 0 0 -19 } 3 "SELECT z AS x, x AS z FROM d1 ORDER BY z" { - 3 1 8 1 7 1 -20 1 93 1 -1 1 -1 2 93 2 + /# 1 # 1 # 1 # 1 # 1 # 1 # 2 # 2/ } 4 "SELECT z AS x, x AS z FROM d1 ORDER BY x" { - -20 1 -1 2 -1 1 3 1 7 1 8 1 93 2 93 1 + /-20 1 -1 # -1 # 3 1 7 1 8 1 93 # 93 #/ } } diff --git a/test/orderby1.test b/test/orderby1.test index 2837dfcab3..400659b475 100644 --- a/test/orderby1.test +++ b/test/orderby1.test @@ -114,7 +114,7 @@ do_test 1.4c { EXPLAIN QUERY PLAN SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn } -} {~/ORDER BY/} ;# ORDER BY optimized-out +} {/ORDER BY/} ;# separate sorting pass due to mixed DESC/ASC do_test 1.5a { @@ -130,9 +130,9 @@ do_test 1.5b { do_test 1.5c { db eval { EXPLAIN QUERY PLAN - SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn DESC } -} {~/ORDER BY/} ;# ORDER BY optimized-out +} {/ORDER BY/} ;# separate sorting pass due to mixed DESC/ASC do_test 1.6a { db eval { @@ -152,4 +152,275 @@ do_test 1.6c { } {~/ORDER BY/} ;# ORDER BY optimized-out +# Reconstruct the test data to use indices rather than integer primary keys. +# +do_test 2.0 { + db eval { + BEGIN; + DROP TABLE album; + DROP TABLE track; + CREATE TABLE album( + aid INT PRIMARY KEY, + title TEXT NOT NULL + ); + CREATE INDEX album_i1 ON album(title, aid); + CREATE TABLE track( + aid INTEGER NOT NULL REFERENCES album, + tn INTEGER NOT NULL, + name TEXT, + UNIQUE(aid, tn) + ); + INSERT INTO album VALUES(1, '1-one'), (2, '2-two'), (3, '3-three'); + INSERT INTO track VALUES + (1, 1, 'one-a'), + (2, 2, 'two-b'), + (3, 3, 'three-c'), + (1, 3, 'one-c'), + (2, 1, 'two-a'), + (3, 1, 'three-a'); + COMMIT; + } +} {} +do_test 2.1a { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn + } +} {one-a one-c two-a two-b three-a three-c} + +# Verify that the ORDER BY clause is optimized out +# +do_test 2.1b { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn + } +} {~/ORDER BY/} ;# ORDER BY optimized out + +# The same query with ORDER BY clause optimization disabled via + operators +# should give exactly the same answer. +# +do_test 2.2a { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn + } +} {one-a one-c two-a two-b three-a three-c} + +# The output is sorted manually in this case. +# +do_test 2.2b { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn + } +} {/ORDER BY/} ;# separate sorting pass due to "+" on ORDER BY terms + +# The same query with ORDER BY optimizations turned off via built-in test. +# +do_test 2.3a { + optimization_control db order-by-idx-join 0 + db cache flush + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn + } +} {one-a one-c two-a two-b three-a three-c} +do_test 2.3b { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn + } +} {/ORDER BY/} ;# separate sorting pass due to disabled optimization +optimization_control db all 1 +db cache flush + +# Reverse order sorts +# +do_test 2.4a { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn + } +} {three-a three-c two-a two-b one-a one-c} +do_test 2.4b { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY +title DESC, +tn + } +} {three-a three-c two-a two-b one-a one-c} ;# verify same order after sorting +do_test 2.4c { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn + } +} {/ORDER BY/} ;# separate sorting pass due to mixed DESC/ASC + + +do_test 2.5a { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn DESC + } +} {one-c one-a two-b two-a three-c three-a} +do_test 2.5b { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn DESC + } +} {one-c one-a two-b two-a three-c three-a} ;# verify same order after sorting +do_test 2.5c { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn DESC + } +} {/ORDER BY/} ;# separate sorting pass due to mixed ASC/DESC + +do_test 2.6a { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn DESC + } +} {three-c three-a two-b two-a one-c one-a} +do_test 2.6b { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY +title DESC, +tn DESC + } +} {three-c three-a two-b two-a one-c one-a} ;# verify same order after sorting +do_test 2.6c { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn DESC + } +} {~/ORDER BY/} ;# ORDER BY optimized-out + + +# Generate another test dataset, but this time using mixed ASC/DESC indices. +# +do_test 3.0 { + db eval { + BEGIN; + DROP TABLE album; + DROP TABLE track; + CREATE TABLE album( + aid INTEGER PRIMARY KEY, + title TEXT UNIQUE NOT NULL + ); + CREATE TABLE track( + tid INTEGER PRIMARY KEY, + aid INTEGER NOT NULL REFERENCES album, + tn INTEGER NOT NULL, + name TEXT, + UNIQUE(aid ASC, tn DESC) + ); + INSERT INTO album VALUES(1, '1-one'), (2, '2-two'), (3, '3-three'); + INSERT INTO track VALUES + (NULL, 1, 1, 'one-a'), + (NULL, 2, 2, 'two-b'), + (NULL, 3, 3, 'three-c'), + (NULL, 1, 3, 'one-c'), + (NULL, 2, 1, 'two-a'), + (NULL, 3, 1, 'three-a'); + COMMIT; + } +} {} +do_test 3.1a { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn DESC + } +} {one-c one-a two-b two-a three-c three-a} + +# Verify that the ORDER BY clause is optimized out +# +do_test 3.1b { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn DESC + } +} {~/ORDER BY/} ;# ORDER BY optimized out + +# The same query with ORDER BY clause optimization disabled via + operators +# should give exactly the same answer. +# +do_test 3.2a { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn DESC + } +} {one-c one-a two-b two-a three-c three-a} + +# The output is sorted manually in this case. +# +do_test 3.2b { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn DESC + } +} {/ORDER BY/} ;# separate sorting pass due to "+" on ORDER BY terms + +# The same query with ORDER BY optimizations turned off via built-in test. +# +do_test 3.3a { + optimization_control db order-by-idx-join 0 + db cache flush + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn DESC + } +} {one-c one-a two-b two-a three-c three-a} +do_test 3.3b { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn DESC + } +} {/ORDER BY/} ;# separate sorting pass due to disabled optimization +optimization_control db all 1 +db cache flush + +# Without the mixed ASC/DESC on ORDER BY +# +do_test 3.4a { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn + } +} {one-a one-c two-a two-b three-a three-c} +do_test 3.4b { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY +title, +tn + } +} {one-a one-c two-a two-b three-a three-c} ;# verify same order after sorting +do_test 3.4c { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY title, tn + } +} {/ORDER BY/} ;# separate sorting pass due to mismatched DESC/ASC + + +do_test 3.5a { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn DESC + } +} {three-c three-a two-b two-a one-c one-a} +do_test 3.5b { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY +title DESC, +tn DESC + } +} {three-c three-a two-b two-a one-c one-a} ;# verify same order after sorting +do_test 3.5c { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn DESC + } +} {/ORDER BY/} ;# separate sorting pass due to mismatched ASC/DESC + + +do_test 3.6a { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn + } +} {three-a three-c two-a two-b one-a one-c} +do_test 3.6b { + db eval { + SELECT name FROM album JOIN track USING (aid) ORDER BY +title DESC, +tn + } +} {three-a three-c two-a two-b one-a one-c} ;# verify same order after sorting +do_test 3.6c { + db eval { + EXPLAIN QUERY PLAN + SELECT name FROM album JOIN track USING (aid) ORDER BY title DESC, tn + } +} {~/ORDER BY/} ;# inverted ASC/DESC is optimized out + + finish_test diff --git a/test/tester.tcl b/test/tester.tcl index 74f1e2a1b3..d3fa607d32 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -538,10 +538,10 @@ proc do_test {name cmd expected} { } else { if {[regexp {^~?/.*/$} $expected]} { if {[string index $expected 0]=="~"} { - set re [string range $expected 2 end-1] + set re [string map {# {[-0-9.]+}} [string range $expected 2 end-1]] set ok [expr {![regexp $re $result]}] } else { - set re [string range $expected 1 end-1] + set re [string map {# {[-0-9.]+}} [string range $expected 1 end-1]] set ok [regexp $re $result] } } else { diff --git a/test/tkt-cbd054fa6b.test b/test/tkt-cbd054fa6b.test index 180acf56df..51e01991d4 100644 --- a/test/tkt-cbd054fa6b.test +++ b/test/tkt-cbd054fa6b.test @@ -50,7 +50,7 @@ do_test tkt-cbd05-1.3 { WHERE idx = 't1_x' GROUP BY tbl,idx } -} {t1 t1_x { A B C D E F G H I}} +} {/t1 t1_x .[ ABCDEFGHI]{10}./} do_test tkt-cbd05-2.1 { db eval { @@ -82,6 +82,6 @@ do_test tkt-cbd05-2.3 { WHERE idx = 't1_x' GROUP BY tbl,idx } -} {t1 t1_x { A B C D E F G H I}} +} {/t1 t1_x .[ ABCDEFGHI]{10}./} finish_test diff --git a/test/where.test b/test/where.test index 1dab38ce55..e57722c430 100644 --- a/test/where.test +++ b/test/where.test @@ -383,7 +383,7 @@ ifcapable subquery { count { SELECT * FROM t1 WHERE w IN (-1,1,2,3) order by 1; } - } {1 0 4 2 1 9 3 1 16 14} + } {1 0 4 2 1 9 3 1 16 13} do_test where-5.4 { count { SELECT * FROM t1 WHERE w+0 IN (-1,1,2,3) order by 1; @@ -1109,13 +1109,13 @@ do_test where-14.5 { cksort { SELECT x.a || '/' || y.a FROM t8 x, t8 y ORDER BY x.b, x.a||x.b } -} {4/4 4/1 1/4 1/1 sort} +} {/4/[14] 4/[14] 1/[14] 1/[14] sort/} do_test where-14.6 { # This test case changed from "nosort" to "sort". See ticket 2a5629202f. cksort { SELECT x.a || '/' || y.a FROM t8 x, t8 y ORDER BY x.b, x.a||x.b DESC } -} {4/4 4/1 1/4 1/1 sort} +} {/4/[14] 4/[14] 1/[14] 1/[14] sort/} do_test where-14.7 { cksort { SELECT x.a || '/' || y.a FROM t8 x, t8 y ORDER BY x.b, y.a||y.b From a9e3fc05f58a055ffdb67dca2a8215efb698a955 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Sep 2012 23:27:23 +0000 Subject: [PATCH 9/9] Fix some corner case behavior in the new ORDER BY optimization logic. Remove the SQLITE_OrderByIdx bit from the SQLITE_TESTCTRL_OPTIMIZATIONS mask, since enabling it caused many TH3 tests to fail when the NO_OPT configuration parameter was engaged, and since there really isn't any need to turn that optimization off. The SQLITE_OrderByIdxJoin bit remains. FossilOrigin-Name: 98b633717a1c9a08f6a1d00bc6bc891564ae7e9b --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/sqliteInt.h | 7 +++---- src/test1.c | 1 - src/where.c | 4 ++-- test/collate4.test | 2 +- test/where.test | 2 +- 7 files changed, 18 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 1b05b5bd9b..5624000edc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\stest\scases\san\sbug\sfixes\sfor\sthe\sORDER\sBY\soptimization\sof\sjoins.\s\sAll\nveryquick\stests\snow\spass. -D 2012-09-27T19:53:38.142 +C Fix\ssome\scorner\scase\sbehavior\sin\sthe\snew\sORDER\sBY\soptimization\slogic.\nRemove\sthe\sSQLITE_OrderByIdx\sbit\sfrom\sthe\s\nSQLITE_TESTCTRL_OPTIMIZATIONS\smask,\ssince\senabling\sit\scaused\smany\nTH3\stests\sto\sfail\swhen\sthe\sNO_OPT\sconfiguration\sparameter\swas\sengaged,\nand\ssince\sthere\sreally\sisn't\sany\sneed\sto\sturn\sthat\soptimization\soff.\nThe\sSQLITE_OrderByIdxJoin\sbit\sremains. +D 2012-09-27T23:27:23.679 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,12 +179,12 @@ F src/shell.c 8ee5a3cb502e2d574f97b43972e6c1e275e7bec7 F src/sqlite.h.in cbe846facaba903654b4136c97e7f57b3ac0bac7 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h 6ad55f4efbd1bb209711032e40b7c81c8855d634 +F src/sqliteInt.h c29395d6e68cfbcb2661787ae4820e5e256c916a F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c e4de2458b3ef38fdd0498bc4e5ea5367a241b0f3 -F src/test1.c 9000293bfdcd67de05bf91d257343392c4b194f4 +F src/test1.c 0354b555639c92d2a63c0ad4e74ed152ba47e604 F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c 5acb3e7bbd31f10ba39acad9ce6b399055337a9d F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c cd047c1223f6364f869d44b7d3941a687de7fcb6 +F src/where.c d836df3a2096c41c39e48ab5636f09f94ba02676 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -324,7 +324,7 @@ F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 F test/collate1.test e3eaa48c21e150814be1a7b852d2a8af24458d04 F test/collate2.test 04cebe4a033be319d6ddbb3bbc69464e01700b49 F test/collate3.test d28d2cfab2c3a3d4628ae4b2b7afc9965daa3b4c -F test/collate4.test 27adc324b58ec367bdd0bc6baab628af5129e9f1 +F test/collate4.test d37682293d3c32223dec2e6afdeaf9de18415248 F test/collate5.test 65d928034d30d2d263a80f6359f7549ee1598ec6 F test/collate6.test 8be65a182abaac8011a622131486dafb8076e907 F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 @@ -955,7 +955,7 @@ F test/walro.test a31deb621033442a76c3a61e44929250d06f81b1 F test/walshared.test 6dda2293880c300baf5d791c307f653094585761 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e -F test/where.test ea9659ff6e31681d0cdaf964747139cb10808200 +F test/where.test a6bfb5a29286811d798d326a8f1153a58c0fb2bd F test/where2.test 43d4becaf5a5df854e6c21d624a1cb84c6904554 F test/where3.test 667e75642102c97a00bf9b23d3cb267db321d006 F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2 @@ -1017,7 +1017,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P 75cda864ededb6dc0f84bd52ed3311753a58e351 -R 2ac4d1c44b199e10f1d07ecdc3868045 +P 0d573320057b0903a5589cabfb1b1ece1c57958e +R 346b48b15f539e2a05bf762bef6378fa U drh -Z 9df007fa9fa34b8089a8009c6ca9320b +Z 489ee88372e320839eda2ae45521e844 diff --git a/manifest.uuid b/manifest.uuid index f52b6aaffd..2b03b066ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d573320057b0903a5589cabfb1b1ece1c57958e \ No newline at end of file +98b633717a1c9a08f6a1d00bc6bc891564ae7e9b \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 4501c737f3..a03b7ac1e7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -827,7 +827,7 @@ struct sqlite3 { unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ - u16 dbOptFlags; /* Flags to enable/disable optimizations */ + u8 dbOptFlags; /* Flags to enable/disable optimizations */ u8 autoCommit; /* The auto-commit flag. */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ @@ -971,9 +971,8 @@ struct sqlite3 { #define SQLITE_IdxRealAsInt 0x0010 /* Store REAL as INT in indices */ #define SQLITE_DistinctOpt 0x0020 /* DISTINCT using indexes */ #define SQLITE_CoverIdxScan 0x0040 /* Covering index scans */ -#define SQLITE_OrderByIdx 0x0180 /* ORDER BY using indices */ -#define SQLITE_OrderByIdxJoin 0x0100 /* ORDER BY of joins via index */ -#define SQLITE_AllOpts 0x01ff /* All optimizations */ +#define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ +#define SQLITE_AllOpts 0x00ff /* All optimizations */ /* ** Macros for testing whether or not optimizations are enabled or disabled. diff --git a/src/test1.c b/src/test1.c index 45ca124bb2..3d2fb02034 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5941,7 +5941,6 @@ static int optimization_control( { "real-as-int", SQLITE_IdxRealAsInt }, { "distinct-opt", SQLITE_DistinctOpt }, { "cover-idx-scan", SQLITE_CoverIdxScan }, - { "order-by-idx", SQLITE_OrderByIdx }, { "order-by-idx-join",SQLITE_OrderByIdxJoin }, }; diff --git a/src/where.c b/src/where.c index 9bdcd536c4..7f386289c6 100644 --- a/src/where.c +++ b/src/where.c @@ -1647,7 +1647,6 @@ static int isSortingIndex( int seenRowid = 0; /* True if an ORDER BY rowid term is seen */ int nEqOneRow; /* Idx columns that ref unique values */ - if( OptimizationDisabled(db, SQLITE_OrderByIdx) ) return 0; if( p->i==0 ){ nPriorSat = 0; nEqOneRow = nEqCol; @@ -3154,6 +3153,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ } }else if( pTerm->eOperator & WO_ISNULL ){ wsFlags |= WHERE_COLUMN_NULL; + if( nEq==nOrdered ) nOrdered++; }else if( bSort && nEq==nOrdered && isOrderedTerm(p, pTerm, &bRev) ){ nOrdered++; } @@ -3216,7 +3216,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ bSort = 0; wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; } - if( bRev ) wsFlags |= WHERE_REVERSE; + if( bRev & 1 ) wsFlags |= WHERE_REVERSE; } /* If there is a DISTINCT qualifier and this index will scan rows in diff --git a/test/collate4.test b/test/collate4.test index 1cf8caf04c..6b3a1c7aee 100644 --- a/test/collate4.test +++ b/test/collate4.test @@ -389,7 +389,7 @@ ifcapable subquery { SELECT a FROM collate4t1 WHERE a IN (SELECT * FROM collate4t2) ORDER BY rowid } - } {a A 5} + } {a A 6} do_test collate4-2.1.8 { count { SELECT a FROM collate4t1 WHERE a IN ('z', 'a'); diff --git a/test/where.test b/test/where.test index e57722c430..83fec2cf60 100644 --- a/test/where.test +++ b/test/where.test @@ -383,7 +383,7 @@ ifcapable subquery { count { SELECT * FROM t1 WHERE w IN (-1,1,2,3) order by 1; } - } {1 0 4 2 1 9 3 1 16 13} + } {1 0 4 2 1 9 3 1 16 14} do_test where-5.4 { count { SELECT * FROM t1 WHERE w+0 IN (-1,1,2,3) order by 1;