paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improvements to the way the query planner handles sorting costs, so that

Browse Source 
very large sorting costs do not overwhelm the loop costs.

FossilOrigin-Name: bdaa6947371a60a31b6a13267b0ba6e46df2a8ce
This commit is contained in:
drh
2014-08-08 17:49:55 +00:00
parent 3c19bbea0a e2c278513d
commit ee5141de47
4 changed files with 120 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
manifest
+8 -7
View File
@@ -1,5 +1,5 @@
C The\sSQLITE_IOERR_BLOCKED\sextended\serror\scode\sis\snot\slonger\sused,\sso\sremove\nassert()\sstatements\sand\sdocumentation\sfor\sthat\serror\scode.\s\sAlso\smake\sother\ndocumentation\simprovements.
D 2014-08-08T15:38:11.174
C Improvements\sto\sthe\sway\sthe\squery\splanner\shandles\ssorting\scosts,\sso\sthat\nvery\slarge\ssorting\scosts\sdo\snot\soverwhelm\sthe\sloop\scosts.
D 2014-08-08T17:49:55.898
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -296,8 +296,8 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd
F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a
F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45
F src/where.c 4cd4b3622caa81b71ebce8fea8f7811ae052d664
F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6
F src/where.c ab20f9c24a422ee8900831b343c3d1e5e7aca87b
F src/whereInt.h 923820bee9726033a501a08d2fc69b9c1ee4feb3
F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6
@@ -1185,7 +1185,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 54f1df7b63166f14b2f0d5955c546820e5cbdec9
R 7baa54a57d0f22faa98f7b7348dc4b71
P 36b7c5cefcad6bad044806092593c84876fee8bc 43c59c85436dc8001c81f4aac7f5231b13d741cb
R 1f8265817308d29fd5a420f3d8d1525f
T +closed 43c59c85436dc8001c81f4aac7f5231b13d741cb
U drh
Z 8357d07b76c5300f8dd47c47cbfcc4a1
Z 45c189f6c18b0be836f726dcb23b5c12
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
36b7c5cefcad6bad044806092593c84876fee8bc
bdaa6947371a60a31b6a13267b0ba6e46df2a8ce
src/where.c
+110 -55
View File
@@ -5400,6 +5400,45 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){
}
#endif
/*
** Return the cost of sorting nRow rows, assuming that the keys have
** nOrderby columns and that the first nSorted columns are already in
** order.
*/
static LogEst whereSortingCost(
WhereInfo *pWInfo,
LogEst nRow,
int nOrderBy,
int nSorted
){
/* TUNING: Estimated cost of a full external sort, where N is
** the number of rows to sort is:
**
** cost = (3.0 * N * log(N)).
**
** Or, if the order-by clause has X terms but only the last Y
** terms are out of order, then block-sorting will reduce the
** sorting cost to:
**
** cost = (3.0 * N * log(N)) * (Y/X)
**
** The (Y/X) term is implemented using stack variable rScale
** below. */
LogEst rScale, rSortCost;
assert( nOrderBy>0 && 66==sqlite3LogEst(100) );
rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66;
rSortCost = nRow + estLog(nRow) + rScale + 16;
/* TUNING: The cost of implementing DISTINCT using a B-TREE is
** similar but with a larger constant of proportionality.
** Multiply by an additional factor of 3.0. */
if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){
rSortCost += 16;
}
return rSortCost;
}
/*
** Given the list of WhereLoop objects at pWInfo->pLoops, this routine
** attempts to find the lowest cost path that visits each WhereLoop
@@ -5421,10 +5460,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
int ii, jj; /* Loop counters */
int mxI = 0; /* Index of next entry to replace */
int nOrderBy; /* Number of ORDER BY clause terms */
LogEst rCost; /* Cost of a path */
LogEst nOut; /* Number of outputs */
LogEst mxCost = 0; /* Maximum cost of a set of paths */
LogEst mxOut = 0; /* nOut value for maximum cost path */
LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */
int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */
WherePath *aFrom; /* All nFrom paths at the previous level */
WherePath *aTo; /* The nTo best paths at the current level */
@@ -5432,7 +5469,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
WherePath *pTo; /* An element of aTo[] that we are working on */
WhereLoop *pWLoop; /* One of the WhereLoop objects */
WhereLoop **pX; /* Used to divy up the pSpace memory */
LogEst *aSortCost = 0; /* Sorting and partial sorting costs */
char *pSpace; /* Temporary memory used by this routine */
int nSpace; /* Bytes of space allocated at pSpace */
pParse = pWInfo->pParse;
db = pParse->db;
@@ -5444,9 +5483,21 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
assert( nLoop<=pWInfo->pTabList->nSrc );
WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d)\n", nRowEst));
/* Allocate and initialize space for aTo and aFrom */
ii = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2;
pSpace = sqlite3DbMallocRaw(db, ii);
/* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this
** case the purpose of this call is to estimate the number of rows returned
** by the overall query. Once this estimate has been obtained, the caller
** will invoke this function a second time, passing the estimate as the
** nRowEst parameter. */
if( pWInfo->pOrderBy==0 || nRowEst==0 ){
nOrderBy = 0;
}else{
nOrderBy = pWInfo->pOrderBy->nExpr;
}
/* Allocate and initialize space for aTo, aFrom and aSortCost[] */
nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2;
nSpace += sizeof(LogEst) * nOrderBy;
pSpace = sqlite3DbMallocRaw(db, nSpace);
if( pSpace==0 ) return SQLITE_NOMEM;
aTo = (WherePath*)pSpace;
aFrom = aTo+mxChoice;
@@ -5455,6 +5506,18 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){
pFrom->aLoop = pX;
}
if( nOrderBy ){
/* If there is an ORDER BY clause and it is not being ignored, set up
** space for the aSortCost[] array. Each element of the aSortCost array
** is either zero - meaning it has not yet been initialized - or the
** cost of sorting nRowEst rows of data where the first X terms of
** the ORDER BY clause are already in order, where X is the array
** index. */
aSortCost = (LogEst*)pX;
memset(aSortCost, 0, sizeof(LogEst) * nOrderBy);
}
assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] );
assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX );
/* Seed the search with a single WherePath containing zero WhereLoops.
**
@@ -5463,15 +5526,15 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
** rows, then do not use the automatic index. */
aFrom[0].nRow = MIN(pParse->nQueryLoop, 46); assert( 46==sqlite3LogEst(25) );
nFrom = 1;
/* Precompute the cost of sorting the final result set, if the caller
** to sqlite3WhereBegin() was concerned about sorting */
if( pWInfo->pOrderBy==0 || nRowEst==0 ){
aFrom[0].isOrdered = 0;
nOrderBy = 0;
}else{
aFrom[0].isOrdered = nLoop>0 ? -1 : 1;
nOrderBy = pWInfo->pOrderBy->nExpr;
assert( aFrom[0].isOrdered==0 );
if( nOrderBy ){
/* If nLoop is zero, then there are no FROM terms in the query. Since
** in this case the query may return a maximum of one row, the results
** are already in the requested order. Set isOrdered to nOrderBy to
** indicate this. Or, if nLoop is greater than zero, set isOrdered to
** -1, indicating that the result set may or may not be ordered,
** depending on the loops added to the current plan. */
aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy;
}
/* Compute successively longer WherePaths using the previous generation
@@ -5481,55 +5544,44 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
nTo = 0;
for(ii=0, pFrom=aFrom; ii<nFrom; ii++, pFrom++){
for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
Bitmask maskNew;
Bitmask revMask = 0;
i8 isOrdered = pFrom->isOrdered;
LogEst nOut; /* Rows visited by (pFrom+pWLoop) */
LogEst rCost; /* Cost of path (pFrom+pWLoop) */
LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */
i8 isOrdered = pFrom->isOrdered; /* isOrdered for (pFrom+pWLoop) */
Bitmask maskNew; /* Mask of src visited by (..) */
Bitmask revMask = 0; /* Mask of rev-order loops for (..) */
if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue;
if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue;
/* At this point, pWLoop is a candidate to be the next loop.
** Compute its cost */
rCost = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow);
rCost = sqlite3LogEstAdd(rCost, pFrom->rCost);
rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow);
rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted);
nOut = pFrom->nRow + pWLoop->nOut;
maskNew = pFrom->maskLoop | pWLoop->maskSelf;
if( isOrdered<0 ){
isOrdered = wherePathSatisfiesOrderBy(pWInfo,
pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags,
iLoop, pWLoop, &revMask);
if( isOrdered>=0 && isOrdered<nOrderBy ){
/* TUNING: Estimated cost of a full external sort, where N is
** the number of rows to sort is:
**
** cost = (3.0 * N * log(N)).
**
** Or, if the order-by clause has X terms but only the last Y
** terms are out of order, then block-sorting will reduce the
** sorting cost to:
**
** cost = (3.0 * N * log(N)) * (Y/X)
**
** The (Y/X) term is implemented using stack variable rScale
** below. */
LogEst rScale, rSortCost;
assert( nOrderBy>0 && 66==sqlite3LogEst(100) );
rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66;
rSortCost = nRowEst + estLog(nRowEst) + rScale + 16;
/* TUNING: The cost of implementing DISTINCT using a B-TREE is
** similar but with a larger constant of proportionality.
** Multiply by an additional factor of 3.0. */
if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){
rSortCost += 16;
}
WHERETRACE(0x002,
("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n",
rSortCost, (nOrderBy-isOrdered), nOrderBy, rCost,
sqlite3LogEstAdd(rCost,rSortCost)));
rCost = sqlite3LogEstAdd(rCost, rSortCost);
}
}else{
revMask = pFrom->revLoop;
}
if( isOrdered>=0 && isOrdered<nOrderBy ){
if( aSortCost[isOrdered]==0 ){
aSortCost[isOrdered] = whereSortingCost(
pWInfo, nRowEst, nOrderBy, isOrdered
);
}
rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]);
WHERETRACE(0x002,
("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n",
aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy,
rUnsorted, rCost));
}else{
rCost = rUnsorted;
}
/* Check to see if pWLoop should be added to the set of
** mxChoice best-so-far paths.
**
@@ -5552,7 +5604,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
if( jj>=nTo ){
/* None of the existing best-so-far paths match the candidate. */
if( nTo>=mxChoice
&& (rCost>mxCost || (rCost==mxCost && nOut>=mxOut))
&& (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted))
){
/* The current candidate is no better than any of the mxChoice
** paths currently in the best-so-far buffer. So discard
@@ -5624,17 +5676,20 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
pTo->revLoop = revMask;
pTo->nRow = nOut;
pTo->rCost = rCost;
pTo->rUnsorted = rUnsorted;
pTo->isOrdered = isOrdered;
memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop);
pTo->aLoop[iLoop] = pWLoop;
if( nTo>=mxChoice ){
mxI = 0;
mxCost = aTo[0].rCost;
mxOut = aTo[0].nRow;
mxUnsorted = aTo[0].nRow;
for(jj=1, pTo=&aTo[1]; jj<mxChoice; jj++, pTo++){
if( pTo->rCost>mxCost || (pTo->rCost==mxCost && pTo->nRow>mxOut) ){
if( pTo->rCost>mxCost
|| (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted)
){
mxCost = pTo->rCost;
mxOut = pTo->nRow;
mxUnsorted = pTo->rUnsorted;
mxI = jj;
}
}
src/whereInt.h
+1
View File
@@ -183,6 +183,7 @@ struct WherePath {
Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */
LogEst nRow; /* Estimated number of rows generated by this path */
LogEst rCost; /* Total cost of this path */
LogEst rUnsorted; /* Total cost of this path ignoring sorting costs */
i8 isOrdered; /* No. of ORDER BY terms satisfied. -1 for unknown */
WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */
};