diff --git a/manifest b/manifest index ae084022a8..839048226e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sindex_list\spragma,\smake\ssure\sthe\s"r"\scolumn\sis\sthe\ssame\son\soutput\nas\sit\swas\son\sinput\sin\sthe\ssqlite_stat1\stable. -D 2013-10-05T02:56:25.653 +C Begin\san\sexperimental\srefactoring\sto\sestimate\sthe\saverage\snumber\sof\sbytes\nin\stable\sand\sindex\srows\sand\sto\suse\sthat\sinformation\sin\squery\splanner.\nBegin\sby\srenaming\sWhereCost\sto\sLogEst\sand\smaking\sthat\stype\sand\sits\nconversion\sroutines\savailable\soutside\sof\swhere.c. +D 2013-10-05T18:16:02.764 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -221,7 +221,7 @@ F src/shell.c 5ee50ca3e35453bbd6ccdf1bdd0f6bbe9738e9fb F src/sqlite.h.in ec40aa958a270416fb04b4f72210357bf163d2c5 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h da0e92df9550e57107aa43eab63b29d040fe9c80 +F src/sqliteInt.h 75a6f6b26a91703672d62bd8df3bede2e6747e3b F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -275,7 +275,7 @@ F src/tokenize.c 70061085a51f2f4fc15ece94f32c03bcb78e63b2 F src/trigger.c ba0a883cd536b7dfdd4df3733001f5372a4299da F src/update.c f5182157f5d0d0a97bc5f5e3c9bdba0dfbe08f08 F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 -F src/util.c 7f3e35432d6888d1e770c488c35bd98970c44eec +F src/util.c 2fa6c821d28bbdbeec1b2a7b091a281c9ef8f918 F src/vacuum.c f313bc97123a4dd4bfd3f50a00c4d44c08a5b1b7 F src/vdbe.c 56e648f5ba9a91810caf216857adfed9039cd174 F src/vdbe.h 4f554b5627f26710c4c36d919110a3fc611ca5c4 @@ -290,7 +290,7 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 903e295f949590760aabef7203956f86588859f2 +F src/where.c 7b98d09e0e13179a8aeed38578434936369aa5c2 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1085,6 +1085,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/lemon.c 323e54ac86fb2393f9950219224e304620d2fb12 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc +F tool/logest.c 85b5d91cfb24b7c003614663e8de54b7022ef9bb w tool/wherecosttest.c F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 F tool/mkkeywordhash.c bb52064aa614e1426445e4b2b9b00eeecd23cc79 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -1119,9 +1120,11 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c6ac80ed8d5240dd30783d62d9f133e159809dec -R c9cf5017e54db6d924fbbb210ed0508a +P de78250ad2a6210dd4f03045248f7192d64427f2 +R 07e1c8eb04a1a514fab359ac6894024a +T *branch * row-size-est +T *sym-row-size-est * +T -sym-index-scan-rate * U drh -Z 19a867121626a8f2aac930a5af209c29 +Z 058604c78e1bcaeef450c73c9deab69d diff --git a/manifest.uuid b/manifest.uuid index 6435c49355..c5360dbb8d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -de78250ad2a6210dd4f03045248f7192d64427f2 \ No newline at end of file +66c4a251d61582b47d5cbe50cbca160a9209bd06 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5acfe38c5b..6ae36da46b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -470,6 +470,31 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ typedef u32 tRowcnt; /* 32-bit is the default */ #endif +/* +** Estimated quantities used for query planning are stored as 16-bit +** logarithms. For quantity X, the value stored is 10*log2(X). This +** gives a possible range of values of approximately 1.0e986 to 1e-986. +** But the allowed values are "grainy". Not every value is representable. +** For example, quantities 16 and 17 are both represented by a LogEst +** of 40. However, since LogEst quantatites are suppose to be estimates, +** not exact values, this imprecision is not a problem. +** +** "LogEst" is short for "Logarithimic Estimate". +** +** Examples: +** 1 -> 0 20 -> 43 10000 -> 132 +** 2 -> 10 25 -> 46 25000 -> 146 +** 3 -> 16 100 -> 66 1000000 -> 199 +** 4 -> 20 1000 -> 99 1048576 -> 200 +** 10 -> 33 1024 -> 100 4294967296 -> 320 +** +** The LogEst can be negative to indicate fractional values. +** Examples: +** +** 0.5 -> -10 0.1 -> -33 0.0625 -> -40 +*/ +typedef INT16_TYPE LogEst; + /* ** Macros to determine whether the machine is big or little endian, ** evaluated at runtime. @@ -1362,6 +1387,7 @@ struct Table { i16 iPKey; /* If not negative, use aCol[iPKey] as the primary key */ i16 nCol; /* Number of columns in this table */ u16 nRef; /* Number of pointers to this Table */ + LogEst szTabRow; /* Estimated size of each table row in bytes */ u8 tabFlags; /* Mask of TF_* values */ u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ #ifndef SQLITE_OMIT_ALTERTABLE @@ -1560,9 +1586,10 @@ struct Index { char **azColl; /* Array of collation sequence names for index */ Expr *pPartIdxWhere; /* WHERE clause for partial indices */ int tnum; /* DB Page containing root of this index */ + LogEst szIdxRow; /* Estimated average row size in bytes */ u16 nColumn; /* Number of columns in table used by this index */ u8 iScanRatio; /* Scan rate relative to the main table */ - unsigned onError:4; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ + u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ unsigned autoIndex:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ unsigned bUnordered:1; /* Use this index for == or IN queries only */ unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ @@ -2967,6 +2994,12 @@ int sqlite3Atoi(const char*); int sqlite3Utf16ByteLen(const void *pData, int nChar); int sqlite3Utf8CharLen(const char *pData, int nByte); u32 sqlite3Utf8Read(const u8**); +LogEst sqlite3LogEst(u64); +LogEst sqlite3LogEstAdd(LogEst,LogEst); +#ifndef SQLITE_OMIT_VIRTUALTABLE +LogEst sqlite3LogEstFromDouble(double); +#endif +u64 sqlite3LogEstToInt(LogEst); /* ** Routines to read and write variable-length integers. These used to diff --git a/src/util.c b/src/util.c index 7ccf2a1fd0..50ffd98650 100644 --- a/src/util.c +++ b/src/util.c @@ -1208,3 +1208,80 @@ void sqlite3FileSuffix3(const char *zBaseFilename, char *z){ } } #endif + +/* +** Find (an approximate) sum of two LogEst values. This computation is +** not a simple "+" operator because LogEst is stored as a logarithmic +** value. +** +*/ +LogEst sqlite3LogEstAdd(LogEst a, LogEst b){ + static const unsigned char x[] = { + 10, 10, /* 0,1 */ + 9, 9, /* 2,3 */ + 8, 8, /* 4,5 */ + 7, 7, 7, /* 6,7,8 */ + 6, 6, 6, /* 9,10,11 */ + 5, 5, 5, /* 12-14 */ + 4, 4, 4, 4, /* 15-18 */ + 3, 3, 3, 3, 3, 3, /* 19-24 */ + 2, 2, 2, 2, 2, 2, 2, /* 25-31 */ + }; + if( a>=b ){ + if( a>b+49 ) return a; + if( a>b+31 ) return a+1; + return a+x[a-b]; + }else{ + if( b>a+49 ) return b; + if( b>a+31 ) return b+1; + return b+x[b-a]; + } +} + +/* +** Convert an integer into a LogEst. In other words, compute a +** good approximatation for 10*log2(x). +*/ +LogEst sqlite3LogEst(u64 x){ + static LogEst a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; + LogEst y = 40; + if( x<8 ){ + if( x<2 ) return 0; + while( x<8 ){ y -= 10; x <<= 1; } + }else{ + while( x>255 ){ y += 40; x >>= 4; } + while( x>15 ){ y += 10; x >>= 1; } + } + return a[x&7] + y - 10; +} + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Convert a double into a LogEst +** In other words, compute an approximation for 10*log2(x). +*/ +LogEst sqlite3LogEstFromDouble(double x){ + u64 a; + LogEst e; + assert( sizeof(x)==8 && sizeof(a)==8 ); + if( x<=1 ) return 0; + if( x<=2000000000 ) return sqlite3LogEst((u64)x); + memcpy(&a, &x, 8); + e = (a>>52) - 1022; + return e*10; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/* +** Convert a LogEst into an integer. +*/ +u64 sqlite3LogEstToInt(LogEst x){ + u64 n; + if( x<10 ) return 1; + n = x%10; + x /= 10; + if( n>=5 ) n -= 2; + else if( n>=1 ) n -= 1; + if( x>=3 ) return (n+8)<<(x-3); + return (n+8)>>(3-x); +} diff --git a/src/where.c b/src/where.c index 1e0f6e2de6..96af9f382a 100644 --- a/src/where.c +++ b/src/where.c @@ -48,26 +48,6 @@ typedef struct WhereScan WhereScan; typedef struct WhereOrCost WhereOrCost; typedef struct WhereOrSet WhereOrSet; -/* -** Cost X is tracked as 10*log2(X) stored in a 16-bit integer. The -** maximum cost for ordinary tables is 64*(2**63) which becomes 6900. -** (Virtual tables can return a larger cost, but let's assume they do not.) -** So all costs can be stored in a 16-bit integer without risk -** of overflow. -** -** Costs are estimates, so no effort is made to compute 10*log2(X) exactly. -** Instead, a close estimate is used. Any value of X=1 is stored as 0. -** X=2 is 10. X=3 is 16. X=1000 is 99. etc. Negative values are allowed. -** A WhereCost of -10 means 0.5. WhereCost of -20 means 0.25. And so forth. -** -** The tool/wherecosttest.c source file implements a command-line program -** that will convert WhereCosts to integers, convert integers to WhereCosts -** and do addition and multiplication on WhereCost values. The wherecosttest -** command-line program is a useful utility to have around when working with -** this module. -*/ -typedef short int WhereCost; - /* ** This object contains information needed to implement a single nested ** loop in WHERE clause. @@ -132,9 +112,9 @@ struct WhereLoop { #endif u8 iTab; /* Position in FROM clause of table for this loop */ u8 iSortIdx; /* Sorting index number. 0==None */ - WhereCost rSetup; /* One-time setup cost (ex: create transient index) */ - WhereCost rRun; /* Cost of running each loop */ - WhereCost nOut; /* Estimated number of output rows */ + LogEst rSetup; /* One-time setup cost (ex: create transient index) */ + LogEst rRun; /* Cost of running each loop */ + LogEst nOut; /* Estimated number of output rows */ union { struct { /* Information for internal btree tables */ int nEq; /* Number of equality constraints */ @@ -164,8 +144,8 @@ struct WhereLoop { */ struct WhereOrCost { Bitmask prereq; /* Prerequisites */ - WhereCost rRun; /* Cost of running this subquery */ - WhereCost nOut; /* Number of outputs for this subquery */ + LogEst rRun; /* Cost of running this subquery */ + LogEst nOut; /* Number of outputs for this subquery */ }; /* The WhereOrSet object holds a set of possible WhereOrCosts that @@ -203,8 +183,8 @@ static int whereLoopResize(sqlite3*, WhereLoop*, int); struct WherePath { Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */ - WhereCost nRow; /* Estimated number of rows generated by this path */ - WhereCost rCost; /* Total cost of this path */ + LogEst nRow; /* Estimated number of rows generated by this path */ + LogEst rCost; /* Total cost of this path */ u8 isOrdered; /* True if this path satisfies ORDER BY */ u8 isOrderedValid; /* True if the isOrdered field is valid */ WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ @@ -270,7 +250,7 @@ struct WhereTerm { WhereOrInfo *pOrInfo; /* Extra information if (eOperator & WO_OR)!=0 */ WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */ } u; - WhereCost truthProb; /* Probability of truth for this expression */ + LogEst truthProb; /* Probability of truth for this expression */ u16 eOperator; /* A WO_xx value describing */ u8 wtFlags; /* TERM_xxx bit flags. See below */ u8 nChild; /* Number of children that must disable us */ @@ -418,7 +398,7 @@ struct WhereInfo { ExprList *pResultSet; /* Result set. DISTINCT operates on these */ WhereLoop *pLoops; /* List of all WhereLoop objects */ Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ - WhereCost nRowOut; /* Estimated number of output rows */ + LogEst nRowOut; /* Estimated number of output rows */ u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ u8 bOBSat; /* ORDER BY satisfied by indices */ u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ @@ -478,26 +458,11 @@ struct WhereInfo { #define WHERE_MULTI_OR 0x00002000 /* OR using multiple indices */ #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ - -/* Convert a WhereCost value (10 times log2(X)) into its integer value X. -** A rough approximation is used. The value returned is not exact. -*/ -static u64 whereCostToInt(WhereCost x){ - u64 n; - if( x<10 ) return 1; - n = x%10; - x /= 10; - if( n>=5 ) n -= 2; - else if( n>=1 ) n -= 1; - if( x>=3 ) return (n+8)<<(x-3); - return (n+8)>>(3-x); -} - /* ** Return the estimated number of output rows from a WHERE clause */ u64 sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ - return whereCostToInt(pWInfo->nRowOut); + return sqlite3LogEstToInt(pWInfo->nRowOut); } /* @@ -559,8 +524,8 @@ static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){ static int whereOrInsert( WhereOrSet *pSet, /* The WhereOrSet to be updated */ Bitmask prereq, /* Prerequisites of the new entry */ - WhereCost rRun, /* Run-cost of the new entry */ - WhereCost nOut /* Number of outputs for the new entry */ + LogEst rRun, /* Run-cost of the new entry */ + LogEst nOut /* Number of outputs for the new entry */ ){ u16 i; WhereOrCost *p; @@ -645,9 +610,6 @@ static void whereClauseClear(WhereClause *pWC){ } } -/* Forward declaration */ -static WhereCost whereCost(tRowcnt x); - /* ** Add a single new WhereTerm entry to the WhereClause object pWC. ** The new WhereTerm object is constructed from Expr p and with wtFlags. @@ -690,7 +652,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ } pTerm = &pWC->a[idx = pWC->nTerm++]; if( p && ExprHasProperty(p, EP_Unlikely) ){ - pTerm->truthProb = whereCost(p->iTable) - 99; + pTerm->truthProb = sqlite3LogEst(p->iTable) - 99; }else{ pTerm->truthProb = -1; } @@ -1954,75 +1916,12 @@ static int isDistinctRedundant( return 0; } -/* -** Find (an approximate) sum of two WhereCosts. This computation is -** not a simple "+" operator because WhereCost is stored as a logarithmic -** value. -** -*/ -static WhereCost whereCostAdd(WhereCost a, WhereCost b){ - static const unsigned char x[] = { - 10, 10, /* 0,1 */ - 9, 9, /* 2,3 */ - 8, 8, /* 4,5 */ - 7, 7, 7, /* 6,7,8 */ - 6, 6, 6, /* 9,10,11 */ - 5, 5, 5, /* 12-14 */ - 4, 4, 4, 4, /* 15-18 */ - 3, 3, 3, 3, 3, 3, /* 19-24 */ - 2, 2, 2, 2, 2, 2, 2, /* 25-31 */ - }; - if( a>=b ){ - if( a>b+49 ) return a; - if( a>b+31 ) return a+1; - return a+x[a-b]; - }else{ - if( b>a+49 ) return b; - if( b>a+31 ) return b+1; - return b+x[b-a]; - } -} - -/* -** Convert an integer into a WhereCost. In other words, compute a -** good approximatation for 10*log2(x). -*/ -static WhereCost whereCost(tRowcnt x){ - static WhereCost a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; - WhereCost y = 40; - if( x<8 ){ - if( x<2 ) return 0; - while( x<8 ){ y -= 10; x <<= 1; } - }else{ - while( x>255 ){ y += 40; x >>= 4; } - while( x>15 ){ y += 10; x >>= 1; } - } - return a[x&7] + y - 10; -} - -#ifndef SQLITE_OMIT_VIRTUALTABLE -/* -** Convert a double (as received from xBestIndex of a virtual table) -** into a WhereCost. In other words, compute an approximation for -** 10*log2(x). -*/ -static WhereCost whereCostFromDouble(double x){ - u64 a; - WhereCost e; - assert( sizeof(x)==8 && sizeof(a)==8 ); - if( x<=1 ) return 0; - if( x<=2000000000 ) return whereCost((tRowcnt)x); - memcpy(&a, &x, 8); - e = (a>>52) - 1022; - return e*10; -} -#endif /* SQLITE_OMIT_VIRTUALTABLE */ /* ** Estimate the logarithm of the input value to base 2. */ -static WhereCost estLog(WhereCost N){ - WhereCost x = whereCost(N); +static LogEst estLog(LogEst N){ + LogEst x = sqlite3LogEst(N); return x>33 ? x - 33 : 0; } @@ -2535,7 +2434,7 @@ static void whereKeyStats( ** ** then nEq is set to 0. ** -** When this function is called, *pnOut is set to the whereCost() of the +** When this function is called, *pnOut is set to the sqlite3LogEst() of the ** number of rows that the index scan is expected to visit without ** considering the range constraints. If nEq is 0, this is the number of ** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) @@ -2551,11 +2450,11 @@ static int whereRangeScanEst( WhereLoopBuilder *pBuilder, WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ - WhereCost *pnOut /* IN/OUT: Number of rows visited */ + LogEst *pnOut /* IN/OUT: Number of rows visited */ ){ int rc = SQLITE_OK; int nOut = (int)*pnOut; - WhereCost nNew; + LogEst nNew; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 Index *p = pBuilder->pNew->u.btree.pIndex; @@ -2640,14 +2539,14 @@ static int whereRangeScanEst( pBuilder->pRec = pRec; if( rc==SQLITE_OK ){ if( iUpper>iLower ){ - nNew = whereCost(iUpper - iLower); + nNew = sqlite3LogEst(iUpper - iLower); }else{ - nNew = 10; assert( 10==whereCost(2) ); + nNew = 10; assert( 10==sqlite3LogEst(2) ); } if( nNewwtFlags & TERM_VNULL)==0 ){ - nNew -= 20; assert( 20==whereCost(4) ); + nNew -= 20; assert( 20==sqlite3LogEst(4) ); nOut--; } if( pUpper ){ - nNew -= 20; assert( 20==whereCost(4) ); + nNew -= 20; assert( 20==sqlite3LogEst(4) ); nOut--; } if( nNew<10 ) nNew = 10; if( nNewpWInfo; /* WHERE analyse context */ Parse *pParse = pWInfo->pParse; /* Parsing context */ @@ -4328,11 +4227,11 @@ static int whereLoopAddBtreeIndex( u16 saved_nLTerm; /* Original value of pNew->nLTerm */ int saved_nEq; /* Original value of pNew->u.btree.nEq */ u32 saved_wsFlags; /* Original value of pNew->wsFlags */ - WhereCost saved_nOut; /* Original value of pNew->nOut */ + LogEst saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ - WhereCost nRowEst; /* Estimated index selectivity */ - WhereCost rLogSize; /* Logarithm of table size */ + LogEst nRowEst; /* Estimated index selectivity */ + LogEst rLogSize; /* Logarithm of table size */ WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ pNew = pBuilder->pNew; @@ -4352,7 +4251,7 @@ static int whereLoopAddBtreeIndex( assert( pNew->u.btree.nEq<=pProbe->nColumn ); if( pNew->u.btree.nEq < pProbe->nColumn ){ iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - nRowEst = whereCost(pProbe->aiRowEst[pNew->u.btree.nEq+1]); + nRowEst = sqlite3LogEst(pProbe->aiRowEst[pNew->u.btree.nEq+1]); if( nRowEst==0 && pProbe->onError==OE_None ) nRowEst = 1; }else{ iCol = -1; @@ -4366,7 +4265,7 @@ static int whereLoopAddBtreeIndex( saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; pNew->rSetup = 0; - rLogSize = estLog(whereCost(pProbe->aiRowEst[0])); + rLogSize = estLog(sqlite3LogEst(pProbe->aiRowEst[0])); for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -4393,10 +4292,10 @@ static int whereLoopAddBtreeIndex( pNew->wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ - nIn = 46; assert( 46==whereCost(25) ); + nIn = 46; assert( 46==sqlite3LogEst(25) ); }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ /* "x IN (value, value, ...)" */ - nIn = whereCost(pExpr->x.pList->nExpr); + nIn = sqlite3LogEst(pExpr->x.pList->nExpr); } pNew->rRun += nIn; pNew->u.btree.nEq++; @@ -4418,7 +4317,7 @@ static int whereLoopAddBtreeIndex( pNew->wsFlags |= WHERE_COLUMN_NULL; pNew->u.btree.nEq++; /* TUNING: IS NULL selects 2 rows */ - nIn = 10; assert( 10==whereCost(2) ); + nIn = 10; assert( 10==sqlite3LogEst(2) ); pNew->nOut = nRowEst + nInMul + nIn; }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ testcase( pTerm->eOperator & WO_GT ); @@ -4458,7 +4357,7 @@ static int whereLoopAddBtreeIndex( } assert( nOut==0 || rc==SQLITE_OK ); if( nOut ){ - nOut = whereCost(nOut); + nOut = sqlite3LogEst(nOut); pNew->nOut = MIN(nOut, saved_nOut); } } @@ -4466,10 +4365,10 @@ static int whereLoopAddBtreeIndex( if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ /* Each row involves a step of the index, then a binary search of ** the main table */ - pNew->rRun = whereCostAdd(pNew->rRun, rLogSize>27 ? rLogSize-17 : 10); + pNew->rRun = sqlite3LogEstAdd(pNew->rRun,rLogSize>27 ? rLogSize-17 : 10); } /* Step cost for each output row */ - pNew->rRun = whereCostAdd(pNew->rRun, pNew->nOut); + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut); whereLoopOutputAdjust(pBuilder->pWC, pNew, pSrc->iCursor); rc = whereLoopInsert(pBuilder, pNew); if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 @@ -4569,8 +4468,8 @@ static int whereLoopAddBtree( int rc = SQLITE_OK; /* Return code */ int iSortIdx = 1; /* Index number */ int b; /* A boolean value */ - WhereCost rSize; /* number of rows in the table */ - WhereCost rLogSize; /* Logarithm of the number of rows in the table */ + LogEst rSize; /* number of rows in the table */ + LogEst rLogSize; /* Logarithm of the number of rows in the table */ WhereClause *pWC; /* The parsed WHERE clause */ pNew = pBuilder->pNew; @@ -4605,7 +4504,7 @@ static int whereLoopAddBtree( } pProbe = &sPk; } - rSize = whereCost(pSrc->pTab->nRowEst); + rSize = sqlite3LogEst(pSrc->pTab->nRowEst); rLogSize = estLog(rSize); #ifndef SQLITE_OMIT_AUTOMATIC_INDEX @@ -4630,13 +4529,13 @@ static int whereLoopAddBtree( /* TUNING: One-time cost for computing the automatic index is ** approximately 7*N*log2(N) where N is the number of rows in ** the table being indexed. */ - pNew->rSetup = rLogSize + rSize + 28; assert( 28==whereCost(7) ); + pNew->rSetup = rLogSize + rSize + 28; assert( 28==sqlite3LogEst(7) ); /* TUNING: Each index lookup yields 20 rows in the table. This ** is more than the usual guess of 10 rows, since we have no way ** of knowning how selective the index will ultimately be. It would ** not be unreasonable to make this value much larger. */ - pNew->nOut = 43; assert( 43==whereCost(20) ); - pNew->rRun = whereCostAdd(rLogSize,pNew->nOut); + pNew->nOut = 43; assert( 43==sqlite3LogEst(20) ); + pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut); pNew->wsFlags = WHERE_AUTO_INDEX; pNew->prereq = mExtra | pTerm->prereqRight; rc = whereLoopInsert(pBuilder, pNew); @@ -4669,9 +4568,9 @@ static int whereLoopAddBtree( /* Full table scan */ pNew->iSortIdx = b ? iSortIdx : 0; /* TUNING: Cost of full table scan is 3*(N + log2(N)). - ** + The extra 4 factor is to encourage the use of indexed lookups + ** + The extra 3 factor is to encourage the use of indexed lookups ** over full scans. */ - pNew->rRun = whereCostAdd(rSize,rLogSize) + 16; + pNew->rRun = sqlite3LogEstAdd(rSize,rLogSize) + 16; whereLoopOutputAdjust(pWC, pNew, pSrc->iCursor); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -4693,11 +4592,11 @@ static int whereLoopAddBtree( pNew->iSortIdx = b ? iSortIdx : 0; if( m==0 ){ /* TUNING: Cost of a covering index scan is K*(N + log2(N)). - ** + The extra factor K of between 1.0 and 3.0 is added to - ** encourage the use of indexed lookups. The value of K - ** depends on the iScanRatio value for the index. + ** + The extra factor K of between 1.1 (iScanRatio between 0 + ** and 9) and 2.8 (iScanRatio between 126 and 127) is added + ** to encourage the use of indexed lookups. */ - pNew->rRun = whereCostAdd(rSize,rLogSize) + pProbe->iScanRatio/9 + 1; + pNew->rRun = sqlite3LogEstAdd(rSize,rLogSize) + pProbe->iScanRatio/9 + 1; }else{ assert( b!=0 ); /* TUNING: Cost of scanning a non-covering index is (N+1)*log2(N) @@ -4875,9 +4774,9 @@ static int whereLoopAddVirtual( pNew->u.vtab.isOrdered = (u8)((pIdxInfo->nOrderBy!=0) && pIdxInfo->orderByConsumed); pNew->rSetup = 0; - pNew->rRun = whereCostFromDouble(pIdxInfo->estimatedCost); + pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); /* TUNING: Every virtual table query returns 25 rows */ - pNew->nOut = 46; assert( 46==whereCost(25) ); + pNew->nOut = 46; assert( 46==sqlite3LogEst(25) ); whereLoopInsert(pBuilder, pNew); if( pNew->u.vtab.needFree ){ sqlite3_free(pNew->u.vtab.idxStr); @@ -4967,8 +4866,8 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ for(i=0; inQueryLoop, 46); assert( 46==whereCost(25) ); + aFrom[0].nRow = MIN(pParse->nQueryLoop, 46); assert( 46==sqlite3LogEst(25) ); nFrom = 1; /* Precompute the cost of sorting the final result set, if the caller @@ -5385,8 +5284,8 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; /* At this point, pWLoop is a candidate to be the next loop. ** Compute its cost */ - rCost = whereCostAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); - rCost = whereCostAdd(rCost, pFrom->rCost); + rCost = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); + rCost = sqlite3LogEstAdd(rCost, pFrom->rCost); nOut = pFrom->nRow + pWLoop->nOut; maskNew = pFrom->maskLoop | pWLoop->maskSelf; if( !isOrderedValid ){ @@ -5400,7 +5299,7 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ case 0: /* No. pFrom+pWLoop will require a separate sort */ isOrdered = 0; isOrderedValid = 1; - rCost = whereCostAdd(rCost, rSortCost); + rCost = sqlite3LogEstAdd(rCost, rSortCost); break; default: /* Cannot tell yet. Try again on the next iteration */ break; @@ -5607,7 +5506,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->nLTerm = 1; pLoop->u.btree.nEq = 1; /* TUNING: Cost of a rowid lookup is 10 */ - pLoop->rRun = 33; /* 33==whereCost(10) */ + pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */ }else{ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ assert( pLoop->aLTermSpace==pLoop->aLTerm ); @@ -5630,12 +5529,12 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->u.btree.nEq = j; pLoop->u.btree.pIndex = pIdx; /* TUNING: Cost of a unique index lookup is 15 */ - pLoop->rRun = 39; /* 39==whereCost(15) */ + pLoop->rRun = 39; /* 39==sqlite3LogEst(15) */ break; } } if( pLoop->wsFlags ){ - pLoop->nOut = (WhereCost)1; + pLoop->nOut = (LogEst)1; pWInfo->a[0].pWLoop = pLoop; pLoop->maskSelf = getMask(&pWInfo->sMaskSet, iCur); pWInfo->a[0].iTabCur = iCur; diff --git a/tool/wherecosttest.c b/tool/logest.c similarity index 54% rename from tool/wherecosttest.c rename to tool/logest.c index 39d4a7a368..ae793cf547 100644 --- a/tool/wherecosttest.c +++ b/tool/logest.c @@ -10,39 +10,42 @@ ** ************************************************************************* ** This file contains a simple command-line utility for converting from -** integers and WhereCost values and back again and for doing simple -** arithmetic operations (multiple and add) on WhereCost values. +** integers and LogEst values and back again and for doing simple +** arithmetic operations (multiple and add) on LogEst values. ** ** Usage: ** -** ./wherecosttest ARGS +** ./LogEst ARGS ** ** Arguments: ** ** 'x' Multiple the top two elements of the stack ** '+' Add the top two elements of the stack -** NUM Convert NUM from integer to WhereCost and push onto the stack -** ^NUM Interpret NUM as a WhereCost and push onto stack. +** NUM Convert NUM from integer to LogEst and push onto the stack +** ^NUM Interpret NUM as a LogEst and push onto stack. ** ** Examples: ** -** To convert 123 from WhereCost to integer: +** To convert 123 from LogEst to integer: ** -** ./wherecosttest ^123 +** ./LogEst ^123 ** -** To convert 123456 from integer to WhereCost: +** To convert 123456 from integer to LogEst: ** -** ./wherecosttest 123456 +** ./LogEst 123456 ** */ #include #include #include +#include +#include +#include "sqlite3.h" -typedef unsigned short int WhereCost; /* 10 times log2() */ +typedef short int LogEst; /* 10 times log2() */ -WhereCost whereCostMultiply(WhereCost a, WhereCost b){ return a+b; } -WhereCost whereCostAdd(WhereCost a, WhereCost b){ +LogEst logEstMultiply(LogEst a, LogEst b){ return a+b; } +LogEst logEstAdd(LogEst a, LogEst b){ static const unsigned char x[] = { 10, 10, /* 0,1 */ 9, 9, /* 2,3 */ @@ -54,14 +57,14 @@ WhereCost whereCostAdd(WhereCost a, WhereCost b){ 3, 3, 3, 3, 3, 3, /* 19-24 */ 2, 2, 2, 2, 2, 2, 2, /* 25-31 */ }; - if( ab+49 ) return a; if( a>b+31 ) return a+1; return a+x[a-b]; } -WhereCost whereCostFromInteger(int x){ - static WhereCost a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; - WhereCost y = 40; +LogEst logEstFromInteger(sqlite3_uint64 x){ + static LogEst a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; + LogEst y = 40; if( x<8 ){ if( x<2 ) return 0; while( x<8 ){ y -= 10; x <<= 1; } @@ -71,7 +74,7 @@ WhereCost whereCostFromInteger(int x){ } return a[x&7] + y - 10; } -static unsigned long int whereCostToInt(WhereCost x){ +static unsigned long int logEstToInt(LogEst x){ unsigned long int n; if( x<10 ) return 1; n = x%10; @@ -81,31 +84,56 @@ static unsigned long int whereCostToInt(WhereCost x){ if( x>=3 ) return (n+8)<<(x-3); return (n+8)>>(3-x); } +static LogEst logEstFromDouble(double x){ + sqlite3_uint64 a; + LogEst e; + assert( sizeof(x)==8 && sizeof(a)==8 ); + if( x<=0 ) return -32768; + if( x<1 ) return -logEstFromDouble(1/x); + if( x<=2000000000 ) return logEstFromInteger((sqlite3_uint64)x); + memcpy(&a, &x, 8); + e = (a>>52) - 1022; + return e*10; +} + +int isFloat(const char *z){ + while( z[0] ){ + if( z[0]=='.' || z[0]=='E' || z[0]=='e' ) return 1; + z++; + } + return 0; +} int main(int argc, char **argv){ int i; int n = 0; - WhereCost a[100]; + LogEst a[100]; for(i=1; i=2 ){ - a[n-2] = whereCostAdd(a[n-2],a[n-1]); + a[n-2] = logEstAdd(a[n-2],a[n-1]); n--; } }else if( z[0]=='x' ){ if( n>=2 ){ - a[n-2] = whereCostMultiply(a[n-2],a[n-1]); + a[n-2] = logEstMultiply(a[n-2],a[n-1]); n--; } }else if( z[0]=='^' ){ a[n++] = atoi(z+1); + }else if( isFloat(z) ){ + a[n++] = logEstFromDouble(atof(z)); }else{ - a[n++] = whereCostFromInteger(atoi(z)); + a[n++] = logEstFromInteger(atoi(z)); } } for(i=n-1; i>=0; i--){ - printf("%d (%lu)\n", a[i], whereCostToInt(a[i])); + if( a[i]<0 ){ + printf("%d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i])); + }else{ + printf("%d (%lu)\n", a[i], logEstToInt(a[i])); + } } return 0; }