From aa9933c11529cabf659615d9c94277846dacbd15 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 24 Apr 2014 20:04:49 +0000 Subject: [PATCH 01/19] Changes to the way the planner calculates the costs of various table and index scans. Some test cases still failing. FossilOrigin-Name: c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 --- manifest | 25 +++++----- manifest.uuid | 2 +- src/where.c | 109 ++++++++++++++++++++++--------------------- test/analyze9.test | 8 ++-- test/autoindex1.test | 2 + test/where3.test | 3 +- test/whereG.test | 32 +++++++++++++ 7 files changed, 111 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index 7f4e9189fd..2b621bd738 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Comment\stweaks\son\sthe\stest\scase\sfor\sthe\s[b75a9ca6b0]\sbug\sfix. -D 2014-04-21T13:36:54.639 +C Changes\sto\sthe\sway\sthe\splanner\scalculates\sthe\scosts\sof\svarious\stable\sand\sindex\sscans.\sSome\stest\scases\sstill\sfailing. +D 2014-04-24T20:04:49.939 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 3b127bdc24b7aa84ffa69729170be11555cd7733 +F src/where.c c12bc20cd649bcae39de3e452bfc1a3f164454ee F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -311,7 +311,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test e072a5172d55afcba98d6ca6a219ce8878c2f5c9 +F test/analyze9.test e219daa58fd8677c6a43d771798cf37d68f51d3e F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b @@ -329,7 +329,7 @@ F test/auth.test 5bdf154eb28c0e4bbc0473f335858c0d96171768 F test/auth2.test c3b415b76c033bedb81292118fb7c01f5f10cbcd F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 -F test/autoindex1.test d4dfe14001dfcb74cfbd7107f45a79fc1ab6183e +F test/autoindex1.test 762ff3f8e25d852aae55c6462ca166a80c0cde61 F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 @@ -1079,7 +1079,7 @@ F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e F test/where.test 28b64e93428961b07b0d486778d63fd672948f6b F test/where2.test 455a2eb2666e66c1e84e2cb5815173a85e6237db -F test/where3.test d28c51f257e60be30f74308fa385ceeddfb54a6e +F test/where3.test 1ad55ba900bd7747f98b6082e65bd3e442c5004e F test/where4.test d8420ceeb8323a41ceff1f1841fc528e824e1ecf F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b @@ -1093,7 +1093,7 @@ F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a F test/whereD.test fd9120e262f9da3c45940f52aefeef4d15b904e5 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 -F test/whereG.test 2533b72ed4a31fd1687230a499b557b911525344 +F test/whereG.test 8189fedf3b98ab581bb70f830175e403a0ef1722 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c @@ -1161,7 +1161,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P de9a490f594183f337a2ec9e0f87792eac83548b -R ce888b84132e0cad3bcca115a32951d3 -U drh -Z cf9f241149456ab1fa24984e95a412d2 +P 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb +R 6cc54703275bf2ed6708c34ae52cf7ea +T *branch * experimental-costs +T *sym-experimental-costs * +T -sym-trunk * +U dan +Z 868aa60f36dda291ae018583501645e5 diff --git a/manifest.uuid b/manifest.uuid index a2761a597b..4c2b675cff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -65d2544af9adc1e2f1d193e57f8be0422fb0d5eb \ No newline at end of file +c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 9bde27e52f..6a75840525 100644 --- a/src/where.c +++ b/src/where.c @@ -227,7 +227,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ if( p && ExprHasProperty(p, EP_Unlikely) ){ pTerm->truthProb = sqlite3LogEst(p->iTable) - 99; }else{ - pTerm->truthProb = -1; + pTerm->truthProb = 1; } pTerm->pExpr = sqlite3ExprSkipCollate(p); pTerm->wtFlags = wtFlags; @@ -1975,6 +1975,31 @@ static void whereKeyStats( } #endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +/* +** If it is not NULL, pTerm is a term that provides an upper or lower +** bound on a range scan. Without considering pTerm, it is estimated +** that the scan will visit nNew rows. This function returns the number +** estimated to be visited after taking pTerm into account. +** +** If the user explicitly specified a likelihood() value for this term, +** then the return value is the likelihood multiplied by the number of +** input rows. Otherwise, this function assumes that an "IS NOT NULL" term +** has a likelihood of 0.50, and any other term a likelihood of 0.25. +*/ +static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ + LogEst nRet = nNew; + if( pTerm ){ + if( pTerm->truthProb<=0 ){ + nRet += pTerm->truthProb; + }else if( pTerm->wtFlags & TERM_VNULL ){ + nRet -= 10; assert( 10==sqlite3LogEst(2) ); + }else{ + nRet -= 20; assert( 20==sqlite3LogEst(4) ); + } + } + return nRet; +} + /* ** This function is used to estimate the number of rows that will be visited ** by scanning an index for a range of values. The range may have an upper @@ -2127,17 +2152,9 @@ static int whereRangeScanEst( UNUSED_PARAMETER(pBuilder); #endif assert( pLower || pUpper ); - /* TUNING: Each inequality constraint reduces the search space 4-fold. - ** A BETWEEN operator, therefore, reduces the search space 16-fold */ - nNew = nOut; - if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ){ - nNew -= 20; assert( 20==sqlite3LogEst(4) ); - nOut--; - } - if( pUpper ){ - nNew -= 20; assert( 20==sqlite3LogEst(4) ); - nOut--; - } + nNew = whereRangeAdjust(pLower, nOut); + nNew = whereRangeAdjust(pUpper, nNew); + nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; if( nNewnOut = (LogEst)nOut; @@ -3987,7 +4004,9 @@ static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ if( pX==pTerm ) break; if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; } - if( j<0 ) pLoop->nOut += pTerm->truthProb; + if( j<0 ){ + pLoop->nOut += (pTerm->truthProb<=0 ? pTerm->truthProb : -1); + } } } @@ -4081,6 +4100,7 @@ static int whereLoopAddBtreeIndex( pNew->nOut = saved_nOut; } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ + LogEst rCostIdx; int nIn = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nRecValid = pBuilder->nRecValid; @@ -4154,7 +4174,8 @@ static int whereLoopAddBtreeIndex( pNew->aLTerm[pNew->nLTerm-2] : 0; } if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ - /* Adjust nOut and rRun for STAT3 range values */ + /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 + ** data, using some other estimate. */ assert( pNew->nOut==saved_nOut ); whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); } @@ -4181,13 +4202,16 @@ static int whereLoopAddBtreeIndex( } } #endif + /* Set rCostIdx to the cost of visiting selected rows in index. Add + ** it to pNew->rRun, which is currently set to the cost of the index + ** seek only. Then, if this is a non-covering index, add the cost of + ** visiting the rows in the main table. */ + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rCostIdx); if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ - /* Each row involves a step of the index, then a binary search of - ** the main table */ - pNew->rRun = sqlite3LogEstAdd(pNew->rRun,rLogSize>27 ? rLogSize-17 : 10); + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } - /* Step cost for each output row */ - pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut); + whereLoopOutputAdjust(pBuilder->pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 @@ -4319,6 +4343,7 @@ static int whereLoopAddBtree( sPk.aiRowEst = aiRowEstPk; sPk.onError = OE_Replace; sPk.pTable = pTab; + sPk.szIdxRow = pTab->szTabRow; aiRowEstPk[0] = pTab->nRowEst; aiRowEstPk[1] = 1; pFirst = pSrc->pTab->pIndex; @@ -4396,10 +4421,8 @@ static int whereLoopAddBtree( /* Full table scan */ pNew->iSortIdx = b ? iSortIdx : 0; - /* TUNING: Cost of full table scan is 3*(N + log2(N)). - ** + The extra 3 factor is to encourage the use of indexed lookups - ** over full scans. FIXME */ - pNew->rRun = sqlite3LogEstAdd(rSize,rLogSize) + 16; + /* TUNING: Cost of full table scan is (N*3.0). */ + pNew->rRun = rSize + 16; whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -4426,35 +4449,16 @@ static int whereLoopAddBtree( ) ){ pNew->iSortIdx = b ? iSortIdx : 0; - /* TUNING: The base cost of an index scan is N + log2(N). - ** The log2(N) is for the initial seek to the beginning and the N - ** is for the scan itself. */ - pNew->rRun = sqlite3LogEstAdd(rSize, rLogSize); - if( m==0 ){ - /* TUNING: Cost of a covering index scan is K*(N + log2(N)). - ** + The extra factor K of between 1.1 and 3.0 that depends - ** on the relative sizes of the table and the index. K - ** is smaller for smaller indices, thus favoring them. - ** The upper bound on K (3.0) matches the penalty factor - ** on a full table scan that tries to encourage the use of - ** indexed lookups over full scans. - */ - pNew->rRun += 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; - }else{ - /* TUNING: The cost of scanning a non-covering index is multiplied - ** by log2(N) to account for the binary search of the main table - ** that must happen for each row of the index. - ** TODO: Should there be a multiplier here, analogous to the 3x - ** multiplier for a fulltable scan or covering index scan, to - ** further discourage the use of an index scan? Or is the log2(N) - ** term sufficient discouragement? - ** TODO: What if some or all of the WHERE clause terms can be - ** computed without reference to the original table. Then the - ** penality should reduce to logK where K is the number of output - ** rows. - */ - pNew->rRun += rLogSize; + + /* The cost of visiting the index rows is N*K, where K is + ** between 1.1 and 3.0, depending on the relative sizes of the + ** index and table rows. If this is a non-covering index scan, + ** also add the cost of visiting table rows (N*3.0). */ + pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; + if( m!=0 ){ + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16); } + whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -4732,8 +4736,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ pNew->iSortIdx = 0; memset(&pNew->u, 0, sizeof(pNew->u)); for(i=0; rc==SQLITE_OK && irRun = sSum.a[i].rRun + 18; + pNew->rRun = sSum.a[i].rRun; pNew->nOut = sSum.a[i].nOut; pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); diff --git a/test/analyze9.test b/test/analyze9.test index 820bcdb0e7..125cecf182 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -577,16 +577,16 @@ do_test 13.1 { execsql ANALYZE } {} do_eqp_test 13.2.1 { - SELECT * FROM t1 WHERE a='abc' AND rowid<15 AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<15 AND b<12 } {/SEARCH TABLE t1 USING INDEX i1/} do_eqp_test 13.2.2 { - SELECT * FROM t1 WHERE a='abc' AND rowid<'15' AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<'15' AND b<12 } {/SEARCH TABLE t1 USING INDEX i1/} do_eqp_test 13.3.1 { - SELECT * FROM t1 WHERE a='abc' AND rowid<100 AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<100 AND b<12 } {/SEARCH TABLE t1 USING INDEX i2/} do_eqp_test 13.3.2 { - SELECT * FROM t1 WHERE a='abc' AND rowid<'100' AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<'100' AND b<12 } {/SEARCH TABLE t1 USING INDEX i2/} #------------------------------------------------------------------------- diff --git a/test/autoindex1.test b/test/autoindex1.test index 0e5032b9b9..6cb0ab146a 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -97,6 +97,8 @@ do_test autoindex1-210 { PRAGMA automatic_index=ON; ANALYZE; UPDATE sqlite_stat1 SET stat='10000' WHERE tbl='t1'; + -- Table t2 actually contains 8 rows. + UPDATE sqlite_stat1 SET stat='16' WHERE tbl='t2'; ANALYZE sqlite_master; SELECT b, (SELECT d FROM t2 WHERE c=a) FROM t1; } diff --git a/test/where3.test b/test/where3.test index 8fa9fa7840..c2804b5579 100644 --- a/test/where3.test +++ b/test/where3.test @@ -231,6 +231,7 @@ do_execsql_test where3-3.0 { CREATE TABLE t301(a INTEGER PRIMARY KEY,b,c); CREATE INDEX t301c ON t301(c); INSERT INTO t301 VALUES(1,2,3); + INSERT INTO t301 VALUES(2,2,3); CREATE TABLE t302(x, y); INSERT INTO t302 VALUES(4,5); ANALYZE; @@ -251,7 +252,7 @@ do_execsql_test where3-3.2 { } {} do_execsql_test where3-3.3 { SELECT * FROM t301 WHERE c=3 AND a IS NOT NULL; -} {1 2 3} +} {1 2 3 2 2 3} if 0 { # Query planner no longer does this # Verify that when there are multiple tables in a join which must be diff --git a/test/whereG.test b/test/whereG.test index 17d5653223..6274213491 100644 --- a/test/whereG.test +++ b/test/whereG.test @@ -14,6 +14,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix whereG do_execsql_test whereG-1.0 { CREATE TABLE composer( @@ -179,5 +180,36 @@ do_execsql_test whereG-4.0 { ORDER BY x; } {right} +#------------------------------------------------------------------------- +# + +reset_db +do_execsql_test 5.1 { + CREATE TABLE t1(a, b, c); + CREATE INDEX i1 ON t1(a, b); +} +do_eqp_test 5.1.2 { + SELECT * FROM t1 WHERE a>? +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a>?)}} +do_eqp_test 5.1.3 { + SELECT * FROM t1 WHERE likelihood(a>?, 0.9) +} {0 0 0 {SCAN TABLE t1}} + +do_test 5.2 { + for {set i 0} {$i < 100} {incr i} { + execsql { INSERT INTO t1 VALUES('abc', $i, $i); } + } + execsql { INSERT INTO t1 SELECT 'def', b, c FROM t1; } + execsql { ANALYZE } +} {} + +do_eqp_test 5.2.2 { + SELECT * FROM t1 WHERE likelihood(b>?, 0.01) +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>?)}} + +do_eqp_test 5.2.3 { + SELECT * FROM t1 WHERE likelihood(b>?, 0.9) +} {0 0 0 {SCAN TABLE t1}} finish_test + From cfc9df76e11b05d275e77d067f359c12044ea943 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 25 Apr 2014 15:01:01 +0000 Subject: [PATCH 02/19] Store values loaded from the stat1 table as logarithmic values in memory. FossilOrigin-Name: 1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 --- manifest | 25 +++++++++++-------------- manifest.uuid | 2 +- src/analyze.c | 19 ++++++++++++------- src/build.c | 26 +++++++++++++++++++------- src/pragma.c | 6 ++++-- src/select.c | 10 +++++----- src/sqliteInt.h | 5 ++++- src/where.c | 30 ++++++++++++++++-------------- 8 files changed, 72 insertions(+), 51 deletions(-) diff --git a/manifest b/manifest index 2b621bd738..0c855c53e6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\sthe\sway\sthe\splanner\scalculates\sthe\scosts\sof\svarious\stable\sand\sindex\sscans.\sSome\stest\scases\sstill\sfailing. -D 2014-04-24T20:04:49.939 +C Store\svalues\sloaded\sfrom\sthe\sstat1\stable\sas\slogarithmic\svalues\sin\smemory. +D 2014-04-25T15:01:01.691 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -158,7 +158,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c 663e0b291d27eb03c9fd6b421e2d61ba348a2389 +F src/analyze.c 92f1495304dd33b4f9e0b0e5aa030b068ada504d F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -167,7 +167,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 6c9b51abd404ce5b78b173b6f2248e8cb824758c F src/btree.h d79306df4ed9181b48916737fe8871a4392c4594 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c 5bfeea8f302ec2926c9eea321a61daea92a29fa4 +F src/build.c 9f7b2ed2af66dd2d186c0835d1c2a672d1f768e0 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -211,18 +211,18 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c -F src/pragma.c 21ece94d4f3e76e8e150deecafb9c7abd398ec67 +F src/pragma.c 810ef31ccfaa233201dcf100637a9777cc24e897 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be -F src/select.c bc7feff0fb4c4a1b9d655b717bef166846b48e33 +F src/select.c ed459f7f478a1e533d19c4b953693b3ffa2efd15 F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 03e2f60ccb0745fa2d3a072cb4f75fa29251d2ee +F src/sqliteInt.h bad694fd6b91b10a7a5aafa16fd05b504bad6b6e F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c c12bc20cd649bcae39de3e452bfc1a3f164454ee +F src/where.c 15a5c94c8c93500e141c6cb25af600615dc196d8 F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1161,10 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb -R 6cc54703275bf2ed6708c34ae52cf7ea -T *branch * experimental-costs -T *sym-experimental-costs * -T -sym-trunk * +P c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 +R 63fad85eb66cf540e6aa11923b167cbf U dan -Z 868aa60f36dda291ae018583501645e5 +Z 04f4e3645ebd9e33526df3bd26c04a76 diff --git a/manifest.uuid b/manifest.uuid index 4c2b675cff..5436721392 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 \ No newline at end of file +1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 2a03e292c8..4fbaaa5173 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1371,6 +1371,7 @@ static void decodeIntArray( char *zIntArray, /* String containing int array to decode */ int nOut, /* Number of slots in aOut[] */ tRowcnt *aOut, /* Store integers here */ + LogEst *aLog, /* Or, if aOut==0, here */ Index *pIndex /* Handle extra flags for this index, if not NULL */ ){ char *z = zIntArray; @@ -1389,7 +1390,11 @@ static void decodeIntArray( v = v*10 + c - '0'; z++; } - aOut[i] = v; + if( aOut ){ + aOut[i] = v; + }else{ + aLog[i] = sqlite3LogEst(v); + } if( *z==' ' ) z++; } #ifndef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -1445,12 +1450,12 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ z = argv[2]; if( pIndex ){ - decodeIntArray((char*)z, pIndex->nKeyCol+1, pIndex->aiRowEst, pIndex); - if( pIndex->pPartIdxWhere==0 ) pTable->nRowEst = pIndex->aiRowEst[0]; + decodeIntArray((char*)z, pIndex->nKeyCol+1, 0, pIndex->aiRowLogEst, pIndex); + if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; }else{ Index fakeIdx; fakeIdx.szIdxRow = pTable->szTabRow; - decodeIntArray((char*)z, 1, &pTable->nRowEst, &fakeIdx); + decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); pTable->szTabRow = fakeIdx.szIdxRow; } @@ -1642,9 +1647,9 @@ static int loadStatTbl( pPrevIdx = pIdx; } pSample = &pIdx->aSample[pIdx->nSample]; - decodeIntArray((char*)sqlite3_column_text(pStmt,1), nCol, pSample->anEq, 0); - decodeIntArray((char*)sqlite3_column_text(pStmt,2), nCol, pSample->anLt, 0); - decodeIntArray((char*)sqlite3_column_text(pStmt,3), nCol, pSample->anDLt,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,3),nCol,pSample->anDLt,0,0); /* Take a copy of the sample. Add two 0x00 bytes the end of the buffer. ** This is in case the sample record is corrupted. In that case, the diff --git a/src/build.c b/src/build.c index 4d4155ba90..fb025495dc 100644 --- a/src/build.c +++ b/src/build.c @@ -905,7 +905,7 @@ void sqlite3StartTable( pTable->iPKey = -1; pTable->pSchema = db->aDb[iDb].pSchema; pTable->nRef = 1; - pTable->nRowEst = 1048576; + pTable->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); assert( pParse->pNewTable==0 ); pParse->pNewTable = pTable; @@ -2730,15 +2730,15 @@ Index *sqlite3AllocateIndexObject( nByte = ROUND8(sizeof(Index)) + /* Index structure */ ROUND8(sizeof(char*)*nCol) + /* Index.azColl */ - ROUND8(sizeof(tRowcnt)*(nCol+1) + /* Index.aiRowEst */ + ROUND8(sizeof(LogEst)*(nCol+1) + /* Index.aiRowLogEst */ sizeof(i16)*nCol + /* Index.aiColumn */ sizeof(u8)*nCol); /* Index.aSortOrder */ p = sqlite3DbMallocZero(db, nByte + nExtra); if( p ){ char *pExtra = ((char*)p)+ROUND8(sizeof(Index)); - p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); - p->aiRowEst = (tRowcnt*)pExtra; pExtra += sizeof(tRowcnt)*(nCol+1); - p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; + p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); + p->aiRowLogEst = (LogEst*)pExtra; pExtra += sizeof(LogEst)*(nCol+1); + p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; p->aSortOrder = (u8*)pExtra; p->nColumn = nCol; p->nKeyCol = nCol - 1; @@ -2968,7 +2968,7 @@ Index *sqlite3CreateIndex( if( db->mallocFailed ){ goto exit_create_index; } - assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowEst) ); + assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowLogEst) ); assert( EIGHT_BYTE_ALIGNMENT(pIndex->azColl) ); pIndex->zName = zExtra; zExtra += nName + 1; @@ -3249,7 +3249,7 @@ exit_create_index: ** Since we do not know, guess 1 million. aiRowEst[1] is an estimate of the ** number of rows in the table that match any particular value of the ** first column of the index. aiRowEst[2] is an estimate of the number -** of rows that match any particular combiniation of the first 2 columns +** of rows that match any particular combination of the first 2 columns ** of the index. And so forth. It must always be the case that * ** aiRowEst[N]<=aiRowEst[N-1] @@ -3260,6 +3260,7 @@ exit_create_index: ** are based on typical values found in actual indices. */ void sqlite3DefaultRowEst(Index *pIdx){ +#if 0 tRowcnt *a = pIdx->aiRowEst; int i; tRowcnt n; @@ -3274,6 +3275,17 @@ void sqlite3DefaultRowEst(Index *pIdx){ if( pIdx->onError!=OE_None ){ a[pIdx->nKeyCol] = 1; } +#endif + /* 1000000, 10, 9, 8, 7, 6, 5, 4, 3, 2 */ + LogEst aVal[] = { 33, 32, 30, 28, 26, 23, 20, 16, 10 }; + LogEst *a = pIdx->aiRowLogEst; + int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); + + a[0] = pIdx->pTable->nRowLogEst; + memcpy(&a[1], aVal, nCopy*sizeof(LogEst)); + if( pIdx->onError!=OE_None ){ + a[pIdx->nKeyCol] = 0; + } } /* diff --git a/src/pragma.c b/src/pragma.c index 66d0c3328e..4c69ceb4fd 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1488,13 +1488,15 @@ void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Null, 0, 2); sqlite3VdbeAddOp2(v, OP_Integer, (int)sqlite3LogEstToInt(pTab->szTabRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, (int)pTab->nRowEst, 4); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pTab->nRowLogEst), 4); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); sqlite3VdbeAddOp2(v, OP_Integer, (int)sqlite3LogEstToInt(pIdx->szIdxRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, (int)pIdx->aiRowEst[0], 4); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pIdx->aiRowLogEst[0]), 4); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); } } diff --git a/src/select.c b/src/select.c index 6efdde4b72..dfca6d3f83 100644 --- a/src/select.c +++ b/src/select.c @@ -1690,7 +1690,7 @@ Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ assert( db->lookaside.bEnabled==0 ); pTab->nRef = 1; pTab->zName = 0; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); selectAddColumnTypeAndCollation(pParse, pTab, pSelect); pTab->iPKey = -1; @@ -3829,7 +3829,7 @@ static int withExpand( pTab->nRef = 1; pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); if( db->mallocFailed ) return SQLITE_NOMEM; @@ -4005,7 +4005,7 @@ static int selectExpander(Walker *pWalker, Select *p){ while( pSel->pPrior ){ pSel = pSel->pPrior; } selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); pTab->iPKey = -1; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; #endif }else{ @@ -4655,7 +4655,7 @@ int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); pItem->viaCoroutine = 1; pItem->regResult = dest.iSdst; sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); @@ -4686,7 +4686,7 @@ int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); VdbeComment((v, "end %s", pItem->pTab->zName)); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8e81b7a970..3ddcc8610f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1471,7 +1471,7 @@ struct Table { #ifndef SQLITE_OMIT_CHECK ExprList *pCheck; /* All CHECK constraints */ #endif - tRowcnt nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ + LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ int tnum; /* Root BTree node for this table (see note above) */ i16 iPKey; /* If not negative, use aCol[iPKey] as the primary key */ i16 nCol; /* Number of columns in this table */ @@ -1680,7 +1680,10 @@ struct UnpackedRecord { struct Index { char *zName; /* Name of this index */ i16 *aiColumn; /* Which columns are used by this index. 1st is 0 */ +#if 0 tRowcnt *aiRowEst; /* From ANALYZE: Est. rows selected by each column */ +#endif + LogEst *aiRowLogEst; /* From ANALYZE: Est. rows selected by each column */ Table *pTable; /* The SQL table being indexed */ char *zColAff; /* String defining the affinity of each column */ Index *pNext; /* The next index associated with the same table */ diff --git a/src/where.c b/src/where.c index 6a75840525..909129f666 100644 --- a/src/where.c +++ b/src/where.c @@ -1956,7 +1956,8 @@ static void whereKeyStats( iLower = 0; iUpper = aSample[0].anLt[iCol]; }else{ - iUpper = i>=pIdx->nSample ? pIdx->aiRowEst[0] : aSample[i].anLt[iCol]; + i64 nRow0 = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); + iUpper = i>=pIdx->nSample ? nRow0 : aSample[i].anLt[iCol]; iLower = aSample[i-1].anEq[iCol] + aSample[i-1].anLt[iCol]; } aStat[1] = (pIdx->nKeyCol>iCol ? pIdx->aAvgEq[iCol] : 1); @@ -2092,7 +2093,7 @@ static int whereRangeScanEst( /* Determine iLower and iUpper using ($P) only. */ if( nEq==0 ){ iLower = 0; - iUpper = p->aiRowEst[0]; + iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]); }else{ /* Note: this call could be optimized away - since the same values must ** have been requested when testing key $P in whereEqualScanEst(). */ @@ -2251,6 +2252,7 @@ static int whereInScanEst( tRowcnt *pnRow /* Write the revised row estimate here */ ){ Index *p = pBuilder->pNew->u.btree.pIndex; + i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); int nRecValid = pBuilder->nRecValid; int rc = SQLITE_OK; /* Subfunction return code */ tRowcnt nEst; /* Number of rows for a single term */ @@ -2259,14 +2261,14 @@ static int whereInScanEst( assert( p->aSample!=0 ); for(i=0; rc==SQLITE_OK && inExpr; i++){ - nEst = p->aiRowEst[0]; + nEst = nRow0; rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); nRowEst += nEst; pBuilder->nRecValid = nRecValid; } if( rc==SQLITE_OK ){ - if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0]; + if( nRowEst > nRow0 ) nRowEst = nRow0; *pnRow = nRowEst; WHERETRACE(0x10,("IN row estimate: est=%g\n", nRowEst)); } @@ -4059,8 +4061,7 @@ static int whereLoopAddBtreeIndex( assert( pNew->u.btree.nEq<=pProbe->nKeyCol ); if( pNew->u.btree.nEq < pProbe->nKeyCol ){ iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - nRowEst = sqlite3LogEst(pProbe->aiRowEst[pNew->u.btree.nEq+1]); - if( nRowEst==0 && pProbe->onError==OE_None ) nRowEst = 1; + nRowEst = pProbe->aiRowLogEst[pNew->u.btree.nEq+1]; }else{ iCol = -1; nRowEst = 0; @@ -4074,7 +4075,7 @@ static int whereLoopAddBtreeIndex( saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; pNew->rSetup = 0; - rLogSize = estLog(sqlite3LogEst(pProbe->aiRowEst[0])); + rLogSize = estLog(pProbe->aiRowLogEst[0]); /* Consider using a skip-scan if there are no WHERE clause constraints ** available for the left-most terms of the index, and if the average @@ -4082,10 +4083,11 @@ static int whereLoopAddBtreeIndex( ** number 18 was found by experimentation to be the payoff point where ** skip-scan become faster than a full-scan. */ + assert( 42==sqlite3LogEst(18) ); if( pTerm==0 && saved_nEq==saved_nSkip && saved_nEq+1nKeyCol - && pProbe->aiRowEst[saved_nEq+1]>=18 /* TUNING: Minimum for skip-scan */ + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK ){ LogEst nIter; @@ -4093,7 +4095,7 @@ static int whereLoopAddBtreeIndex( pNew->u.btree.nSkip++; pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = sqlite3LogEst(pProbe->aiRowEst[0]/pProbe->aiRowEst[saved_nEq+1]); + nIter = pProbe->aiRowLogEst[0] - pProbe->aiRowLogEst[saved_nEq+1]; pNew->rRun = rLogSize + nIter; pNew->nOut += nIter; whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter); @@ -4305,7 +4307,7 @@ static int whereLoopAddBtree( WhereInfo *pWInfo; /* WHERE analysis context */ Index *pProbe; /* An index we are evaluating */ Index sPk; /* A fake index object for the primary key */ - tRowcnt aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ + LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */ i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */ SrcList *pTabList; /* The FROM clause */ struct SrcList_item *pSrc; /* The FROM clause btree term to add */ @@ -4340,12 +4342,12 @@ static int whereLoopAddBtree( memset(&sPk, 0, sizeof(Index)); sPk.nKeyCol = 1; sPk.aiColumn = &aiColumnPk; - sPk.aiRowEst = aiRowEstPk; + sPk.aiRowLogEst = aiRowEstPk; sPk.onError = OE_Replace; sPk.pTable = pTab; sPk.szIdxRow = pTab->szTabRow; - aiRowEstPk[0] = pTab->nRowEst; - aiRowEstPk[1] = 1; + aiRowEstPk[0] = pTab->nRowLogEst; + aiRowEstPk[1] = 0; pFirst = pSrc->pTab->pIndex; if( pSrc->notIndexed==0 ){ /* The real indices of the table are only considered if the @@ -4354,7 +4356,7 @@ static int whereLoopAddBtree( } pProbe = &sPk; } - rSize = sqlite3LogEst(pTab->nRowEst); + rSize = pTab->nRowLogEst; rLogSize = estLog(rSize); #ifndef SQLITE_OMIT_AUTOMATIC_INDEX From 8ad1d8ba38ef4bf965ab1a657892b0e57be00a97 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 25 Apr 2014 20:22:45 +0000 Subject: [PATCH 03/19] If the user provides likelihood() data for a WHERE clause term used as part of an index key, have the planner use it when calculating the expected number of rows visited by the loop. FossilOrigin-Name: c51efaa5d29ee0a91b9e6a83a8dd82530670811a --- manifest | 18 +++--- manifest.uuid | 2 +- src/sqliteInt.h | 3 - src/where.c | 146 ++++++++++++++++++++++++++++------------------- src/whereInt.h | 1 + test/whereG.test | 18 ++++-- 6 files changed, 111 insertions(+), 77 deletions(-) diff --git a/manifest b/manifest index 0c855c53e6..afdc8889fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Store\svalues\sloaded\sfrom\sthe\sstat1\stable\sas\slogarithmic\svalues\sin\smemory. -D 2014-04-25T15:01:01.691 +C If\sthe\suser\sprovides\slikelihood()\sdata\sfor\sa\sWHERE\sclause\sterm\sused\sas\spart\sof\san\sindex\skey,\shave\sthe\splanner\suse\sit\swhen\scalculating\sthe\sexpected\snumber\sof\srows\svisited\sby\sthe\sloop. +D 2014-04-25T20:22:45.291 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -222,7 +222,7 @@ F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h bad694fd6b91b10a7a5aafa16fd05b504bad6b6e +F src/sqliteInt.h 63656cfa5a8221c3eb1a182e97d61b1fe2dfd7da F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -291,8 +291,8 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 15a5c94c8c93500e141c6cb25af600615dc196d8 -F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 +F src/where.c 2b3f47801939c2853c03bba3d5aa83abdd51211e +F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a F test/whereD.test fd9120e262f9da3c45940f52aefeef4d15b904e5 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 -F test/whereG.test 8189fedf3b98ab581bb70f830175e403a0ef1722 +F test/whereG.test 0ac23e5e8311b69d87245f4a85112de321031658 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 -R 63fad85eb66cf540e6aa11923b167cbf +P 1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 +R 804d3aff8175381c892eb4eebbb86307 U dan -Z 04f4e3645ebd9e33526df3bd26c04a76 +Z 6dd1c2a2c429274ab108441094bd0963 diff --git a/manifest.uuid b/manifest.uuid index 5436721392..71ddf4b2d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 \ No newline at end of file +c51efaa5d29ee0a91b9e6a83a8dd82530670811a \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3ddcc8610f..b584ff401d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1680,9 +1680,6 @@ struct UnpackedRecord { struct Index { char *zName; /* Name of this index */ i16 *aiColumn; /* Which columns are used by this index. 1st is 0 */ -#if 0 - tRowcnt *aiRowEst; /* From ANALYZE: Est. rows selected by each column */ -#endif LogEst *aiRowLogEst; /* From ANALYZE: Est. rows selected by each column */ Table *pTable; /* The SQL table being indexed */ char *zColAff; /* String defining the affinity of each column */ diff --git a/src/where.c b/src/where.c index 909129f666..73d3a22b7c 100644 --- a/src/where.c +++ b/src/where.c @@ -4040,7 +4040,6 @@ static int whereLoopAddBtreeIndex( LogEst saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ - LogEst nRowEst; /* Estimated index selectivity */ LogEst rLogSize; /* Logarithm of table size */ WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ @@ -4061,10 +4060,8 @@ static int whereLoopAddBtreeIndex( assert( pNew->u.btree.nEq<=pProbe->nKeyCol ); if( pNew->u.btree.nEq < pProbe->nKeyCol ){ iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - nRowEst = pProbe->aiRowLogEst[pNew->u.btree.nEq+1]; }else{ iCol = -1; - nRowEst = 0; } pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, pProbe); @@ -4095,35 +4092,40 @@ static int whereLoopAddBtreeIndex( pNew->u.btree.nSkip++; pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = pProbe->aiRowLogEst[0] - pProbe->aiRowLogEst[saved_nEq+1]; - pNew->rRun = rLogSize + nIter; - pNew->nOut += nIter; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter); + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + pNew->nOut -= nIter; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); pNew->nOut = saved_nOut; } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ + u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ LogEst rCostIdx; + LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */ int nIn = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nRecValid = pBuilder->nRecValid; #endif - if( (pTerm->eOperator==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) + if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) && (iCol<0 || pSrc->pTab->aCol[iCol].notNull) ){ continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ } if( pTerm->prereqRight & pNew->maskSelf ) continue; - assert( pNew->nOut==saved_nOut ); - pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; pNew->nLTerm = saved_nLTerm; if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ pNew->aLTerm[pNew->nLTerm++] = pTerm; pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; - pNew->rRun = rLogSize; /* Baseline cost is log2(N). Adjustments below */ - if( pTerm->eOperator & WO_IN ){ + + assert( nInMul==0 + || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0 + || (pNew->wsFlags & WHERE_COLUMN_IN)!=0 + || (pNew->wsFlags & WHERE_SKIPSCAN)!=0 + ); + + if( eOp & WO_IN ){ Expr *pExpr = pTerm->pExpr; pNew->wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ @@ -4135,87 +4137,111 @@ static int whereLoopAddBtreeIndex( } assert( nIn>0 ); /* RHS always has 2 or more terms... The parser ** changes "x IN (?)" into "x=?". */ - pNew->rRun += nIn; - pNew->u.btree.nEq++; - pNew->nOut = nRowEst + nInMul + nIn; - }else if( pTerm->eOperator & (WO_EQ) ){ - assert( - (pNew->wsFlags & (WHERE_COLUMN_NULL|WHERE_COLUMN_IN|WHERE_SKIPSCAN))!=0 - || nInMul==0 - ); + + }else if( eOp & (WO_EQ) ){ pNew->wsFlags |= WHERE_COLUMN_EQ; - if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1)){ - assert( (pNew->wsFlags & WHERE_COLUMN_IN)==0 || iCol<0 ); + if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ if( iCol>=0 && pProbe->onError==OE_None ){ pNew->wsFlags |= WHERE_UNQ_WANTED; }else{ pNew->wsFlags |= WHERE_ONEROW; } } - pNew->u.btree.nEq++; - pNew->nOut = nRowEst + nInMul; - }else if( pTerm->eOperator & (WO_ISNULL) ){ - pNew->wsFlags |= WHERE_COLUMN_NULL; - pNew->u.btree.nEq++; - /* TUNING: IS NULL selects 2 rows */ - nIn = 10; assert( 10==sqlite3LogEst(2) ); - pNew->nOut = nRowEst + nInMul + nIn; - }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ - testcase( pTerm->eOperator & WO_GT ); - testcase( pTerm->eOperator & WO_GE ); + + }else if( eOp & (WO_GT|WO_GE) ){ + testcase( eOp & WO_GT ); + testcase( eOp & WO_GE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pBtm = pTerm; pTop = 0; - }else{ - assert( pTerm->eOperator & (WO_LT|WO_LE) ); - testcase( pTerm->eOperator & WO_LT ); - testcase( pTerm->eOperator & WO_LE ); + }else if( (eOp & WO_ISNULL)==0 ){ + assert( eOp & (WO_LT|WO_LE) ); + testcase( eOp & WO_LT ); + testcase( eOp & WO_LE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; pTop = pTerm; pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? pNew->aLTerm[pNew->nLTerm-2] : 0; } + + /* At this point pNew->nOut is set to the number of rows expected to + ** be visited by the index scan before considering term pTerm, or the + ** values of nIn and nInMul. In other words, assuming that all + ** "x IN(...)" terms are replaced with "x = ?". This block updates + ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */ + assert( pNew->nOut==saved_nOut ); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 ** data, using some other estimate. */ - assert( pNew->nOut==saved_nOut ); whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); - } + }else{ + int nEq = ++pNew->u.btree.nEq; + assert( eOp & (WO_ISNULL|WO_EQ|WO_IN) ); + + assert( pNew->nOut==saved_nOut ); + if( pTerm->truthProb<=0 ){ + assert( (eOp & WO_IN) || nIn==0 ); + pNew->nOut += pTerm->truthProb; + pNew->nOut -= nIn; + pNew->wsFlags |= WHERE_LIKELIHOOD; + }else{ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( nInMul==0 - && pProbe->nSample - && pNew->u.btree.nEq<=pProbe->nSampleCol - && OptimizationEnabled(db, SQLITE_Stat3) - ){ - Expr *pExpr = pTerm->pExpr; - tRowcnt nOut = 0; - if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))!=0 ){ - testcase( pTerm->eOperator & WO_EQ ); - testcase( pTerm->eOperator & WO_ISNULL ); - rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); - }else if( (pTerm->eOperator & WO_IN) - && !ExprHasProperty(pExpr, EP_xIsSelect) ){ - rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); - } - assert( nOut==0 || rc==SQLITE_OK ); - if( nOut ){ - pNew->nOut = sqlite3LogEst(nOut); - if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; + tRowcnt nOut = 0; + if( nInMul==0 + && pProbe->nSample + && pNew->u.btree.nEq<=pProbe->nSampleCol + && OptimizationEnabled(db, SQLITE_Stat3) + && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) + && (pNew->wsFlags & WHERE_LIKELIHOOD)==0 + ){ + Expr *pExpr = pTerm->pExpr; + if( (eOp & (WO_EQ|WO_ISNULL))!=0 ){ + testcase( eOp & WO_EQ ); + testcase( eOp & WO_ISNULL ); + rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); + }else{ + rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); + } + assert( rc!=SQLITE_OK || nOut>0 ); + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ + if( nOut ){ + pNew->nOut = sqlite3LogEst(nOut); + if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; + pNew->nOut -= nIn; + } + } + if( nOut==0 ) +#endif + { + pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]); + if( eOp & WO_ISNULL ){ + /* TUNING: If there is no likelihood() value, assume that a + ** "col IS NULL" expression matches twice as many rows + ** as (col=?). */ + pNew->nOut += 10; + } + } } } -#endif + /* Set rCostIdx to the cost of visiting selected rows in index. Add ** it to pNew->rRun, which is currently set to the cost of the index ** seek only. Then, if this is a non-covering index, add the cost of ** visiting the rows in the main table. */ rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; - pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rCostIdx); + pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } + nOutUnadjusted = pNew->nOut; + pNew->rRun += nInMul + nIn; + pNew->nOut += nInMul + nIn; whereLoopOutputAdjust(pBuilder->pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); + pNew->nOut = nOutUnadjusted; + if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEq<(pProbe->nKeyCol + (pProbe->zName!=0)) ){ diff --git a/src/whereInt.h b/src/whereInt.h index 72e7530db9..010cd6e8ae 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -458,3 +458,4 @@ struct WhereInfo { #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ +#define WHERE_LIKELIHOOD 0x00020000 /* A likelihood() is affecting nOut */ diff --git a/test/whereG.test b/test/whereG.test index 6274213491..66918a35fa 100644 --- a/test/whereG.test +++ b/test/whereG.test @@ -181,9 +181,14 @@ do_execsql_test whereG-4.0 { } {right} #------------------------------------------------------------------------- -# - +# Test that likelihood() specifications on indexed terms are taken into +# account by various forms of loops. +# +# 5.1.*: open ended range scans +# 5.2.*: skip-scans +# reset_db + do_execsql_test 5.1 { CREATE TABLE t1(a, b, c); CREATE INDEX i1 ON t1(a, b); @@ -202,14 +207,19 @@ do_test 5.2 { execsql { INSERT INTO t1 SELECT 'def', b, c FROM t1; } execsql { ANALYZE } } {} - do_eqp_test 5.2.2 { SELECT * FROM t1 WHERE likelihood(b>?, 0.01) } {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>?)}} - do_eqp_test 5.2.3 { SELECT * FROM t1 WHERE likelihood(b>?, 0.9) } {0 0 0 {SCAN TABLE t1}} +do_eqp_test 5.3.1 { + SELECT * FROM t1 WHERE a=? +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}} +do_eqp_test 5.3.2 { + SELECT * FROM t1 WHERE likelihood(a=?, 0.9) +} {0 0 0 {SCAN TABLE t1}} + finish_test From 2dd3cdcfab89b47ee8b47260d0cdf73801094018 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 26 Apr 2014 20:21:14 +0000 Subject: [PATCH 04/19] Fix an sqlite3_stmt_status() problem caused by recent changs on this branch. FossilOrigin-Name: dee204092421a239f9f60ab83c3a5b3e24d1baea --- manifest | 15 +++++----- manifest.uuid | 2 +- src/where.c | 10 ++++--- test/cost.test | 68 ++++++++++++++++++++++++++++++++++++++++++++++ test/orderby5.test | 4 +-- 5 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 test/cost.test diff --git a/manifest b/manifest index afdc8889fe..c32bc38c65 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthe\suser\sprovides\slikelihood()\sdata\sfor\sa\sWHERE\sclause\sterm\sused\sas\spart\sof\san\sindex\skey,\shave\sthe\splanner\suse\sit\swhen\scalculating\sthe\sexpected\snumber\sof\srows\svisited\sby\sthe\sloop. -D 2014-04-25T20:22:45.291 +C Fix\san\ssqlite3_stmt_status()\sproblem\scaused\sby\srecent\schangs\son\sthis\sbranch. +D 2014-04-26T20:21:14.140 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 2b3f47801939c2853c03bba3d5aa83abdd51211e +F src/where.c 488cc018a39d33d97555b5a1e079dfafc32d348a F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -406,6 +406,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 +F test/cost.test 8a2b846e896c744b940c29e8bab46416a1324717 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -722,7 +723,7 @@ F test/orderby1.test 9b524aff9147288da43a6d7ddfdcff47fa2303c6 F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 -F test/orderby5.test 2490183fef54417209d1df253633a605d46bd350 +F test/orderby5.test 8f08a54836d21fb7c70245360751aedd1c2286fb F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa @@ -1161,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 -R 804d3aff8175381c892eb4eebbb86307 +P c51efaa5d29ee0a91b9e6a83a8dd82530670811a +R 95df5cff60bc1a7189139d237d77a9d7 U dan -Z 6dd1c2a2c429274ab108441094bd0963 +Z c6c47e040169e1387f5404e0d25c212f diff --git a/manifest.uuid b/manifest.uuid index 71ddf4b2d3..22512678d7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c51efaa5d29ee0a91b9e6a83a8dd82530670811a \ No newline at end of file +dee204092421a239f9f60ab83c3a5b3e24d1baea \ No newline at end of file diff --git a/src/where.c b/src/where.c index 73d3a22b7c..6d924e0782 100644 --- a/src/where.c +++ b/src/where.c @@ -4147,14 +4147,15 @@ static int whereLoopAddBtreeIndex( pNew->wsFlags |= WHERE_ONEROW; } } - + }else if( eOp & WO_ISNULL ){ + pNew->wsFlags |= WHERE_COLUMN_NULL; }else if( eOp & (WO_GT|WO_GE) ){ testcase( eOp & WO_GT ); testcase( eOp & WO_GE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pBtm = pTerm; pTop = 0; - }else if( (eOp & WO_ISNULL)==0 ){ + }else{ assert( eOp & (WO_LT|WO_LE) ); testcase( eOp & WO_LT ); testcase( eOp & WO_LE ); @@ -5225,9 +5226,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** scans instead? */ LogEst rScale, rSortCost; - assert( nOrderBy>0 ); + assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66; - rSortCost = nRowEst + estLog(nRowEst) + rScale; + rSortCost = nRowEst + estLog(nRowEst) + rScale + 16; + /* TUNING: The cost of implementing DISTINCT using a B-TREE is ** also N*log(N) but it has a larger constant of proportionality. ** Multiply by 3.0. */ diff --git a/test/cost.test b/test/cost.test new file mode 100644 index 0000000000..8df32d8ed1 --- /dev/null +++ b/test/cost.test @@ -0,0 +1,68 @@ +# 2014-04-26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix cost + + +do_execsql_test 1.1 { + CREATE TABLE t3(id INTEGER PRIMARY KEY, b NOT NULL); + CREATE TABLE t4(c, d, e); + CREATE UNIQUE INDEX i3 ON t3(b); + CREATE UNIQUE INDEX i4 ON t4(c, d); +} +do_eqp_test 1.2 { + SELECT e FROM t3, t4 WHERE b=c ORDER BY b, d; +} { + 0 0 0 {SCAN TABLE t3 USING COVERING INDEX i3} + 0 1 1 {SEARCH TABLE t4 USING INDEX i4 (c=?)} +} + + +do_execsql_test 2.1 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); +} + +# It is better to use an index for ORDER BY than sort externally, even +# if the index is a non-covering index. +do_eqp_test 2.2 { + SELECT * FROM t1 ORDER BY a; +} { + 0 0 0 {SCAN TABLE t1 USING INDEX i1} +} + +do_execsql_test 3.1 { + CREATE TABLE t5(a INTEGER PRIMARY KEY,b,c,d,e,f,g); + CREATE INDEX t5b ON t5(b); + CREATE INDEX t5c ON t5(c); + CREATE INDEX t5d ON t5(d); + CREATE INDEX t5e ON t5(e); + CREATE INDEX t5f ON t5(f); + CREATE INDEX t5g ON t5(g); +} + +do_eqp_test 3.2 { + SELECT a FROM t5 + WHERE b IS NULL OR c IS NULL OR d IS NULL + ORDER BY a; +} { + 0 0 0 {SEARCH TABLE t5 USING INDEX t5b (b=?)} + 0 0 0 {SEARCH TABLE t5 USING INDEX t5c (c=?)} + 0 0 0 {SEARCH TABLE t5 USING INDEX t5d (d=?)} + 0 0 0 {USE TEMP B-TREE FOR ORDER BY} +} + + + +finish_test diff --git a/test/orderby5.test b/test/orderby5.test index bccd469f25..c9cce703ba 100644 --- a/test/orderby5.test +++ b/test/orderby5.test @@ -80,12 +80,12 @@ do_execsql_test 2.1a { EXPLAIN QUERY PLAN SELECT * FROM t2 WHERE a=0 ORDER BY a, b, c; } {~/B-TREE/} + do_execsql_test 2.1b { EXPLAIN QUERY PLAN - SELECT * FROM t1 WHERE a=0 ORDER BY a, b, c; + SELECT * FROM t1 WHERE likelihood(a=0, 0.05) ORDER BY a, b, c; } {/B-TREE/} - do_execsql_test 2.2 { EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE +a=0 ORDER BY a, b, c; From 440e6ff3b023b66ecefa545990d5eeb85bda03ea Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 08:49:54 +0000 Subject: [PATCH 05/19] Fix an error in estimating of the number of rows visited by a range scan. FossilOrigin-Name: d491de62fce69d93e89f65f7713972f7c2c451f7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 7 ++++++- test/cost.test | 27 +++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c32bc38c65..2a1d6b7303 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\ssqlite3_stmt_status()\sproblem\scaused\sby\srecent\schangs\son\sthis\sbranch. -D 2014-04-26T20:21:14.140 +C Fix\san\serror\sin\sestimating\sof\sthe\snumber\sof\srows\svisited\sby\sa\srange\sscan. +D 2014-04-28T08:49:54.584 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 488cc018a39d33d97555b5a1e079dfafc32d348a +F src/where.c d4efd09b7b2c3604e48b015199f5393578ed417b F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -406,7 +406,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 8a2b846e896c744b940c29e8bab46416a1324717 +F test/cost.test 39d014a90b67169f8482ede8713bff694761d879 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c51efaa5d29ee0a91b9e6a83a8dd82530670811a -R 95df5cff60bc1a7189139d237d77a9d7 +P dee204092421a239f9f60ab83c3a5b3e24d1baea +R 3b87cec4307de74fea71def903d4f518 U dan -Z c6c47e040169e1387f5404e0d25c212f +Z dce0ef4becf30f1f5b6dab0637076369 diff --git a/manifest.uuid b/manifest.uuid index 22512678d7..94e48b42ff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dee204092421a239f9f60ab83c3a5b3e24d1baea \ No newline at end of file +d491de62fce69d93e89f65f7713972f7c2c451f7 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 6d924e0782..dd80ab4ccb 100644 --- a/src/where.c +++ b/src/where.c @@ -4241,7 +4241,12 @@ static int whereLoopAddBtreeIndex( pNew->nOut += nInMul + nIn; whereLoopOutputAdjust(pBuilder->pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); - pNew->nOut = nOutUnadjusted; + + if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ + pNew->nOut = saved_nOut; + }else{ + pNew->nOut = nOutUnadjusted; + } if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEq<(pProbe->nKeyCol + (pProbe->zName!=0)) diff --git a/test/cost.test b/test/cost.test index 8df32d8ed1..faaa7bdf16 100644 --- a/test/cost.test +++ b/test/cost.test @@ -63,6 +63,33 @@ do_eqp_test 3.2 { 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } +#------------------------------------------------------------------------- +# If there is no likelihood() or stat3 data, SQLite assumes that a closed +# range scan (e.g. one constrained by "col BETWEEN ? AND ?" constraint) +# visits 1/16 of the rows in a table. +# +# Note: 1/17 =~ 0.058 +# Note: 1/15 =~ 0.067 +# +reset_db +do_execsql_test 4.1 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); + CREATE INDEX i2 ON t1(b); +} +do_eqp_test 4.2 { + SELECT * FROM t1 WHERE likelihood(a=?, 0.058) AND b BETWEEN ? AND ?; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)} +} +do_eqp_test 4.3 { + SELECT * FROM t1 WHERE likelihood(a=?, 0.067) AND b BETWEEN ? AND ?; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b>? AND b Date: Mon, 28 Apr 2014 09:35:31 +0000 Subject: [PATCH 06/19] Modify internal function whereLoopAdjustCost() so that it does not prefer a skip-scan over a regular index scan even if the regular scan uses a subset of the WHERE terms used by the skip-scan. FossilOrigin-Name: 88a5758dcce891eb7be15432ebdc9f80071d413b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 2a1d6b7303..b301a439d1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\serror\sin\sestimating\sof\sthe\snumber\sof\srows\svisited\sby\sa\srange\sscan. -D 2014-04-28T08:49:54.584 +C Modify\sinternal\sfunction\swhereLoopAdjustCost()\sso\sthat\sit\sdoes\snot\sprefer\sa\sskip-scan\sover\sa\sregular\sindex\sscan\seven\sif\sthe\sregular\sscan\suses\sa\ssubset\sof\sthe\sWHERE\sterms\sused\sby\sthe\sskip-scan. +D 2014-04-28T09:35:31.541 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c d4efd09b7b2c3604e48b015199f5393578ed417b +F src/where.c 263f8940aa21adabbc397590046f040c54aca5f4 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P dee204092421a239f9f60ab83c3a5b3e24d1baea -R 3b87cec4307de74fea71def903d4f518 +P d491de62fce69d93e89f65f7713972f7c2c451f7 +R a36e9b880c1d76e2d8cd1072dd5871be U dan -Z dce0ef4becf30f1f5b6dab0637076369 +Z 07995bb3734998fef6b8bca04739721a diff --git a/manifest.uuid b/manifest.uuid index 94e48b42ff..8be55d6794 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d491de62fce69d93e89f65f7713972f7c2c451f7 \ No newline at end of file +88a5758dcce891eb7be15432ebdc9f80071d413b \ No newline at end of file diff --git a/src/where.c b/src/where.c index dd80ab4ccb..4c1d81e183 100644 --- a/src/where.c +++ b/src/where.c @@ -3779,9 +3779,11 @@ static int whereLoopCheaperProperSubset( */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; + if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return; for(; p; p=p->pNextLoop){ if( p->iTab!=pTemplate->iTab ) continue; if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; + if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its ** subset p */ From b51926e67c4921137cac5e3b66257a445f23ad5f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 10:00:59 +0000 Subject: [PATCH 07/19] Update test script analyze3.test to account for the fact that SQLite now prefers a full-table scan over a non-covering index scan that visits a large percentage of the table rows. FossilOrigin-Name: 35f46a55d866b9a87c1321aab8e0cfe86ccadb93 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/analyze3.test | 42 ++++++++++++++++++++++++++++++------------ 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index b301a439d1..06eee5c7d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sinternal\sfunction\swhereLoopAdjustCost()\sso\sthat\sit\sdoes\snot\sprefer\sa\sskip-scan\sover\sa\sregular\sindex\sscan\seven\sif\sthe\sregular\sscan\suses\sa\ssubset\sof\sthe\sWHERE\sterms\sused\sby\sthe\sskip-scan. -D 2014-04-28T09:35:31.541 +C Update\stest\sscript\sanalyze3.test\sto\saccount\sfor\sthe\sfact\sthat\sSQLite\snow\sprefers\sa\sfull-table\sscan\sover\sa\snon-covering\sindex\sscan\sthat\svisits\sa\slarge\spercentage\sof\sthe\stable\srows. +D 2014-04-28T10:00:59.075 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -305,7 +305,7 @@ F test/alter4.test d6c011fa0d6227abba762498cafbb607c9609e93 F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 1772936d66471c65221e437b6d1999c3a03166c4 -F test/analyze3.test 412f690dfe95b337475e3e78a84a85d25f6f125d +F test/analyze3.test bf41f0f680dd1e0d44eed5e769531e93a5320275 F test/analyze4.test eff2df19b8dd84529966420f29ea52edc6b56213 F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d491de62fce69d93e89f65f7713972f7c2c451f7 -R a36e9b880c1d76e2d8cd1072dd5871be +P 88a5758dcce891eb7be15432ebdc9f80071d413b +R a1e1c4a3f11d62d0bb62f198394034d8 U dan -Z 07995bb3734998fef6b8bca04739721a +Z fd306a356911acfe444509c2efb0dade diff --git a/manifest.uuid b/manifest.uuid index 8be55d6794..5b247aefe3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -88a5758dcce891eb7be15432ebdc9f80071d413b \ No newline at end of file +35f46a55d866b9a87c1321aab8e0cfe86ccadb93 \ No newline at end of file diff --git a/test/analyze3.test b/test/analyze3.test index fb26303ee3..e7416d5730 100644 --- a/test/analyze3.test +++ b/test/analyze3.test @@ -103,12 +103,21 @@ do_test analyze3-1.1.1 { } } {1} +do_execsql_test analyze3-1.1.x { + SELECT count(*) FROM t1 WHERE x>200 AND x<300; + SELECT count(*) FROM t1 WHERE x>0 AND x<1100; +} {99 1000} + +# The first of the following two SELECT statements visits 99 rows. So +# it is better to use the index. But the second visits every row in +# the table (1000 in total) so it is better to do a full-table scan. +# do_eqp_test analyze3-1.1.2 { SELECT sum(y) FROM t1 WHERE x>200 AND x<300 } {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x0 AND x<1100 -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x200 AND x<300 } @@ -125,17 +134,17 @@ do_test analyze3-1.1.6 { } {199 0 14850} do_test analyze3-1.1.7 { sf_execsql { SELECT sum(y) FROM t1 WHERE x>0 AND x<1100 } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.1.8 { set l [string range "0" 0 end] set u [string range "1100" 0 end] sf_execsql { SELECT sum(y) FROM t1 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.1.9 { set l [expr int(0)] set u [expr int(1100)] sf_execsql { SELECT sum(y) FROM t1 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} # The following tests are similar to the block above. The difference is @@ -152,12 +161,17 @@ do_test analyze3-1.2.1 { ANALYZE; } } {} +do_execsql_test analyze3-2.1.x { + SELECT count(*) FROM t2 WHERE x>1 AND x<2; + SELECT count(*) FROM t2 WHERE x>0 AND x<99; +} {200 990} do_eqp_test analyze3-1.2.2 { SELECT sum(y) FROM t2 WHERE x>1 AND x<2 } {0 0 0 {SEARCH TABLE t2 USING INDEX i2 (x>? AND x0 AND x<99 -} {0 0 0 {SEARCH TABLE t2 USING INDEX i2 (x>? AND x12 AND x<20 } } {161 0 4760} @@ -173,17 +187,17 @@ do_test analyze3-1.2.6 { } {161 0 integer integer 4760} do_test analyze3-1.2.7 { sf_execsql { SELECT sum(y) FROM t2 WHERE x>0 AND x<99 } -} {1981 0 490555} +} {999 999 490555} do_test analyze3-1.2.8 { set l [string range "0" 0 end] set u [string range "99" 0 end] sf_execsql {SELECT typeof($l), typeof($u), sum(y) FROM t2 WHERE x>$l AND x<$u} -} {1981 0 text text 490555} +} {999 999 text text 490555} do_test analyze3-1.2.9 { set l [expr int(0)] set u [expr int(99)] sf_execsql {SELECT typeof($l), typeof($u), sum(y) FROM t2 WHERE x>$l AND x<$u} -} {1981 0 integer integer 490555} +} {999 999 integer integer 490555} # Same tests a third time. This time, column x has INTEGER affinity and # is not the leftmost column of the table. This triggered a bug causing @@ -199,12 +213,16 @@ do_test analyze3-1.3.1 { ANALYZE; } } {} +do_execsql_test analyze3-1.3.x { + SELECT count(*) FROM t3 WHERE x>200 AND x<300; + SELECT count(*) FROM t3 WHERE x>0 AND x<1100 +} {99 1000} do_eqp_test analyze3-1.3.2 { SELECT sum(y) FROM t3 WHERE x>200 AND x<300 } {0 0 0 {SEARCH TABLE t3 USING INDEX i3 (x>? AND x0 AND x<1100 -} {0 0 0 {SEARCH TABLE t3 USING INDEX i3 (x>? AND x200 AND x<300 } @@ -221,17 +239,17 @@ do_test analyze3-1.3.6 { } {199 0 14850} do_test analyze3-1.3.7 { sf_execsql { SELECT sum(y) FROM t3 WHERE x>0 AND x<1100 } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.3.8 { set l [string range "0" 0 end] set u [string range "1100" 0 end] sf_execsql { SELECT sum(y) FROM t3 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.3.9 { set l [expr int(0)] set u [expr int(1100)] sf_execsql { SELECT sum(y) FROM t3 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} #------------------------------------------------------------------------- # Test that the values of bound SQL variables may be used for the LIKE From 9881d60d17451055282b2bde12454fa42aeaabc8 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 12:08:23 +0000 Subject: [PATCH 08/19] Add an extra column to a table in analyze9.test to give the planner a little more reason to select an index. FossilOrigin-Name: 1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/analyze9.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 06eee5c7d2..04a8c1374d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\stest\sscript\sanalyze3.test\sto\saccount\sfor\sthe\sfact\sthat\sSQLite\snow\sprefers\sa\sfull-table\sscan\sover\sa\snon-covering\sindex\sscan\sthat\svisits\sa\slarge\spercentage\sof\sthe\stable\srows. -D 2014-04-28T10:00:59.075 +C Add\san\sextra\scolumn\sto\sa\stable\sin\sanalyze9.test\sto\sgive\sthe\splanner\sa\slittle\smore\sreason\sto\sselect\san\sindex. +D 2014-04-28T12:08:23.821 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -311,7 +311,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test e219daa58fd8677c6a43d771798cf37d68f51d3e +F test/analyze9.test 623e02a99a78fa12fe5def2fd559032d5d887e0f F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 88a5758dcce891eb7be15432ebdc9f80071d413b -R a1e1c4a3f11d62d0bb62f198394034d8 +P 35f46a55d866b9a87c1321aab8e0cfe86ccadb93 +R 8aefc1341eaf7a6ece9efb278712f5a6 U dan -Z fd306a356911acfe444509c2efb0dade +Z ba7221d22e50ff31dad54b0ec475658f diff --git a/manifest.uuid b/manifest.uuid index 5b247aefe3..7dba22fdc4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35f46a55d866b9a87c1321aab8e0cfe86ccadb93 \ No newline at end of file +1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 \ No newline at end of file diff --git a/test/analyze9.test b/test/analyze9.test index 125cecf182..f25e5924e6 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -566,7 +566,7 @@ foreach {tn schema} { drop_all_tables do_test 13.1 { execsql { - CREATE TABLE t1(a, b, c); + CREATE TABLE t1(a, b, c, d); CREATE INDEX i1 ON t1(a); CREATE INDEX i2 ON t1(b, c); } From 6b6828625b2843a719fb123dc46feb5666b2cda3 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 15:11:25 +0000 Subject: [PATCH 09/19] Update unordered.test to take into account for the fact that SQLite now prefers a full-table scan over a non-covering index scan that visits a large percentage of the table rows. FossilOrigin-Name: 20f468dfbcb247e51446fad411a6e6cc0d130411 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/unordered.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 04a8c1374d..05f3d798c0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sextra\scolumn\sto\sa\stable\sin\sanalyze9.test\sto\sgive\sthe\splanner\sa\slittle\smore\sreason\sto\sselect\san\sindex. -D 2014-04-28T12:08:23.821 +C Update\sunordered.test\sto\stake\sinto\saccount\sfor\sthe\sfact\sthat\sSQLite\snow\sprefers\sa\sfull-table\sscan\sover\sa\snon-covering\sindex\sscan\sthat\svisits\sa\slarge\spercentage\sof\sthe\stable\srows. +D 2014-04-28T15:11:25.118 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1023,7 +1023,7 @@ F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 -F test/unordered.test ef85ac8f2f3c93ed2b9e811b684de73175fc464c +F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 1b6c488a8f993d090b7ee9ad0e234faa161b3aeb F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 35f46a55d866b9a87c1321aab8e0cfe86ccadb93 -R 8aefc1341eaf7a6ece9efb278712f5a6 +P 1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 +R 307957877e44688fb3b83b922c0183e6 U dan -Z ba7221d22e50ff31dad54b0ec475658f +Z 085ae09cb0f2460f340245b5be0529cf diff --git a/manifest.uuid b/manifest.uuid index 7dba22fdc4..8fd0bd92ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 \ No newline at end of file +20f468dfbcb247e51446fad411a6e6cc0d130411 \ No newline at end of file diff --git a/test/unordered.test b/test/unordered.test index d8f7aa6add..147e91f0d9 100644 --- a/test/unordered.test +++ b/test/unordered.test @@ -42,7 +42,7 @@ foreach idxmode {ordered unordered} { 1 "SELECT * FROM t1 ORDER BY a" {0 0 0 {SCAN TABLE t1 USING INDEX i1}} {0 0 0 {SCAN TABLE t1} 0 0 0 {USE TEMP B-TREE FOR ORDER BY}} - 2 "SELECT * FROM t1 WHERE a >?" + 2 "SELECT * FROM t1 WHERE a > 100" {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a>?)}} {0 0 0 {SCAN TABLE t1}} 3 "SELECT * FROM t1 WHERE a = ? ORDER BY rowid" From 42685f211e0ba5610a2702f39a80f950c9d4ae4f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 19:34:06 +0000 Subject: [PATCH 10/19] The trunk assumes that an open range constraint on an indexed term (col>?) term matches 1/4 of the indexed rows, and that a closed constraint (col BETWEEN ? AND ?) matches 1/64. Change this branch to do the same. FossilOrigin-Name: 4047ac75e2a8f0b330255501c42e4f04e5ab500d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 05f3d798c0..b037731f09 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sunordered.test\sto\stake\sinto\saccount\sfor\sthe\sfact\sthat\sSQLite\snow\sprefers\sa\sfull-table\sscan\sover\sa\snon-covering\sindex\sscan\sthat\svisits\sa\slarge\spercentage\sof\sthe\stable\srows. -D 2014-04-28T15:11:25.118 +C The\strunk\sassumes\sthat\san\sopen\srange\sconstraint\son\san\sindexed\sterm\s(col>?)\sterm\smatches\s1/4\sof\sthe\sindexed\srows,\sand\sthat\sa\sclosed\sconstraint\s(col\sBETWEEN\s?\sAND\s?)\smatches\s1/64.\sChange\sthis\sbranch\sto\sdo\sthe\ssame. +D 2014-04-28T19:34:06.281 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 263f8940aa21adabbc397590046f040c54aca5f4 +F src/where.c 0a518940065c10ad8dedf8f2d6e73f1e14eb8472 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 -R 307957877e44688fb3b83b922c0183e6 +P 20f468dfbcb247e51446fad411a6e6cc0d130411 +R eabc843f8d46cb0d3b0b7dd8faf4ef00 U dan -Z 085ae09cb0f2460f340245b5be0529cf +Z 37bdf577e72e4cd110b1edb4853d30e0 diff --git a/manifest.uuid b/manifest.uuid index 8fd0bd92ad..405d43fc27 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -20f468dfbcb247e51446fad411a6e6cc0d130411 \ No newline at end of file +4047ac75e2a8f0b330255501c42e4f04e5ab500d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4c1d81e183..4b2eef9c52 100644 --- a/src/where.c +++ b/src/where.c @@ -2155,6 +2155,12 @@ static int whereRangeScanEst( assert( pLower || pUpper ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); + /* TUNING: If there is both an upper and lower limit, assume the range is + ** reduced by an additional 75%. This means that, by default, an open-ended + ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the + ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to + ** match 1/64 of the index. */ + if( pLower && pUpper ) nNew -= 20; nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; if( nNew Date: Mon, 28 Apr 2014 20:11:20 +0000 Subject: [PATCH 11/19] Do not reduce the number of rows scanned at all for "IS NOT NULL" constraints. Fix a bug in calculating the number of rows visited by scans of partial indicies. FossilOrigin-Name: a8ae93f0cffa116df0ba34d46a53f49d42dace41 --- manifest | 16 ++++---- manifest.uuid | 2 +- src/where.c | 8 ++-- test/cost.test | 95 ++++++++++++++++++++++++++++++++++++++++++++++++ test/index6.test | 5 ++- 5 files changed, 112 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index b037731f09..823653ac7d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\strunk\sassumes\sthat\san\sopen\srange\sconstraint\son\san\sindexed\sterm\s(col>?)\sterm\smatches\s1/4\sof\sthe\sindexed\srows,\sand\sthat\sa\sclosed\sconstraint\s(col\sBETWEEN\s?\sAND\s?)\smatches\s1/64.\sChange\sthis\sbranch\sto\sdo\sthe\ssame. -D 2014-04-28T19:34:06.281 +C Do\snot\sreduce\sthe\snumber\sof\srows\sscanned\sat\sall\sfor\s"IS\sNOT\sNULL"\sconstraints.\sFix\sa\sbug\sin\scalculating\sthe\snumber\sof\srows\svisited\sby\sscans\sof\spartial\sindicies. +D 2014-04-28T20:11:20.674 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 0a518940065c10ad8dedf8f2d6e73f1e14eb8472 +F src/where.c da9c2c1d0c2ecf51ea3b7a27c826999972f24086 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -406,7 +406,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 39d014a90b67169f8482ede8713bff694761d879 +F test/cost.test 84473f27749e0f3b6837a8e1403967010c347ca5 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -611,7 +611,7 @@ F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test 55a90cff99834305e8141df7afaef39674b57062 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test fc07c14193c0430814e7a08b5da46888ee795c33 -F test/index6.test a0a2d286ffa6d35813f5003fdb7be124825b4422 +F test/index6.test fb370966ac3cd0989053dd5385757b5c3e24ab6a F test/index7.test a3baf9a625bda7fd49471e99aeae04095fbfeecf F test/indexedby.test b2f22f3e693a53813aa3f50b812eb609ba6df1ec F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 20f468dfbcb247e51446fad411a6e6cc0d130411 -R eabc843f8d46cb0d3b0b7dd8faf4ef00 +P 4047ac75e2a8f0b330255501c42e4f04e5ab500d +R fc72ae401d0e8a316c9dd5e0ec98069f U dan -Z 37bdf577e72e4cd110b1edb4853d30e0 +Z f6f096240da2e8d5cf26af9c9310dce5 diff --git a/manifest.uuid b/manifest.uuid index 405d43fc27..99d11f9b9b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4047ac75e2a8f0b330255501c42e4f04e5ab500d \ No newline at end of file +a8ae93f0cffa116df0ba34d46a53f49d42dace41 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4b2eef9c52..8abe6bd1a1 100644 --- a/src/where.c +++ b/src/where.c @@ -1992,9 +1992,7 @@ static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ if( pTerm ){ if( pTerm->truthProb<=0 ){ nRet += pTerm->truthProb; - }else if( pTerm->wtFlags & TERM_VNULL ){ - nRet -= 10; assert( 10==sqlite3LogEst(2) ); - }else{ + }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ nRet -= 20; assert( 20==sqlite3LogEst(4) ); } } @@ -2153,14 +2151,17 @@ static int whereRangeScanEst( UNUSED_PARAMETER(pBuilder); #endif assert( pLower || pUpper ); + assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); + /* TUNING: If there is both an upper and lower limit, assume the range is ** reduced by an additional 75%. This means that, by default, an open-ended ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to ** match 1/64 of the index. */ if( pLower && pUpper ) nNew -= 20; + nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; if( nNewiTab, pWC, pProbe->pPartIdxWhere) ){ continue; /* Partial index inappropriate for this query */ } + rSize = pProbe->aiRowLogEst[0]; pNew->u.btree.nEq = 0; pNew->u.btree.nSkip = 0; pNew->nLTerm = 0; diff --git a/test/cost.test b/test/cost.test index faaa7bdf16..0f9314fc8e 100644 --- a/test/cost.test +++ b/test/cost.test @@ -89,6 +89,101 @@ do_eqp_test 4.3 { } +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 5.1 { + CREATE TABLE t2(x, y); + CREATE INDEX t2i1 ON t2(x); +} + +do_eqp_test 5.2 { + SELECT * FROM t2 ORDER BY x, y; +} {} +#exit + +# TODO: Check this one out!! +# set sqlite_where_trace 0xfff +do_eqp_test 5.3 { + SELECT * FROM t2 WHERE x BETWEEN ? AND ? ORDER BY rowid; +} {} +#exit + +# where7.test, where8.test: +# +do_execsql_test 6.1 { + CREATE TABLE t3(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX t3i1 ON t3(b); + CREATE INDEX t3i2 ON t3(c); +} + +#set sqlite_where_trace 0xfff +# eqp.test +do_eqp_test 6.2 { + SELECT a FROM t3 WHERE (b BETWEEN 2 AND 4) OR c=100 ORDER BY a +} { +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 7.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY,b,c,d,e,f,g); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + CREATE INDEX t1d ON t1(d); + CREATE INDEX t1e ON t1(e); + CREATE INDEX t1f ON t1(f); + CREATE INDEX t1g ON t1(g); +} + +do_eqp_test 7.2 { + SELECT a FROM t1 + WHERE (b>=950 AND b<=1010) OR (b IS NULL AND c NOT NULL) + ORDER BY a +} { +} + +#set sqlite_where_trace 0xfff +do_eqp_test 7.3 { + SELECT rowid FROM t1 + WHERE (+b IS NULL AND c NOT NULL AND d NOT NULL) + OR (b NOT NULL AND c IS NULL AND d NOT NULL) + OR (b NOT NULL AND c NOT NULL AND d IS NULL) +} {} +#exit + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 8.1 { + CREATE TABLE composer( + cid INTEGER PRIMARY KEY, + cname TEXT + ); + CREATE TABLE album( + aid INTEGER PRIMARY KEY, + aname TEXT + ); + CREATE TABLE track( + tid INTEGER PRIMARY KEY, + cid INTEGER REFERENCES composer, + aid INTEGER REFERENCES album, + title TEXT + ); + CREATE INDEX track_i1 ON track(cid); + CREATE INDEX track_i2 ON track(aid); +} + +do_eqp_test 8.2 { + SELECT DISTINCT aname + FROM album, composer, track + WHERE cname LIKE '%bach%' + AND unlikely(composer.cid=track.cid) + AND unlikely(album.aid=track.aid); +} { +} + finish_test diff --git a/test/index6.test b/test/index6.test index 3451e5c1df..68bdd06c14 100644 --- a/test/index6.test +++ b/test/index6.test @@ -145,11 +145,11 @@ do_test index6-2.1 { execsql { CREATE TABLE t2(a,b); INSERT INTO t2(a,b) SELECT value, value FROM nums WHERE value<1000; - UPDATE t2 SET a=NULL WHERE b%5==0; + UPDATE t2 SET a=NULL WHERE b%2==0; CREATE INDEX t2a1 ON t2(a) WHERE a IS NOT NULL; SELECT count(*) FROM t2 WHERE a IS NOT NULL; } -} {800} +} {500} do_test index6-2.2 { execsql { EXPLAIN QUERY PLAN @@ -157,6 +157,7 @@ do_test index6-2.2 { } } {/.* TABLE t2 USING INDEX t2a1 .*/} ifcapable stat4||stat3 { + execsql ANALYZE do_test index6-2.3stat4 { execsql { EXPLAIN QUERY PLAN From 67a03cfd6e5213f1b7997356d5db620bd21e5329 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 29 Apr 2014 12:01:35 +0000 Subject: [PATCH 12/19] Fix a test case so that it updates sqlite_stat1 consistently. FossilOrigin-Name: 2dc5a0b55567f13f0528ed17242e680cde2f2a29 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/skipscan2.test | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 823653ac7d..a7e8fbdb50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sreduce\sthe\snumber\sof\srows\sscanned\sat\sall\sfor\s"IS\sNOT\sNULL"\sconstraints.\sFix\sa\sbug\sin\scalculating\sthe\snumber\sof\srows\svisited\sby\sscans\sof\spartial\sindicies. -D 2014-04-28T20:11:20.674 +C Fix\sa\stest\scase\sso\sthat\sit\supdates\ssqlite_stat1\sconsistently. +D 2014-04-29T12:01:35.119 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -814,7 +814,7 @@ F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f -F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d +F test/skipscan2.test d77f79cdbba25f0f6f35298136cff21a7d7a553a F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4047ac75e2a8f0b330255501c42e4f04e5ab500d -R fc72ae401d0e8a316c9dd5e0ec98069f +P a8ae93f0cffa116df0ba34d46a53f49d42dace41 +R 176e01464114f39a9f52a0420836d3f7 U dan -Z f6f096240da2e8d5cf26af9c9310dce5 +Z 9c22069b17d97b138b62636ee980813e diff --git a/manifest.uuid b/manifest.uuid index 99d11f9b9b..5debc1c5f8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8ae93f0cffa116df0ba34d46a53f49d42dace41 \ No newline at end of file +2dc5a0b55567f13f0528ed17242e680cde2f2a29 \ No newline at end of file diff --git a/test/skipscan2.test b/test/skipscan2.test index e39b16ed27..27d193e94d 100644 --- a/test/skipscan2.test +++ b/test/skipscan2.test @@ -74,6 +74,7 @@ do_execsql_test skipscan2-1.4 { -- of a skip-scan. So make a manual adjustment to the stat1 table -- to make it seem like there are many more. UPDATE sqlite_stat1 SET stat='10000 5000 20' WHERE idx='people_idx1'; + UPDATE sqlite_stat1 SET stat='10000 1' WHERE idx='sqlite_autoindex_people_1'; ANALYZE sqlite_master; } db cache flush From 09e1df6c009bc90558ec0250f005bb59fe8f808e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 29 Apr 2014 16:10:22 +0000 Subject: [PATCH 13/19] Ignore likelihood() values on indexed IPK lookups. FossilOrigin-Name: 5bb7757a7b32a74482d3e93e9c9eea02273fe981 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 2 +- test/cost.test | 36 ++++++++++++++++++++++-------------- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index a7e8fbdb50..cf16f91a6e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\scase\sso\sthat\sit\supdates\ssqlite_stat1\sconsistently. -D 2014-04-29T12:01:35.119 +C Ignore\slikelihood()\svalues\son\sindexed\sIPK\slookups. +D 2014-04-29T16:10:22.104 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c da9c2c1d0c2ecf51ea3b7a27c826999972f24086 +F src/where.c 397bd5d4f402238c396d73b3c98b9fa2c312eea4 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -406,7 +406,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 84473f27749e0f3b6837a8e1403967010c347ca5 +F test/cost.test 62386ccac862ea9a808a0954037f430690974b0f F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a8ae93f0cffa116df0ba34d46a53f49d42dace41 -R 176e01464114f39a9f52a0420836d3f7 +P 2dc5a0b55567f13f0528ed17242e680cde2f2a29 +R b5feca3cdcf90dbcdc4d81625fcb13e8 U dan -Z 9c22069b17d97b138b62636ee980813e +Z d7294cbb283905147264033ecb5a94d9 diff --git a/manifest.uuid b/manifest.uuid index 5debc1c5f8..0097b17163 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2dc5a0b55567f13f0528ed17242e680cde2f2a29 \ No newline at end of file +5bb7757a7b32a74482d3e93e9c9eea02273fe981 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 8abe6bd1a1..566c907e55 100644 --- a/src/where.c +++ b/src/where.c @@ -4189,7 +4189,7 @@ static int whereLoopAddBtreeIndex( assert( eOp & (WO_ISNULL|WO_EQ|WO_IN) ); assert( pNew->nOut==saved_nOut ); - if( pTerm->truthProb<=0 ){ + if( pTerm->truthProb<=0 && iCol>=0 ){ assert( (eOp & WO_IN) || nIn==0 ); pNew->nOut += pTerm->truthProb; pNew->nOut -= nIn; diff --git a/test/cost.test b/test/cost.test index 0f9314fc8e..979815f9f7 100644 --- a/test/cost.test +++ b/test/cost.test @@ -66,10 +66,10 @@ do_eqp_test 3.2 { #------------------------------------------------------------------------- # If there is no likelihood() or stat3 data, SQLite assumes that a closed # range scan (e.g. one constrained by "col BETWEEN ? AND ?" constraint) -# visits 1/16 of the rows in a table. +# visits 1/64 of the rows in a table. # -# Note: 1/17 =~ 0.058 -# Note: 1/15 =~ 0.067 +# Note: 1/63 =~ 0.016 +# Note: 1/65 =~ 0.015 # reset_db do_execsql_test 4.1 { @@ -78,12 +78,12 @@ do_execsql_test 4.1 { CREATE INDEX i2 ON t1(b); } do_eqp_test 4.2 { - SELECT * FROM t1 WHERE likelihood(a=?, 0.058) AND b BETWEEN ? AND ?; + SELECT * FROM t1 WHERE likelihood(a=?, 0.014) AND b BETWEEN ? AND ?; } { 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)} } do_eqp_test 4.3 { - SELECT * FROM t1 WHERE likelihood(a=?, 0.067) AND b BETWEEN ? AND ?; + SELECT * FROM t1 WHERE likelihood(a=?, 0.016) AND b BETWEEN ? AND ?; } { 0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b>? AND b? AND x? AND b=950 AND b<=1010) OR (b IS NULL AND c NOT NULL) ORDER BY a } { + 0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b Date: Tue, 29 Apr 2014 19:01:57 +0000 Subject: [PATCH 14/19] Test that the default values used when sqlite_stat1 data is not available are calculated correctly. Fixes for the same. FossilOrigin-Name: e2d42f909de85a0586389f2dc0e654f7af2e351a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 35 +++++++++++++++-------------------- test/cost.test | 45 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index 5dc1bf6a38..f4b5efc137 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthis\sbranch. -D 2014-04-29T16:46:24.176 +C Test\sthat\sthe\sdefault\svalues\sused\swhen\ssqlite_stat1\sdata\sis\snot\savailable\sare\scalculated\scorrectly.\sFixes\sfor\sthe\ssame. +D 2014-04-29T19:01:57.481 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 6c9b51abd404ce5b78b173b6f2248e8cb824758c F src/btree.h d79306df4ed9181b48916737fe8871a4392c4594 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c 9f7b2ed2af66dd2d186c0835d1c2a672d1f768e0 +F src/build.c 02665ca158431a0926b10cbd7d8178a4c9fc4a22 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -407,7 +407,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 62386ccac862ea9a808a0954037f430690974b0f +F test/cost.test 41b350fcc811a4fcccffbab06a3a79f33b8ea0de F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5bb7757a7b32a74482d3e93e9c9eea02273fe981 af2cbe64adab5f9e3b0f3da00d06428088589d7f -R 9c74729ef8e10b97c890b0be6d308bd3 +P d74299f037f3a6a4b3bce8b4d1c76c407c1f3b3e +R e73cccb2b7e246d7f5476a97d396afde U dan -Z e5760c6e9001de9044df542c0c7c165c +Z 19f8efc43d85a395f7ff59d4035b6f55 diff --git a/manifest.uuid b/manifest.uuid index e9d3392789..92aa1b2d7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d74299f037f3a6a4b3bce8b4d1c76c407c1f3b3e \ No newline at end of file +e2d42f909de85a0586389f2dc0e654f7af2e351a \ No newline at end of file diff --git a/src/build.c b/src/build.c index fb025495dc..a79abed3b6 100644 --- a/src/build.c +++ b/src/build.c @@ -3260,32 +3260,27 @@ exit_create_index: ** are based on typical values found in actual indices. */ void sqlite3DefaultRowEst(Index *pIdx){ -#if 0 - tRowcnt *a = pIdx->aiRowEst; - int i; - tRowcnt n; - assert( a!=0 ); - a[0] = pIdx->pTable->nRowEst; - if( a[0]<10 ) a[0] = 10; - n = 10; - for(i=1; i<=pIdx->nKeyCol; i++){ - a[i] = n; - if( n>5 ) n--; - } - if( pIdx->onError!=OE_None ){ - a[pIdx->nKeyCol] = 1; - } -#endif - /* 1000000, 10, 9, 8, 7, 6, 5, 4, 3, 2 */ - LogEst aVal[] = { 33, 32, 30, 28, 26, 23, 20, 16, 10 }; + /* 10, 9, 8, 7, 6 */ + LogEst aVal[] = { 33, 32, 30, 28, 26 }; LogEst *a = pIdx->aiRowLogEst; int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); + int i; + /* Set the first entry (number of rows in the index) to the estimated + ** number of rows in the table. Or 10, if the estimated number of rows + ** in the table is less than that. */ a[0] = pIdx->pTable->nRowLogEst; + if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); + + /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is + ** 6 and each subsequent value (if any) is 5. */ memcpy(&a[1], aVal, nCopy*sizeof(LogEst)); - if( pIdx->onError!=OE_None ){ - a[pIdx->nKeyCol] = 0; + for(i=nCopy+1; i<=pIdx->nKeyCol; i++){ + a[i] = 23; assert( 23==sqlite3LogEst(5) ); } + + assert( 0==sqlite3LogEst(1) ); + if( pIdx->onError!=OE_None ) a[pIdx->nKeyCol] = 0; } /* diff --git a/test/cost.test b/test/cost.test index 979815f9f7..e04dc9392a 100644 --- a/test/cost.test +++ b/test/cost.test @@ -192,6 +192,51 @@ do_eqp_test 8.2 { 0 0 0 {USE TEMP B-TREE FOR DISTINCT} } +#------------------------------------------------------------------------- +# +do_execsql_test 9.1 { + CREATE TABLE t1( + a,b,c,d,e, f,g,h,i,j, + k,l,m,n,o, p,q,r,s,t + ); + CREATE INDEX i1 ON t1(k,l,m,n,o,p,q,r,s,t); +} +do_test 9.2 { + for {set i 0} {$i < 100} {incr i} { + execsql { INSERT INTO t1 DEFAULT VALUES } + } + execsql { + ANALYZE; + CREATE INDEX i2 ON t1(a,b,c,d,e,f,g,h,i,j); + } +} {} + +set L [list a=? b=? c=? d=? e=? f=? g=? h=? i=? j=?] +foreach {tn nTerm nRow} { + 1 1 10 + 2 2 9 + 3 3 8 + 4 4 7 + 5 5 6 + 6 6 5 + 7 7 5 + 8 8 5 + 9 9 5 + 10 10 5 +} { + set w [join [lrange $L 0 [expr $nTerm-1]] " AND "] + set p1 [expr ($nRow-1) / 100.0] + set p2 [expr ($nRow+1) / 100.0] + + set sql1 "SELECT * FROM t1 WHERE likelihood(k=?, $p1) AND $w" + set sql2 "SELECT * FROM t1 WHERE likelihood(k=?, $p2) AND $w" + + do_eqp_test 9.3.$tn.1 $sql1 {/INDEX i1/} + do_eqp_test 9.3.$tn.2 $sql2 {/INDEX i2/} +} + + + finish_test From 224155dd1c165a133279ebaa52f80f771844cedc Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 30 Apr 2014 13:19:09 +0000 Subject: [PATCH 15/19] Fix long-standing typos in comments. FossilOrigin-Name: b9f91317c34d07769a95dc2f905a6ccabceb64a3 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/sqliteInt.h | 4 ++-- src/util.c | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index f4b5efc137..44e14c4938 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\sthat\sthe\sdefault\svalues\sused\swhen\ssqlite_stat1\sdata\sis\snot\savailable\sare\scalculated\scorrectly.\sFixes\sfor\sthe\ssame. -D 2014-04-29T19:01:57.481 +C Fix\slong-standing\stypos\sin\scomments. +D 2014-04-30T13:19:09.070 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 63656cfa5a8221c3eb1a182e97d61b1fe2dfd7da +F src/sqliteInt.h b2947801eccefd7ba3e5f14e1353289351a83cf3 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -277,7 +277,7 @@ F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c -F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf +F src/util.c 2b5fb283a190aacdb286f7835a447c45b345b83c F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c 699693bea6710ed436392c928b02cb4e91944137 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d74299f037f3a6a4b3bce8b4d1c76c407c1f3b3e -R e73cccb2b7e246d7f5476a97d396afde -U dan -Z 19f8efc43d85a395f7ff59d4035b6f55 +P e2d42f909de85a0586389f2dc0e654f7af2e351a +R cdeda4c4d8ae1ea7276fce5a8779e48d +U drh +Z 6915f97ffb226384426800091a7da39d diff --git a/manifest.uuid b/manifest.uuid index 92aa1b2d7c..42cd607e10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2d42f909de85a0586389f2dc0e654f7af2e351a \ No newline at end of file +b9f91317c34d07769a95dc2f905a6ccabceb64a3 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b584ff401d..4bd65a690f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -525,10 +525,10 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ ** gives a possible range of values of approximately 1.0e986 to 1e-986. ** But the allowed values are "grainy". Not every value is representable. ** For example, quantities 16 and 17 are both represented by a LogEst -** of 40. However, since LogEst quantatites are suppose to be estimates, +** of 40. However, since LogEst quantaties are suppose to be estimates, ** not exact values, this imprecision is not a problem. ** -** "LogEst" is short for "Logarithimic Estimate". +** "LogEst" is short for "Logarithmic Estimate". ** ** Examples: ** 1 -> 0 20 -> 43 10000 -> 132 diff --git a/src/util.c b/src/util.c index d88c17b759..577d552ac3 100644 --- a/src/util.c +++ b/src/util.c @@ -1246,8 +1246,8 @@ LogEst sqlite3LogEstAdd(LogEst a, LogEst b){ } /* -** Convert an integer into a LogEst. In other words, compute a -** good approximatation for 10*log2(x). +** Convert an integer into a LogEst. In other words, compute an +** approximation for 10*log2(x). */ LogEst sqlite3LogEst(u64 x){ static LogEst a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; From 253666e5203974b28d990e321b243153f3a432dd Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 30 Apr 2014 14:22:38 +0000 Subject: [PATCH 16/19] Improved rendering of LogEst values corresponding to real values near 0.0 in the tool/logest.c utility program. FossilOrigin-Name: 32910c8c595858245bb7ecfe3aa0f90eeae641af --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/logest.c | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 44e14c4938..29bd3a7832 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\slong-standing\stypos\sin\scomments. -D 2014-04-30T13:19:09.070 +C Improved\srendering\sof\sLogEst\svalues\scorresponding\sto\sreal\svalues\snear\s0.0\nin\sthe\stool/logest.c\sutility\sprogram. +D 2014-04-30T14:22:38.579 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1128,7 +1128,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/lemon.c 07aba6270d5a5016ba8107b09e431eea4ecdc123 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc -F tool/logest.c 388c318c7ac8b52b7c08ca1e2de0f4ca9a8f7e81 +F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 F tool/mkkeywordhash.c c9e05e4a7bcab8fab9f583d5b321fb72f565ad97 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e2d42f909de85a0586389f2dc0e654f7af2e351a -R cdeda4c4d8ae1ea7276fce5a8779e48d +P b9f91317c34d07769a95dc2f905a6ccabceb64a3 +R 0e5a317f465f102c0002288cd477a07d U drh -Z 6915f97ffb226384426800091a7da39d +Z 88c640e8cc6f42fa637c9ebb251cd8ac diff --git a/manifest.uuid b/manifest.uuid index 42cd607e10..7421cca676 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9f91317c34d07769a95dc2f905a6ccabceb64a3 \ No newline at end of file +32910c8c595858245bb7ecfe3aa0f90eeae641af \ No newline at end of file diff --git a/tool/logest.c b/tool/logest.c index 1ac337d36c..347fa68a4f 100644 --- a/tool/logest.c +++ b/tool/logest.c @@ -83,7 +83,8 @@ static LogEst logEstFromDouble(double x){ LogEst e; assert( sizeof(x)==8 && sizeof(a)==8 ); if( x<=0.0 ) return -32768; - if( x<1.0 ) return -logEstFromDouble(1/x); + if( x<0.01 ) return -logEstFromDouble(1.0/x); + if( x<1.0 ) return logEstFromDouble(100.0*x) - 66; if( x<1024.0 ) return logEstFromInteger((sqlite3_uint64)(1024.0*x)) - 100; if( x<=2000000000.0 ) return logEstFromInteger((sqlite3_uint64)x); memcpy(&a, &x, 8); @@ -156,8 +157,10 @@ int main(int argc, char **argv){ } } for(i=n-1; i>=0; i--){ - if( a[i]<0 ){ + if( a[i]<-40 ){ printf("%5d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i])); + }else if( a[i]<10 ){ + printf("%5d (%f)\n", a[i], logEstToInt(a[i]+100)/1024.0); }else{ sqlite3_uint64 x = logEstToInt(a[i]+100)*100/1024; printf("%5d (%lld.%02lld)\n", a[i], x/100, x%100); From 4a6b8a05cd203606f227263f9146aa7dff36c888 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 30 Apr 2014 14:47:01 +0000 Subject: [PATCH 17/19] Fix a couple of out-of-date comments in where.c. FossilOrigin-Name: eefeda32d54efbbdf7d20b719299eda48b891fae --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 47 ++++++++++++++++++++++++++++++----------------- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 29bd3a7832..8167b75420 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\srendering\sof\sLogEst\svalues\scorresponding\sto\sreal\svalues\snear\s0.0\nin\sthe\stool/logest.c\sutility\sprogram. -D 2014-04-30T14:22:38.579 +C Fix\sa\scouple\sof\sout-of-date\scomments\sin\swhere.c. +D 2014-04-30T14:47:01.628 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 9651faf05f900d9235bf3166fa54a9e5f12a40f2 +F src/where.c e91b92908c973b2b6b03145dcd131a8ca8b1bf64 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b9f91317c34d07769a95dc2f905a6ccabceb64a3 -R 0e5a317f465f102c0002288cd477a07d -U drh -Z 88c640e8cc6f42fa637c9ebb251cd8ac +P 32910c8c595858245bb7ecfe3aa0f90eeae641af +R 26fff8d56fc56eff56bc1e57cba958ab +U dan +Z b9aa63fbb519903c6b0c5b0a6a8e051f diff --git a/manifest.uuid b/manifest.uuid index 7421cca676..6c9c8ac5b7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -32910c8c595858245bb7ecfe3aa0f90eeae641af \ No newline at end of file +eefeda32d54efbbdf7d20b719299eda48b891fae \ No newline at end of file diff --git a/src/where.c b/src/where.c index aea0be3042..ec7a908b1a 100644 --- a/src/where.c +++ b/src/where.c @@ -4022,8 +4022,13 @@ static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ } /* -** We have so far matched pBuilder->pNew->u.btree.nEq terms of the index pIndex. -** Try to match one more. +** We have so far matched pBuilder->pNew->u.btree.nEq terms of the +** index pIndex. Try to match one more. +** +** When this function is called, pBuilder->pNew->nOut contains the +** number of rows expected to be visited by filtering using the nEq +** terms only. If it is modified, this value is restored before this +** function returns. ** ** If pProbe->tnum==0, that means pIndex is a fake index used for the ** INTEGER PRIMARY KEY. @@ -4085,10 +4090,14 @@ static int whereLoopAddBtreeIndex( /* Consider using a skip-scan if there are no WHERE clause constraints ** available for the left-most terms of the index, and if the average - ** number of repeats in the left-most terms is at least 18. The magic - ** number 18 was found by experimentation to be the payoff point where - ** skip-scan become faster than a full-scan. - */ + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ assert( 42==sqlite3LogEst(18) ); if( pTerm==0 && saved_nEq==saved_nSkip @@ -5226,23 +5235,27 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); if( isOrdered>=0 && isOrdered0 && 66==sqlite3LogEst(100) ); rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66; rSortCost = nRowEst + estLog(nRowEst) + rScale + 16; /* TUNING: The cost of implementing DISTINCT using a B-TREE is - ** also N*log(N) but it has a larger constant of proportionality. - ** Multiply by 3.0. */ + ** similar but with a larger constant of proportionality. + ** Multiply by an additional factor of 3.0. */ if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ rSortCost += 16; } From 75525cbe2423852e551e1843e5fad91acf803024 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 30 Apr 2014 14:53:21 +0000 Subject: [PATCH 18/19] Update a couple of test cases to account for the fact that this branch prefers an index scan and partial sort over a full-table scan and full external sort. FossilOrigin-Name: 9b975bf33cd8fc28c64183a9642bf9fb436a4746 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/cost.test | 5 ++++- test/eqp.test | 12 ++++++------ 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 8167b75420..9f27ec6f9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\sof\sout-of-date\scomments\sin\swhere.c. -D 2014-04-30T14:47:01.628 +C Update\sa\scouple\sof\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sthis\sbranch\sprefers\san\sindex\sscan\sand\spartial\ssort\sover\sa\sfull-table\sscan\sand\sfull\sexternal\ssort. +D 2014-04-30T14:53:21.345 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -407,7 +407,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 41b350fcc811a4fcccffbab06a3a79f33b8ea0de +F test/cost.test 3f7904d623ef8dc6e55f2206db5ce0549077b438 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -454,7 +454,7 @@ F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test 57c6c604c2807fb5531731c5323133453c24afac +F test/eqp.test 90b56d03a93a2e7bb90f88be6083a8ea53f11a0e F test/errmsg.test f31592a594b44ee121371d25ddd5d63497bb3401 F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test c7ebbc756eacf544c108b15eed64d7d4e5f86b75 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 32910c8c595858245bb7ecfe3aa0f90eeae641af -R 26fff8d56fc56eff56bc1e57cba958ab +P eefeda32d54efbbdf7d20b719299eda48b891fae +R 50e654a1c0b26308b8a927bcd26c6421 U dan -Z b9aa63fbb519903c6b0c5b0a6a8e051f +Z f4d62e136a5d6e05753f5050c6fb0c4a diff --git a/manifest.uuid b/manifest.uuid index 6c9c8ac5b7..00e612df5f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eefeda32d54efbbdf7d20b719299eda48b891fae \ No newline at end of file +9b975bf33cd8fc28c64183a9642bf9fb436a4746 \ No newline at end of file diff --git a/test/cost.test b/test/cost.test index e04dc9392a..045a4c08d6 100644 --- a/test/cost.test +++ b/test/cost.test @@ -99,7 +99,10 @@ do_execsql_test 5.1 { do_eqp_test 5.2 { SELECT * FROM t2 ORDER BY x, y; -} {} +} { + 0 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 0 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} +} do_eqp_test 5.3 { SELECT * FROM t2 WHERE x BETWEEN ? AND ? ORDER BY rowid; diff --git a/test/eqp.test b/test/eqp.test index 610ebdc5ca..9e2e9505cc 100644 --- a/test/eqp.test +++ b/test/eqp.test @@ -312,8 +312,8 @@ do_eqp_test 4.2.3 { } { 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2} - 2 0 0 {USE TEMP B-TREE FOR ORDER BY} + 2 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 2 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION)} } do_eqp_test 4.2.4 { @@ -321,8 +321,8 @@ do_eqp_test 4.2.4 { } { 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2} - 2 0 0 {USE TEMP B-TREE FOR ORDER BY} + 2 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 2 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (INTERSECT)} } do_eqp_test 4.2.5 { @@ -330,8 +330,8 @@ do_eqp_test 4.2.5 { } { 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2} - 2 0 0 {USE TEMP B-TREE FOR ORDER BY} + 2 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 2 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (EXCEPT)} } From 8164722c5806410eb5443a2498f619475fee3fff Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 30 Apr 2014 15:00:16 +0000 Subject: [PATCH 19/19] Add text to the header comment of whereLoopAddBtree() describing how the costs of various b-tree loops are estimated. FossilOrigin-Name: 05e6e16cb28c9ffb4596bd2ef81f687c5403ecbb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 23 +++++++++++++++++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 9f27ec6f9b..35f78b46c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sa\scouple\sof\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sthis\sbranch\sprefers\san\sindex\sscan\sand\spartial\ssort\sover\sa\sfull-table\sscan\sand\sfull\sexternal\ssort. -D 2014-04-30T14:53:21.345 +C Add\stext\sto\sthe\sheader\scomment\sof\swhereLoopAddBtree()\sdescribing\show\sthe\scosts\sof\svarious\sb-tree\sloops\sare\sestimated. +D 2014-04-30T15:00:16.197 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c e91b92908c973b2b6b03145dcd131a8ca8b1bf64 +F src/where.c 4aeb1caa0a16c76e0c0566af4c64ba003836a0aa F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P eefeda32d54efbbdf7d20b719299eda48b891fae -R 50e654a1c0b26308b8a927bcd26c6421 +P 9b975bf33cd8fc28c64183a9642bf9fb436a4746 +R 20e49e26218874ddfe5fb9438df8f580 U dan -Z f4d62e136a5d6e05753f5050c6fb0c4a +Z c1e32968c738a6da213a45755f91539c diff --git a/manifest.uuid b/manifest.uuid index 00e612df5f..6ee1deefef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b975bf33cd8fc28c64183a9642bf9fb436a4746 \ No newline at end of file +05e6e16cb28c9ffb4596bd2ef81f687c5403ecbb \ No newline at end of file diff --git a/src/where.c b/src/where.c index ec7a908b1a..7d01db61ee 100644 --- a/src/where.c +++ b/src/where.c @@ -4349,6 +4349,29 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ ** Add all WhereLoop objects for a single table of the join where the table ** is idenfied by pBuilder->pNew->iTab. That table is guaranteed to be ** a b-tree table, not a virtual table. +** +** The costs (WhereLoop.rRun) of the b-tree loops added by this function +** are calculated as follows: +** +** For a full scan, assuming the table (or index) contains nRow rows: +** +** cost = nRow * 3.0 // full-table scan +** cost = nRow * K // scan of covering index +** cost = nRow * (K+3.0) // scan of non-covering index +** +** where K is a value between 1.1 and 3.0 set based on the relative +** estimated average size of the index and table records. +** +** For an index scan, where nVisit is the number of index rows visited +** by the scan, and nSeek is the number of seek operations required on +** the index b-tree: +** +** cost = nSeek * (log(nRow) + K * nVisit) // covering index +** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index +** +** Normally, nSeek is 1. nSeek values greater than 1 come about if the +** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when +** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans. */ static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */