From 9d33d9e7ac064aed9de16d25544c7f45dcb76465 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 10 Mar 2020 01:24:13 +0000 Subject: [PATCH 1/9] Background work for experiments trying to enhance ANALYZE so that it runs off of samples of the entire index and does not need to read the entire index. FossilOrigin-Name: 29d1cc5c3619a88229f18c3c8131228f8a2d151ac3d9203f0c7fc538a996ecec --- manifest | 15 +++++---- manifest.uuid | 2 +- src/analyze.c | 90 +++++++++++++++++++++++++++++---------------------- 3 files changed, 61 insertions(+), 46 deletions(-) diff --git a/manifest b/manifest index 9f3c8dd408..3cb4d4a724 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cleaner\sseparation\sof\sthe\sSTAT4-specific\slogic\sin\sthe\simplementation\sof\nANALYZE. -D 2020-03-09T18:26:11.821 +C Background\swork\sfor\sexperiments\strying\sto\senhance\sANALYZE\sso\sthat\sit\sruns\noff\sof\ssamples\sof\sthe\sentire\sindex\sand\sdoes\snot\sneed\sto\sread\sthe\sentire\nindex. +D 2020-03-10T01:24:13.546 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -466,7 +466,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c f48a4423c8f198d7f1ae4940f74b606707d05384ac79fb219be8e3323af2a2de -F src/analyze.c 831bb090988477a00d3b4c000746e1b0454dcc93b10b793e6ebe1c47f25d193a +F src/analyze.c cc3401286f1ff97aba5f2d94a2e8f9108beeaa25fa67faee090c38e387db396f F src/attach.c fa5addce233a2bb2dfdefeee3b37000e154c47214d3269cab1bb331416e330db F src/auth.c a3d5bfdba83d25abed1013a8c7a5f204e2e29b0c25242a56bc02bb0c07bf1e06 F src/backup.c 5e617c087f1c2d6005c2ec694ce80d6e16bc68d906e1b1c556d7c7c2228b636b @@ -1860,7 +1860,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cab1834cfc71f71bfed3c5170a0ba40a39385c3b2c50b7c6b6f09cc830dd1b1e -R c62e646df7b5e5719c4a0602837bf1ef +P 3df07e5a9a3781a4cf866fc6ee0e5c6f9cd7ca35ce0a6eb3aa7f5f3502e0ffae +R e35bdeff6c928526c7218be9d4bb475b +T *branch * approximate-analyze +T *sym-approximate-analyze * +T -sym-trunk * U drh -Z 8147aa7ce1e35e16be35124cd1704d38 +Z 2efa3c6a2e23594b6e9a59be6c71be4b diff --git a/manifest.uuid b/manifest.uuid index e08451b6b2..a3157116f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3df07e5a9a3781a4cf866fc6ee0e5c6f9cd7ca35ce0a6eb3aa7f5f3502e0ffae \ No newline at end of file +29d1cc5c3619a88229f18c3c8131228f8a2d151ac3d9203f0c7fc538a996ecec \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index eee71fbbde..7db62919bd 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -188,6 +188,11 @@ static void openStatTable( Vdbe *v = sqlite3GetVdbe(pParse); int aRoot[ArraySize(aTable)]; u8 aCreateTbl[ArraySize(aTable)]; +#ifdef SQLITE_ENABLE_STAT4 + const int nToOpen = OptimizationEnabled(db,SQLITE_Stat4) ? 2 : 1; +#else + const int nToOpen = 1; +#endif if( v==0 ) return; assert( sqlite3BtreeHoldsAllMutexes(db) ); @@ -200,8 +205,9 @@ static void openStatTable( for(i=0; izDbSName))==0 ){ - if( aTable[i].zCols ){ + if( iregRoot. This is important @@ -217,7 +223,6 @@ static void openStatTable( ** associated with the table zWhere. If zWhere is NULL, delete the ** entire contents of the table. */ aRoot[i] = pStat->tnum; - aCreateTbl[i] = 0; sqlite3TableLock(pParse, iDb, aRoot[i], 1, zTab); if( zWhere ){ sqlite3NestedParse(pParse, @@ -236,7 +241,7 @@ static void openStatTable( } /* Open the sqlite_stat[134] tables for writing. */ - for(i=0; aTable[i].zCols; i++){ + for(i=0; inCol; i++) sampleClear(p->db, p->aBest+i); - for(i=0; imxSample; i++) sampleClear(p->db, p->a+i); - sampleClear(p->db, &p->current); + if( p->mxSample ){ + int i; + for(i=0; inCol; i++) sampleClear(p->db, p->aBest+i); + for(i=0; imxSample; i++) sampleClear(p->db, p->a+i); + sampleClear(p->db, &p->current); + } #endif sqlite3DbFree(p->db, p); } @@ -400,7 +407,7 @@ static void statInit( int n; /* Bytes of space to allocate */ sqlite3 *db; /* Database connection */ #ifdef SQLITE_ENABLE_STAT4 - int mxSample = SQLITE_STAT4_SAMPLES; + int mxSample = sqlite3_value_int(argv[2]) ? SQLITE_STAT4_SAMPLES : 0; #endif /* Decode the three function arguments */ @@ -437,7 +444,7 @@ static void statInit( p->current.anEq = &p->current.anDLt[nColUp]; #ifdef SQLITE_ENABLE_STAT4 - { + if( mxSample ){ u8 *pSpace; /* Allocated space not yet assigned */ int i; /* Used to iterate through p->aSample[] */ @@ -704,7 +711,7 @@ static void statPush( }else{ /* Second and subsequent calls get processed here */ #ifdef SQLITE_ENABLE_STAT4 - samplePushPrevious(p, iChng); + if( p->mxSample ) samplePushPrevious(p, iChng); #endif /* Update anDLt[], anLt[] and anEq[] to reflect the values that apply @@ -715,26 +722,24 @@ static void statPush( for(i=iChng; inCol; i++){ p->current.anDLt[i]++; #ifdef SQLITE_ENABLE_STAT4 - p->current.anLt[i] += p->current.anEq[i]; + if( p->mxSample ) p->current.anLt[i] += p->current.anEq[i]; #endif p->current.anEq[i] = 1; } } p->nRow++; #ifdef SQLITE_ENABLE_STAT4 - if( sqlite3_value_type(argv[2])==SQLITE_INTEGER ){ - sampleSetRowidInt64(p->db, &p->current, sqlite3_value_int64(argv[2])); - }else{ - sampleSetRowid(p->db, &p->current, sqlite3_value_bytes(argv[2]), - sqlite3_value_blob(argv[2])); - } - p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345; -#endif - -#ifdef SQLITE_ENABLE_STAT4 - { - tRowcnt nLt = p->current.anLt[p->nCol-1]; + if( p->mxSample ){ + tRowcnt nLt; + if( sqlite3_value_type(argv[2])==SQLITE_INTEGER ){ + sampleSetRowidInt64(p->db, &p->current, sqlite3_value_int64(argv[2])); + }else{ + sampleSetRowid(p->db, &p->current, sqlite3_value_bytes(argv[2]), + sqlite3_value_blob(argv[2])); + } + p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345; + nLt = p->current.anLt[p->nCol-1]; /* Check if this is to be a periodic sample. If so, add it. */ if( (nLt/p->nPSample)!=(nLt+1)/p->nPSample ){ p->current.isPSample = 1; @@ -804,6 +809,7 @@ static void statGet( || eCall==STAT_GET_ROWID || eCall==STAT_GET_NLT || eCall==STAT_GET_NDLT ); + assert( eCall==STAT_GET_STAT1 || p->mxSample ); if( eCall==STAT_GET_STAT1 ) #else assert( argc==1 ); @@ -1089,7 +1095,11 @@ static void analyzeOneTable( ** The third argument is only used for STAT4 */ #ifdef SQLITE_ENABLE_STAT4 - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3); + if( OptimizationEnabled(db, SQLITE_Stat4) ){ + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3); + }else{ + sqlite3VdbeAddOp2(v, OP_Integer, 0, regStat4+3); + } #endif sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1); sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2); @@ -1171,21 +1181,23 @@ static void analyzeOneTable( ** if !eof(csr) goto next_row; */ #ifdef SQLITE_ENABLE_STAT4 - assert( regRowid==(regStat4+2) ); - if( HasRowid(pTab) ){ - sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, regRowid); - }else{ - Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); - int j, k, regKey; - regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); - for(j=0; jnKeyCol; j++){ - k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[j]); - assert( k>=0 && knColumn ); - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); - VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); + if( OptimizationEnabled(db, SQLITE_Stat4) ){ + assert( regRowid==(regStat4+2) ); + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, regRowid); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); + int j, k, regKey; + regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); + for(j=0; jnKeyCol; j++){ + k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[j]); + assert( k>=0 && knColumn ); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); + VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid); + sqlite3ReleaseTempRange(pParse, regKey, pPk->nKeyCol); } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid); - sqlite3ReleaseTempRange(pParse, regKey, pPk->nKeyCol); } #endif assert( regChng==(regStat4+1) ); @@ -1206,7 +1218,7 @@ static void analyzeOneTable( /* Add the entries to the stat4 table. */ #ifdef SQLITE_ENABLE_STAT4 - { + if( OptimizationEnabled(db, SQLITE_Stat4) ){ int regEq = regStat1; int regLt = regStat1+1; int regDLt = regStat1+2; From dc4a1687b8b53a4464bbffaf351e25305b6a525e Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 17 Mar 2020 13:07:04 +0000 Subject: [PATCH 2/9] Improved bytecode comments for the ANALYZE command. FossilOrigin-Name: c38ea4139d87535b789f43eb1e38b2dc4b8312352a721035283e67cf0f3d5067 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 994c385818..bc5ec92aad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\schanges\sfrom\strunk. -D 2020-03-17T12:37:27.533 +C Improved\sbytecode\scomments\sfor\sthe\sANALYZE\scommand. +D 2020-03-17T13:07:04.943 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -466,7 +466,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c f48a4423c8f198d7f1ae4940f74b606707d05384ac79fb219be8e3323af2a2de -F src/analyze.c cc3401286f1ff97aba5f2d94a2e8f9108beeaa25fa67faee090c38e387db396f +F src/analyze.c 5f09eb4e93a67653317de64823bda68220d4422efd2a3842baf831d084c9ce13 F src/attach.c fa5addce233a2bb2dfdefeee3b37000e154c47214d3269cab1bb331416e330db F src/auth.c a3d5bfdba83d25abed1013a8c7a5f204e2e29b0c25242a56bc02bb0c07bf1e06 F src/backup.c 5e617c087f1c2d6005c2ec694ce80d6e16bc68d906e1b1c556d7c7c2228b636b @@ -1860,7 +1860,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4945a66237fa3861bc691d5fbef0d40286b596a0902b4f4c032d1d6528bb1a1b 38e3dd389d142e520c71139ec84aa3c7722992af28a5f93a7f16e0ea176b74bb -R eadca1c0864a77f7553dbd5cd11ef9e9 +P 93d710262eb046e2370660b1096ac634373755f92a2e9b1220df3b2bda5f9eeb +R daf5019076e9194243fd0d1204db1bb8 U drh -Z b90799a92dde3b33fc19c32a21522ba6 +Z 540c7862b6fb73b8ea6fa0b5312caee4 diff --git a/manifest.uuid b/manifest.uuid index fe5bcb97ea..0fa8b9df71 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -93d710262eb046e2370660b1096ac634373755f92a2e9b1220df3b2bda5f9eeb \ No newline at end of file +c38ea4139d87535b789f43eb1e38b2dc4b8312352a721035283e67cf0f3d5067 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 7db62919bd..d74687fec1 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1148,6 +1148,7 @@ static void analyzeOneTable( char *pColl = (char*)sqlite3LocateCollSeq(pParse, pIdx->azColl[i]); sqlite3VdbeAddOp2(v, OP_Integer, i, regChng); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp); + VdbeComment((v, "%s.column(%d)", pIdx->zName, i)); aGotoChng[i] = sqlite3VdbeAddOp4(v, OP_Ne, regTemp, 0, regPrev+i, pColl, P4_COLLSEQ); sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); @@ -1168,6 +1169,7 @@ static void analyzeOneTable( for(i=0; izName, i)); } sqlite3VdbeResolveLabel(v, endDistinctTest); sqlite3DbFree(db, aGotoChng); @@ -1193,7 +1195,7 @@ static void analyzeOneTable( k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[j]); assert( k>=0 && knColumn ); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); - VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); + VdbeComment((v, "%s.column(%d)", pIdx->zName, i)); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid); sqlite3ReleaseTempRange(pParse, regKey, pPk->nKeyCol); From e50478d7279b72a9df4836729c9974abfbce08ff Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 17 Mar 2020 13:41:51 +0000 Subject: [PATCH 3/9] Remove the SQLITE_OMIT_BTREECOUNT option. Btree count is required. FossilOrigin-Name: a9bfa47aeea27e91611ba913d33e6635d2016e2c2ab78f9b0657f1bd8933e1a8 --- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- src/analyze.c | 14 ++++++++------ src/btree.c | 2 -- src/btree.h | 2 -- src/ctime.c | 3 --- src/pragma.c | 2 -- src/select.c | 5 +---- src/vdbe.c | 2 -- 9 files changed, 22 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index bc5ec92aad..d9bcf9d7e4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sbytecode\scomments\sfor\sthe\sANALYZE\scommand. -D 2020-03-17T13:07:04.943 +C Remove\sthe\sSQLITE_OMIT_BTREECOUNT\soption.\s\sBtree\scount\sis\srequired. +D 2020-03-17T13:41:51.882 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -466,19 +466,19 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c f48a4423c8f198d7f1ae4940f74b606707d05384ac79fb219be8e3323af2a2de -F src/analyze.c 5f09eb4e93a67653317de64823bda68220d4422efd2a3842baf831d084c9ce13 +F src/analyze.c aaeb41ef74002ffef24363891f3205aa0d544dfc0eb56a859d142fe08f9bc608 F src/attach.c fa5addce233a2bb2dfdefeee3b37000e154c47214d3269cab1bb331416e330db F src/auth.c a3d5bfdba83d25abed1013a8c7a5f204e2e29b0c25242a56bc02bb0c07bf1e06 F src/backup.c 5e617c087f1c2d6005c2ec694ce80d6e16bc68d906e1b1c556d7c7c2228b636b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c 7271a120a66dfd12edcee942443fcd7b3860514a5621cb26a374781af1462117 -F src/btree.h 6111552f19ed7a40f029cf4b33badc6fef9880314fffd80a945f0b7f43ab7471 +F src/btree.c f2e0c75c8321bb4279dcfa6daaf4ac9e15c73ad887f24f7d28f4f2d5f78c02a7 +F src/btree.h 7c0a1c67a378254a6e3cfe29a9d6d2f09fdf58e8f69944076b6dbf517a810887 F src/btreeInt.h dee1a1d0c621524e006bb260bd6b66d5d1867da6fe38cba9ad7b6a9bb9c0c175 F src/build.c 406645db37154920075d90a4ea3c47f33d5f5b6e0769010a54ea8247ee433c1a F src/callback.c d0b853dd413255d2e337b34545e54d888ea02f20da5ad0e63585b389624c4a6c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e -F src/ctime.c 6a77ec9e0eb87aea929e002c816298907e337094a7b556898ae2d1e6be209f90 +F src/ctime.c 060c9bc1c8a848a942326c6e73fff449e54e8e98487f555e49e7b897639db270 F src/date.c 6c408fdd2e9ddf6e8431aba76315a2d061bea2cec8fbb75e25d7c1ba08274712 F src/dbpage.c 8a01e865bf8bc6d7b1844b4314443a6436c07c3efe1d488ed89e81719047833a F src/dbstat.c 0f55297469d4244ab7df395849e1af98eb5e95816af7c661e7d2d8402dea23da @@ -524,14 +524,14 @@ F src/parse.y 8575183809cf30f8c9d1fbea65ca34d1de78b659792bc7c42681e01fc596b520 F src/pcache.c 385ff064bca69789d199a98e2169445dc16e4291fa807babd61d4890c3b34177 F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c 6596e10baf3d8f84cc1585d226cf1ab26564a5f5caf85a15757a281ff977d51a -F src/pragma.c 5fd004b89c77319008ddff6d65dcc83ccca9584d3048f4f66b108b5906a20dba +F src/pragma.c aff7b91cb36e58cee67d5dfc1c3a4b3c02aed104ce3ea1d6082aad2d61b445d7 F src/pragma.h 9473160d220416456b40f27323bb4b316d4e4e08ffbf8bf88c5f7045d49c38e5 F src/prepare.c 8d4d6c8aa6afefc48027c54b41cdf134b4d6bc2fc4badbe483ad7fd9e1728a28 F src/printf.c 9be6945837c839ba57837b4bc3af349eba630920fa5532aa518816defe42a7d4 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 38e3a5636f5bdc92e3683e4cafbba6418c0aa15e0d89ca5b28bd0b621dbb80bf F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 49d46acea0e69311aa891e18253973d63e81da2a9c135924bc827856d33872ad +F src/select.c 04a4138c646b0ecf2ca89142ce5756d2bebed30da15d47e86a69ac59ebbcaafa F src/shell.c.in f76590931c0cbbfef347f44f81ade6b335f80c46bc6e59b8b6114383a8df30e0 F src/sqlite.h.in 802957feeb249ede54f8dfe99b72aa19e70a0b7737969c46e625dc2f9f2d42b0 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -603,7 +603,7 @@ F src/upsert.c 2920de71b20f04fe25eb00b655d086f0ba60ea133c59d7fa3325c49838818e78 F src/utf.c 95fb6e03a5ca679045c5adccd05380f0addccabef5911abddcb06af069500ab7 F src/util.c a285c1e026907b69fa2592bd05047a565a1d8a1aef2b73c924b6a8ffe772871a F src/vacuum.c 813b510ba887fee6492bcb11f2bf77d7eb58b232b83649136372e0a2fc17f4b9 -F src/vdbe.c c1c123c6248fa88940b932a00bcc75056921b6d046d45a82566cb97415d2299c +F src/vdbe.c 1f5e82e73bc8fa158c643ed289a3ae15c0b7cf3c5f75ab96cef5b5f0798fdfee F src/vdbe.h 51282fbe819ee0e8eeeaab176240860d334c20a12b14f3b363e7f1a4e05d60b9 F src/vdbeInt.h a17146053a1aa438474012998fe07e314f3df274a61491ad838ad85d848ac051 F src/vdbeapi.c 1252d80c548711e47a6d84dae88ed4e95d3fbb4e7bd0eaa1347299af7efddf02 @@ -1860,7 +1860,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 93d710262eb046e2370660b1096ac634373755f92a2e9b1220df3b2bda5f9eeb -R daf5019076e9194243fd0d1204db1bb8 +P c38ea4139d87535b789f43eb1e38b2dc4b8312352a721035283e67cf0f3d5067 +R 10acd344478148ed927d0774ca99ede6 U drh -Z 540c7862b6fb73b8ea6fa0b5312caee4 +Z 526c8dc722a58bf8fb8a844ac4513f2a diff --git a/manifest.uuid b/manifest.uuid index 0fa8b9df71..229109d8f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c38ea4139d87535b789f43eb1e38b2dc4b8312352a721035283e67cf0f3d5067 \ No newline at end of file +a9bfa47aeea27e91611ba913d33e6635d2016e2c2ab78f9b0657f1bd8933e1a8 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index d74687fec1..0df9297b7f 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -407,7 +407,8 @@ static void statInit( int n; /* Bytes of space to allocate */ sqlite3 *db; /* Database connection */ #ifdef SQLITE_ENABLE_STAT4 - int mxSample = sqlite3_value_int(argv[2]) ? SQLITE_STAT4_SAMPLES : 0; + /* Maximum number of samples. 0 if STAT4 data is not collected */ + int mxSample = sqlite3_value_int64(argv[2]) ? SQLITE_STAT4_SAMPLES : 0; #endif /* Decode the three function arguments */ @@ -422,13 +423,14 @@ static void statInit( /* Allocate the space required for the StatAccum object */ n = sizeof(*p) + sizeof(tRowcnt)*nColUp /* StatAccum.anEq */ - + sizeof(tRowcnt)*nColUp /* StatAccum.anDLt */ + + sizeof(tRowcnt)*nColUp; /* StatAccum.anDLt */ #ifdef SQLITE_ENABLE_STAT4 - + sizeof(tRowcnt)*nColUp /* StatAccum.anLt */ - + sizeof(StatSample)*(nCol+mxSample) /* StatAccum.aBest[], a[] */ - + sizeof(tRowcnt)*3*nColUp*(nCol+mxSample) + if( mxSample ){ + n += sizeof(tRowcnt)*nColUp /* StatAccum.anLt */ + + sizeof(StatSample)*(nCol+mxSample) /* StatAccum.aBest[], a[] */ + + sizeof(tRowcnt)*3*nColUp*(nCol+mxSample); + } #endif - ; db = sqlite3_context_db_handle(context); p = sqlite3DbMallocZero(db, n); if( p==0 ){ diff --git a/src/btree.c b/src/btree.c index 0839683919..c2103819cc 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9508,7 +9508,6 @@ int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ return rc; } -#ifndef SQLITE_OMIT_BTREECOUNT /* ** The first argument, pCur, is a cursor opened on some b-tree. Count the ** number of entries in the b-tree and write the result to *pnEntry. @@ -9581,7 +9580,6 @@ int sqlite3BtreeCount(sqlite3 *db, BtCursor *pCur, i64 *pnEntry){ /* An error has occurred. Return an error code. */ return rc; } -#endif /* ** Return the pager associated with a BTree. This routine is used for diff --git a/src/btree.h b/src/btree.h index 4bd41f7f37..2085c07677 100644 --- a/src/btree.h +++ b/src/btree.h @@ -336,9 +336,7 @@ int sqlite3BtreeCursorIsValid(BtCursor*); #endif int sqlite3BtreeCursorIsValidNN(BtCursor*); -#ifndef SQLITE_OMIT_BTREECOUNT int sqlite3BtreeCount(sqlite3*, BtCursor*, i64*); -#endif #ifdef SQLITE_TEST int sqlite3BtreeCursorInfo(BtCursor*, int*, int); diff --git a/src/ctime.c b/src/ctime.c index 7a2ace9317..336b912e9e 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -511,9 +511,6 @@ static const char * const sqlite3azCompileOpt[] = { #if SQLITE_OMIT_BLOB_LITERAL "OMIT_BLOB_LITERAL", #endif -#if SQLITE_OMIT_BTREECOUNT - "OMIT_BTREECOUNT", -#endif #if SQLITE_OMIT_CAST "OMIT_CAST", #endif diff --git a/src/pragma.c b/src/pragma.c index c5b5bb6670..e2c625549e 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1729,7 +1729,6 @@ void sqlite3Pragma( } sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); -#ifndef SQLITE_OMIT_BTREECOUNT if( !isQuick ){ sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ @@ -1743,7 +1742,6 @@ void sqlite3Pragma( sqlite3VdbeJumpHere(v, addr); } } -#endif /* SQLITE_OMIT_BTREECOUNT */ } } { diff --git a/src/select.c b/src/select.c index 3128d482a9..fcd2a35eb3 100644 --- a/src/select.c +++ b/src/select.c @@ -6619,7 +6619,6 @@ int sqlite3Select( } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ else { -#ifndef SQLITE_OMIT_BTREECOUNT Table *pTab; if( (pTab = isSimpleCount(p, &sAggInfo))!=0 ){ /* If isSimpleCount() returns a pointer to a Table structure, then @@ -6677,9 +6676,7 @@ int sqlite3Select( sqlite3VdbeAddOp2(v, OP_Count, iCsr, sAggInfo.aFunc[0].iMem); sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); - }else -#endif /* SQLITE_OMIT_BTREECOUNT */ - { + }else{ int regAcc = 0; /* "populate accumulators" flag */ /* If there are accumulator registers but no min() or max() functions diff --git a/src/vdbe.c b/src/vdbe.c index 7f05328660..246442b479 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3193,7 +3193,6 @@ case OP_MakeRecord: { ** Store the number of entries (an integer value) in the table or index ** opened by cursor P1 in register P2 */ -#ifndef SQLITE_OMIT_BTREECOUNT case OP_Count: { /* out2 */ i64 nEntry; BtCursor *pCrsr; @@ -3208,7 +3207,6 @@ case OP_Count: { /* out2 */ pOut->u.i = nEntry; goto check_for_interrupt; } -#endif /* Opcode: Savepoint P1 * * P4 * ** From 9f27463684be9e40072ba7a07b769d15e578a571 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 17 Mar 2020 17:11:23 +0000 Subject: [PATCH 4/9] Provide an estimated row count to stat_init() for STAT1 analysis. FossilOrigin-Name: 714419fe85cfdad22979183a94e4569c87740652758ab76b646753cf2b013b54 --- manifest | 14 +++++------ manifest.uuid | 2 +- src/analyze.c | 65 +++++++++++++++++++++++++-------------------------- src/vdbe.c | 18 ++++++++++---- 4 files changed, 53 insertions(+), 46 deletions(-) diff --git a/manifest b/manifest index d9bcf9d7e4..0d201b4bd7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sSQLITE_OMIT_BTREECOUNT\soption.\s\sBtree\scount\sis\srequired. -D 2020-03-17T13:41:51.882 +C Provide\san\sestimated\srow\scount\sto\sstat_init()\sfor\sSTAT1\sanalysis. +D 2020-03-17T17:11:23.756 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -466,7 +466,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c f48a4423c8f198d7f1ae4940f74b606707d05384ac79fb219be8e3323af2a2de -F src/analyze.c aaeb41ef74002ffef24363891f3205aa0d544dfc0eb56a859d142fe08f9bc608 +F src/analyze.c 0df49eed25e472ef00bfe12184548a5a51890e7cd650c40fe2681430bdcae9d1 F src/attach.c fa5addce233a2bb2dfdefeee3b37000e154c47214d3269cab1bb331416e330db F src/auth.c a3d5bfdba83d25abed1013a8c7a5f204e2e29b0c25242a56bc02bb0c07bf1e06 F src/backup.c 5e617c087f1c2d6005c2ec694ce80d6e16bc68d906e1b1c556d7c7c2228b636b @@ -603,7 +603,7 @@ F src/upsert.c 2920de71b20f04fe25eb00b655d086f0ba60ea133c59d7fa3325c49838818e78 F src/utf.c 95fb6e03a5ca679045c5adccd05380f0addccabef5911abddcb06af069500ab7 F src/util.c a285c1e026907b69fa2592bd05047a565a1d8a1aef2b73c924b6a8ffe772871a F src/vacuum.c 813b510ba887fee6492bcb11f2bf77d7eb58b232b83649136372e0a2fc17f4b9 -F src/vdbe.c 1f5e82e73bc8fa158c643ed289a3ae15c0b7cf3c5f75ab96cef5b5f0798fdfee +F src/vdbe.c 45896ffb6241d1b001a73a84a480af25036de89bab4a74ed1814d9020384d420 F src/vdbe.h 51282fbe819ee0e8eeeaab176240860d334c20a12b14f3b363e7f1a4e05d60b9 F src/vdbeInt.h a17146053a1aa438474012998fe07e314f3df274a61491ad838ad85d848ac051 F src/vdbeapi.c 1252d80c548711e47a6d84dae88ed4e95d3fbb4e7bd0eaa1347299af7efddf02 @@ -1860,7 +1860,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c38ea4139d87535b789f43eb1e38b2dc4b8312352a721035283e67cf0f3d5067 -R 10acd344478148ed927d0774ca99ede6 +P a9bfa47aeea27e91611ba913d33e6635d2016e2c2ab78f9b0657f1bd8933e1a8 +R 83170381358df6124fa4b6f4f7b0bc2e U drh -Z 526c8dc722a58bf8fb8a844ac4513f2a +Z b22aba55e2c8d6bfe6ccc85eac867e5f diff --git a/manifest.uuid b/manifest.uuid index 229109d8f9..73b2e21252 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a9bfa47aeea27e91611ba913d33e6635d2016e2c2ab78f9b0657f1bd8933e1a8 \ No newline at end of file +714419fe85cfdad22979183a94e4569c87740652758ab76b646753cf2b013b54 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 0df9297b7f..a7f70102b1 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -280,7 +280,8 @@ struct StatSample { }; struct StatAccum { sqlite3 *db; /* Database connection, for malloc() */ - tRowcnt nRow; /* Number of rows in the entire table */ + tRowcnt nEst; /* Estimated number of rows */ + tRowcnt nRow; /* Number of rows visited so far */ int nCol; /* Number of columns in index + pk/rowid */ int nKeyCol; /* Number of index columns w/o the pk/rowid */ StatSample current; /* Current row as a StatSample */ @@ -377,14 +378,12 @@ static void statAccumDestructor(void *pOld){ ** are: ** N: The number of columns in the index including the rowid/pk (note 1) ** K: The number of columns in the index excluding the rowid/pk. -** C: The number of rows in the index (note 2) +** C: Estimated number of rows in the index ** ** Note 1: In the special case of the covering index that implements a ** WITHOUT ROWID table, N is the number of PRIMARY KEY columns, not the ** total number of columns in the table. ** -** Note 2: C is only used for STAT4. -** ** For indexes on ordinary rowid tables, N==K+1. But for indexes on ** WITHOUT ROWID tables, N=K+P where P is the number of columns in the ** PRIMARY KEY of the table. The covering index that implements the @@ -439,6 +438,7 @@ static void statInit( } p->db = db; + p->nEst = sqlite3_value_int64(argv[2]); p->nRow = 0; p->nCol = nCol; p->nKeyCol = nKeyCol; @@ -452,7 +452,7 @@ static void statInit( p->iGet = -1; p->mxSample = mxSample; - p->nPSample = (tRowcnt)(sqlite3_value_int64(argv[2])/(mxSample/3+1) + 1); + p->nPSample = (tRowcnt)(p->nEst/(mxSample/3+1) + 1); p->current.anLt = &p->current.anEq[nColUp]; p->iPrn = 0x689e962d*(u32)nCol ^ 0xd0944565*(u32)sqlite3_value_int(argv[2]); @@ -923,16 +923,16 @@ static const FuncDef statGetFuncdef = { {0} }; -static void callStatGet(Parse *pParse, int regStat4, int iParam, int regOut){ +static void callStatGet(Parse *pParse, int regStat, int iParam, int regOut){ #ifdef SQLITE_ENABLE_STAT4 - sqlite3VdbeAddOp2(pParse->pVdbe, OP_Integer, iParam, regStat4+1); + sqlite3VdbeAddOp2(pParse->pVdbe, OP_Integer, iParam, regStat+1); #elif SQLITE_DEBUG assert( iParam==STAT_GET_STAT1 ); #else UNUSED_PARAMETER( iParam ); #endif - assert( regOut!=regStat4 && regOut!=regStat4+1 ); - sqlite3VdbeAddFunctionCall(pParse, 0, regStat4, regOut, 1+IsStat4, + assert( regOut!=regStat && regOut!=regStat+1 ); + sqlite3VdbeAddFunctionCall(pParse, 0, regStat, regOut, 1+IsStat4, &statGetFuncdef, 0); } @@ -958,11 +958,9 @@ static void analyzeOneTable( int iDb; /* Index of database containing pTab */ u8 needTableCnt = 1; /* True to count the table */ int regNewRowid = iMem++; /* Rowid for the inserted record */ - int regStat4 = iMem++; /* Register to hold StatAccum object */ + int regStat = iMem++; /* Register to hold StatAccum object */ int regChng = iMem++; /* Index of changed index field */ -#ifdef SQLITE_ENABLE_STAT4 int regRowid = iMem++; /* Rowid argument passed to stat_push() */ -#endif int regTemp = iMem++; /* Temporary use register */ int regTabname = iMem++; /* Register containing table name */ int regIdxname = iMem++; /* Register containing index name */ @@ -1091,21 +1089,24 @@ static void analyzeOneTable( ** (1) the number of columns in the index including the rowid ** (or for a WITHOUT ROWID table, the number of PK columns), ** (2) the number of columns in the key without the rowid/pk - ** (3) the number of rows in the index, - ** - ** - ** The third argument is only used for STAT4 + ** (3) estimated number of rows in the index, */ + sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat+1); + assert( regRowid==regStat+2 ); + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regRowid); #ifdef SQLITE_ENABLE_STAT4 if( OptimizationEnabled(db, SQLITE_Stat4) ){ - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3); - }else{ - sqlite3VdbeAddOp2(v, OP_Integer, 0, regStat4+3); - } + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regTemp); + addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); + }else #endif - sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1); - sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2); - sqlite3VdbeAddFunctionCall(pParse, 0, regStat4+1, regStat4, 2+IsStat4, + { + addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, 1); + } + sqlite3VdbeAddFunctionCall(pParse, 0, regStat+1, regStat, 3, &statInitFuncdef, 0); /* Implementation of the following: @@ -1116,8 +1117,6 @@ static void analyzeOneTable( ** goto next_push_0; ** */ - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); - VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); addrNextRow = sqlite3VdbeCurrentAddr(v); @@ -1186,7 +1185,7 @@ static void analyzeOneTable( */ #ifdef SQLITE_ENABLE_STAT4 if( OptimizationEnabled(db, SQLITE_Stat4) ){ - assert( regRowid==(regStat4+2) ); + assert( regRowid==(regStat+2) ); if( HasRowid(pTab) ){ sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, regRowid); }else{ @@ -1204,13 +1203,13 @@ static void analyzeOneTable( } } #endif - assert( regChng==(regStat4+1) ); - sqlite3VdbeAddFunctionCall(pParse, 1, regStat4, regTemp, 2+IsStat4, + assert( regChng==(regStat+1) ); + sqlite3VdbeAddFunctionCall(pParse, 1, regStat, regTemp, 2+IsStat4, &statPushFuncdef, 0); sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); /* Add the entry to the stat1 table. */ - callStatGet(pParse, regStat4, STAT_GET_STAT1, regStat1); + callStatGet(pParse, regStat, STAT_GET_STAT1, regStat1); assert( "BBB"[0]==SQLITE_AFF_TEXT ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); @@ -1236,12 +1235,12 @@ static void analyzeOneTable( pParse->nMem = MAX(pParse->nMem, regCol+nCol); addrNext = sqlite3VdbeCurrentAddr(v); - callStatGet(pParse, regStat4, STAT_GET_ROWID, regSampleRowid); + callStatGet(pParse, regStat, STAT_GET_ROWID, regSampleRowid); addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, regSampleRowid); VdbeCoverage(v); - callStatGet(pParse, regStat4, STAT_GET_NEQ, regEq); - callStatGet(pParse, regStat4, STAT_GET_NLT, regLt); - callStatGet(pParse, regStat4, STAT_GET_NDLT, regDLt); + callStatGet(pParse, regStat, STAT_GET_NEQ, regEq); + callStatGet(pParse, regStat, STAT_GET_NLT, regLt); + callStatGet(pParse, regStat, STAT_GET_NDLT, regDLt); sqlite3VdbeAddOp4Int(v, seekOp, iTabCur, addrNext, regSampleRowid, 0); VdbeCoverage(v); for(i=0; iapCsr[pOp->p1]->eCurType==CURTYPE_BTREE ); pCrsr = p->apCsr[pOp->p1]->uc.pCursor; assert( pCrsr ); - nEntry = 0; /* Not needed. Only used to silence a warning. */ - rc = sqlite3BtreeCount(db, pCrsr, &nEntry); - if( rc ) goto abort_due_to_error; + if( pOp->p3 ){ + nEntry = sqlite3BtreeRowCountEst(pCrsr); + }else{ + nEntry = 0; /* Not needed. Only used to silence a warning. */ + rc = sqlite3BtreeCount(db, pCrsr, &nEntry); + if( rc ) goto abort_due_to_error; + } pOut = out2Prerelease(p, pOp); pOut->u.i = nEntry; goto check_for_interrupt; From 59a8cb79316815f4eeb48e17252723df39589dda Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 18 Mar 2020 14:43:05 +0000 Subject: [PATCH 5/9] Add the SQLITE_ANALYZE_LIMIT compile-time option (expected to be temporary) that sets a threshold at which ANALYZE starts to use approximations during the analysis process. FossilOrigin-Name: a773fd4698d474fda5e57bc77ed66a79cf74efee2706f43f6def6f450bfd1fc0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 54 ++++++++++++++++++++++++++++++++++++++++----------- 3 files changed, 50 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 0d201b4bd7..411043d48a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Provide\san\sestimated\srow\scount\sto\sstat_init()\sfor\sSTAT1\sanalysis. -D 2020-03-17T17:11:23.756 +C Add\sthe\sSQLITE_ANALYZE_LIMIT\scompile-time\soption\s(expected\sto\sbe\stemporary)\nthat\ssets\sa\sthreshold\sat\swhich\sANALYZE\sstarts\sto\suse\sapproximations\sduring\nthe\sanalysis\sprocess. +D 2020-03-18T14:43:05.229 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -466,7 +466,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c f48a4423c8f198d7f1ae4940f74b606707d05384ac79fb219be8e3323af2a2de -F src/analyze.c 0df49eed25e472ef00bfe12184548a5a51890e7cd650c40fe2681430bdcae9d1 +F src/analyze.c 2ae3d2e13387eac29ffb4f94279e00ec1f96145f56a06a8aa57533bd0c64bd99 F src/attach.c fa5addce233a2bb2dfdefeee3b37000e154c47214d3269cab1bb331416e330db F src/auth.c a3d5bfdba83d25abed1013a8c7a5f204e2e29b0c25242a56bc02bb0c07bf1e06 F src/backup.c 5e617c087f1c2d6005c2ec694ce80d6e16bc68d906e1b1c556d7c7c2228b636b @@ -1860,7 +1860,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a9bfa47aeea27e91611ba913d33e6635d2016e2c2ab78f9b0657f1bd8933e1a8 -R 83170381358df6124fa4b6f4f7b0bc2e +P 714419fe85cfdad22979183a94e4569c87740652758ab76b646753cf2b013b54 +R 9e076c207b1e065d9ce5cb1bfdf9885a U drh -Z b22aba55e2c8d6bfe6ccc85eac867e5f +Z 74d44bcaa72564678d221e098ce4dbd2 diff --git a/manifest.uuid b/manifest.uuid index 73b2e21252..df0df117b5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -714419fe85cfdad22979183a94e4569c87740652758ab76b646753cf2b013b54 \ No newline at end of file +a773fd4698d474fda5e57bc77ed66a79cf74efee2706f43f6def6f450bfd1fc0 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index a7f70102b1..7290deae19 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -284,6 +284,7 @@ struct StatAccum { tRowcnt nRow; /* Number of rows visited so far */ int nCol; /* Number of columns in index + pk/rowid */ int nKeyCol; /* Number of index columns w/o the pk/rowid */ + u8 nSkipAhead; /* Number of times of skip-ahead */ StatSample current; /* Current row as a StatSample */ #ifdef SQLITE_ENABLE_STAT4 tRowcnt nPSample; /* How often to do a periodic sample */ @@ -404,10 +405,10 @@ static void statInit( int nKeyCol; /* Number of key columns */ int nColUp; /* nCol rounded up for alignment */ int n; /* Bytes of space to allocate */ - sqlite3 *db; /* Database connection */ + sqlite3 *db = sqlite3_context_db_handle(context); /* Database connection */ #ifdef SQLITE_ENABLE_STAT4 /* Maximum number of samples. 0 if STAT4 data is not collected */ - int mxSample = sqlite3_value_int64(argv[2]) ? SQLITE_STAT4_SAMPLES : 0; + int mxSample = OptimizationEnabled(db,SQLITE_Stat4) ?SQLITE_STAT4_SAMPLES :0; #endif /* Decode the three function arguments */ @@ -442,16 +443,17 @@ static void statInit( p->nRow = 0; p->nCol = nCol; p->nKeyCol = nKeyCol; + p->nSkipAhead = 0; p->current.anDLt = (tRowcnt*)&p[1]; p->current.anEq = &p->current.anDLt[nColUp]; #ifdef SQLITE_ENABLE_STAT4 + p->mxSample = mxSample; if( mxSample ){ u8 *pSpace; /* Allocated space not yet assigned */ int i; /* Used to iterate through p->aSample[] */ p->iGet = -1; - p->mxSample = mxSample; p->nPSample = (tRowcnt)(p->nEst/(mxSample/3+1) + 1); p->current.anLt = &p->current.anEq[nColUp]; p->iPrn = 0x689e962d*(u32)nCol ^ 0xd0944565*(u32)sqlite3_value_int(argv[2]); @@ -675,6 +677,15 @@ static void samplePushPrevious(StatAccum *p, int iChng){ } #endif /* SQLITE_ENABLE_STAT4 */ +/* +** A limit on the number of rows of an index that will be examined +** by ANALYZE before it starts going with approximations. Zero means +** "no limit". +*/ +#ifndef SQLITE_ANALYZE_LIMIT +# define SQLITE_ANALYZE_LIMIT 0 +#endif + /* ** Implementation of the stat_push SQL function: stat_push(P,C,R) ** Arguments: @@ -684,10 +695,13 @@ static void samplePushPrevious(StatAccum *p, int iChng){ ** R Rowid for the current row. Might be a key record for ** WITHOUT ROWID tables. ** -** This SQL function always returns NULL. It's purpose it to accumulate -** statistical data and/or samples in the StatAccum object about the -** index being analyzed. The stat_get() SQL function will later be used to -** extract relevant information for constructing the sqlite_statN tables. +** The purpose of this routine is to collect statistical data and/or +** samples from the index being analyzed into the StatAccum object. +** The stat_get() SQL function will be used afterwards to +** retrieve the information gathered. +** +** This SQL function usually returns NULL, but might return an integer +** if it wants the byte-code to do special processing. ** ** The R parameter is only used for STAT4 */ @@ -729,6 +743,7 @@ static void statPush( p->current.anEq[i] = 1; } } + p->nRow++; #ifdef SQLITE_ENABLE_STAT4 if( p->mxSample ){ @@ -757,9 +772,16 @@ static void statPush( sampleCopy(p, &p->aBest[i], &p->current); } } + }else +#endif +#if SQLITE_ANALYZE_LIMIT + if( p->nRow>SQLITE_ANALYZE_LIMIT*(p->nSkipAhead+1) ){ + p->nSkipAhead++; + sqlite3_result_int(context, p->current.anDLt[0]>0); } #endif } + static const FuncDef statPushFuncdef = { 2+IsStat4, /* nArg */ SQLITE_UTF8, /* funcFlags */ @@ -847,7 +869,8 @@ static void statGet( return; } - sqlite3_snprintf(24, zRet, "%llu", (u64)p->nRow); + sqlite3_snprintf(24, zRet, "%llu", + p->nSkipAhead ? (u64)p->nEst : (u64)p->nRow); z = zRet + sqlite3Strlen30(zRet); for(i=0; inKeyCol; i++){ u64 nDistinct = p->current.anDLt[i] + 1; @@ -1204,9 +1227,18 @@ static void analyzeOneTable( } #endif assert( regChng==(regStat+1) ); - sqlite3VdbeAddFunctionCall(pParse, 1, regStat, regTemp, 2+IsStat4, - &statPushFuncdef, 0); - sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); + { + int j1, j2, j3; + sqlite3VdbeAddFunctionCall(pParse, 1, regStat, regTemp, 2+IsStat4, + &statPushFuncdef, 0); + j1 = sqlite3VdbeAddOp1(v, OP_IsNull, regTemp); + j2 = sqlite3VdbeAddOp1(v, OP_If, regTemp); + j3 = sqlite3VdbeAddOp4Int(v, OP_SeekGT, iIdxCur, 0, regPrev, 1); + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, j2); + sqlite3VdbeJumpHere(v, j3); + } /* Add the entry to the stat1 table. */ callStatGet(pParse, regStat, STAT_GET_STAT1, regStat1); From 78a50d728093c4f5d28ffeb308d0e366ba3ffb40 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 18 Mar 2020 15:58:13 +0000 Subject: [PATCH 6/9] Fix the build for when SQLITE_ENABLE_STAT4 is defined. FossilOrigin-Name: 8f0a8c2aa45f7cf7339094d83893aeb046b010b5b97bb4dae99ac07a8ebf2fa6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 411043d48a..0ae75e6bd2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_ANALYZE_LIMIT\scompile-time\soption\s(expected\sto\sbe\stemporary)\nthat\ssets\sa\sthreshold\sat\swhich\sANALYZE\sstarts\sto\suse\sapproximations\sduring\nthe\sanalysis\sprocess. -D 2020-03-18T14:43:05.229 +C Fix\sthe\sbuild\sfor\swhen\sSQLITE_ENABLE_STAT4\sis\sdefined. +D 2020-03-18T15:58:13.804 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -466,7 +466,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c f48a4423c8f198d7f1ae4940f74b606707d05384ac79fb219be8e3323af2a2de -F src/analyze.c 2ae3d2e13387eac29ffb4f94279e00ec1f96145f56a06a8aa57533bd0c64bd99 +F src/analyze.c 3d90f56656f5c6e7f835659b46ccf517c156df5222be86b4ff279eb476e8e373 F src/attach.c fa5addce233a2bb2dfdefeee3b37000e154c47214d3269cab1bb331416e330db F src/auth.c a3d5bfdba83d25abed1013a8c7a5f204e2e29b0c25242a56bc02bb0c07bf1e06 F src/backup.c 5e617c087f1c2d6005c2ec694ce80d6e16bc68d906e1b1c556d7c7c2228b636b @@ -1860,7 +1860,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 714419fe85cfdad22979183a94e4569c87740652758ab76b646753cf2b013b54 -R 9e076c207b1e065d9ce5cb1bfdf9885a +P a773fd4698d474fda5e57bc77ed66a79cf74efee2706f43f6def6f450bfd1fc0 +R 6430bd4bd4bb467e3eec7a041f966c13 U drh -Z 74d44bcaa72564678d221e098ce4dbd2 +Z 544d5db127f99a945af7a33bac1ef578 diff --git a/manifest.uuid b/manifest.uuid index df0df117b5..a8f4744c32 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a773fd4698d474fda5e57bc77ed66a79cf74efee2706f43f6def6f450bfd1fc0 \ No newline at end of file +8f0a8c2aa45f7cf7339094d83893aeb046b010b5b97bb4dae99ac07a8ebf2fa6 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 7290deae19..4414c50807 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -779,6 +779,8 @@ static void statPush( p->nSkipAhead++; sqlite3_result_int(context, p->current.anDLt[0]>0); } +#else + {} #endif } From 49a76a8fe5272426fee553d10111e5cf0babd571 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Mar 2020 20:57:06 +0000 Subject: [PATCH 7/9] Add "PRAGMA analysis_limit=N;" to limit the number of rows visited by ANALYZE when N is positive. Positive N also disables collecting stat4. FossilOrigin-Name: a279b151c1623807774daf4975175c62ea252eefb71f9820ced6773769b392c5 --- manifest | 20 +++++----- manifest.uuid | 2 +- src/analyze.c | 49 ++++++++++++----------- src/pragma.c | 19 +++++++++ src/pragma.h | 94 +++++++++++++++++++++++--------------------- src/sqliteInt.h | 1 + tool/mkpragmatab.tcl | 3 ++ 7 files changed, 109 insertions(+), 79 deletions(-) diff --git a/manifest b/manifest index d4b41459ae..b3aa1c4102 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\senhancements\sfrom\strunk. -D 2020-03-31T18:41:21.038 +C Add\s"PRAGMA\sanalysis_limit=N;"\sto\slimit\sthe\snumber\sof\srows\svisited\sby\nANALYZE\swhen\sN\sis\spositive.\s\sPositive\sN\salso\sdisables\scollecting\sstat4. +D 2020-03-31T20:57:06.787 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -466,7 +466,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c f48a4423c8f198d7f1ae4940f74b606707d05384ac79fb219be8e3323af2a2de -F src/analyze.c 3d90f56656f5c6e7f835659b46ccf517c156df5222be86b4ff279eb476e8e373 +F src/analyze.c 4b139fa045d17192ba11ee97e6123cc011efbae690ec302f5d1eb53fd52d5ded F src/attach.c ff2daea0fe62080192e3f262670e4f61f5a86c1e7bea9cec34e960fe79852aa1 F src/auth.c a3d5bfdba83d25abed1013a8c7a5f204e2e29b0c25242a56bc02bb0c07bf1e06 F src/backup.c 5e617c087f1c2d6005c2ec694ce80d6e16bc68d906e1b1c556d7c7c2228b636b @@ -524,8 +524,8 @@ F src/parse.y c8eff38606f443d5ba245263fa7abc05e4116d95656e050c4b78e9bfbf931add F src/pcache.c 385ff064bca69789d199a98e2169445dc16e4291fa807babd61d4890c3b34177 F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c 6596e10baf3d8f84cc1585d226cf1ab26564a5f5caf85a15757a281ff977d51a -F src/pragma.c aff7b91cb36e58cee67d5dfc1c3a4b3c02aed104ce3ea1d6082aad2d61b445d7 -F src/pragma.h 9473160d220416456b40f27323bb4b316d4e4e08ffbf8bf88c5f7045d49c38e5 +F src/pragma.c 85f763714b192cc26a236e3ec020a9155fe8da248af21eae86c0d1f0f49a0753 +F src/pragma.h 8168e588536bffd95319451f34e9a754dc37d205ebe433031a7813c5b286beae F src/prepare.c 8d4d6c8aa6afefc48027c54b41cdf134b4d6bc2fc4badbe483ad7fd9e1728a28 F src/printf.c 9be6945837c839ba57837b4bc3af349eba630920fa5532aa518816defe42a7d4 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 @@ -536,7 +536,7 @@ F src/shell.c.in 7bb9005bf876c4e1210257a63fa49b556f4eddf59f94b6eb310fcb5096bec0e F src/sqlite.h.in cc7d0949ac32bb68ed97acdb3e7ae91cd413a24d32d6ff049ef8308d620a4367 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9c5269260409eb3275324ccace6a13a96f4ad330c708415f70ca6097901ff4ee -F src/sqliteInt.h b0165686885990e29622f56b2f95b93c12c6f84be4cf1ee60797d80a0dcd7f15 +F src/sqliteInt.h 7bab5c24cd05f8173e1a30917857f42d8e1fbb834e9ea9c02ca23b50383e209d F src/sqliteLimit.h 95cb8479ca459496d9c1c6a9f76b38aee12203a56ce1092fe13e50ae2454c032 F src/status.c 9ff2210207c6c3b4d9631a8241a7d45ab1b26a0e9c84cb07a9b5ce2de9a3b278 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1793,7 +1793,7 @@ F tool/mkmsvcmin.tcl 6ecab9fe22c2c8de4d82d4c46797bda3d2deac8e763885f5a38d0c44a89 F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl 352a4319c0ad869eb26442bf7c3b015aa15594c21f1cce5a6420dbe999367c21 F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa -F tool/mkpragmatab.tcl 62663c65d9191aada624a787e1ee3420f268a3c27999ad0ffb77a6918ddc1e52 +F tool/mkpragmatab.tcl d348a4bf71ac068bddf89326562071cbbd962273d88f9b5e5d622f3e73b78bdf F tool/mkshellc.tcl 70a9978e363b0f3280ca9ce1c46d72563ff479c1930a12a7375e3881b7325712 F tool/mksourceid.c 36aa8020014aed0836fd13c51d6dc9219b0df1761d6b5f58ff5b616211b079b9 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 @@ -1860,7 +1860,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8f0a8c2aa45f7cf7339094d83893aeb046b010b5b97bb4dae99ac07a8ebf2fa6 a49f8ec552bede7da731e0571ccf49de1a30e7be3a5673150436c8b411ba6ffc -R f330422722a52c77c50a3e7d8a9da878 +P c705ce266ad25af71791035590875f0ea9f2c72826b3eda17f065d2bf091de92 +R 89edd918806457e430236657da896c8f U drh -Z 2ae26fc9e4b49bbb8102a35ce551a2aa +Z bd47b9db8446ad6b70c7b3bf3d82180a diff --git a/manifest.uuid b/manifest.uuid index c079a7d71d..788d4b8710 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c705ce266ad25af71791035590875f0ea9f2c72826b3eda17f065d2bf091de92 \ No newline at end of file +a279b151c1623807774daf4975175c62ea252eefb71f9820ced6773769b392c5 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 4414c50807..2cad81817b 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -282,6 +282,7 @@ struct StatAccum { sqlite3 *db; /* Database connection, for malloc() */ tRowcnt nEst; /* Estimated number of rows */ tRowcnt nRow; /* Number of rows visited so far */ + int nLimit; /* Analysis row-scan limit */ int nCol; /* Number of columns in index + pk/rowid */ int nKeyCol; /* Number of index columns w/o the pk/rowid */ u8 nSkipAhead; /* Number of times of skip-ahead */ @@ -375,11 +376,12 @@ static void statAccumDestructor(void *pOld){ } /* -** Implementation of the stat_init(N,K,C) SQL function. The three parameters +** Implementation of the stat_init(N,K,C,L) SQL function. The four parameters ** are: ** N: The number of columns in the index including the rowid/pk (note 1) ** K: The number of columns in the index excluding the rowid/pk. ** C: Estimated number of rows in the index +** L: A limit on the number of rows to scan, or 0 for no-limit ** ** Note 1: In the special case of the covering index that implements a ** WITHOUT ROWID table, N is the number of PRIMARY KEY columns, not the @@ -441,6 +443,11 @@ static void statInit( p->db = db; p->nEst = sqlite3_value_int64(argv[2]); p->nRow = 0; + p->nLimit = sqlite3_value_int64(argv[3]); +#ifdef SQLITE_ENABLE_STAT4 + /* Disable STAT4 statistics gathering if there is a analysis_limit set */ + if( p->nLimit>0 ) p->mxSample = 0; +#endif p->nCol = nCol; p->nKeyCol = nKeyCol; p->nSkipAhead = 0; @@ -677,15 +684,6 @@ static void samplePushPrevious(StatAccum *p, int iChng){ } #endif /* SQLITE_ENABLE_STAT4 */ -/* -** A limit on the number of rows of an index that will be examined -** by ANALYZE before it starts going with approximations. Zero means -** "no limit". -*/ -#ifndef SQLITE_ANALYZE_LIMIT -# define SQLITE_ANALYZE_LIMIT 0 -#endif - /* ** Implementation of the stat_push SQL function: stat_push(P,C,R) ** Arguments: @@ -774,14 +772,10 @@ static void statPush( } }else #endif -#if SQLITE_ANALYZE_LIMIT - if( p->nRow>SQLITE_ANALYZE_LIMIT*(p->nSkipAhead+1) ){ + if( p->nLimit && p->nRow>p->nLimit*(p->nSkipAhead+1) ){ p->nSkipAhead++; sqlite3_result_int(context, p->current.anDLt[0]>0); } -#else - {} -#endif } static const FuncDef statPushFuncdef = { @@ -987,6 +981,7 @@ static void analyzeOneTable( int regChng = iMem++; /* Index of changed index field */ int regRowid = iMem++; /* Rowid argument passed to stat_push() */ int regTemp = iMem++; /* Temporary use register */ + int regTemp2 = iMem++; /* Second temporary use register */ int regTabname = iMem++; /* Register containing table name */ int regIdxname = iMem++; /* Register containing index name */ int regStat1 = iMem++; /* Value for the stat column of sqlite_stat1 */ @@ -1131,7 +1126,9 @@ static void analyzeOneTable( VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, 1); } - sqlite3VdbeAddFunctionCall(pParse, 0, regStat+1, regStat, 3, + assert( regTemp2==regStat+4 ); + sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + sqlite3VdbeAddFunctionCall(pParse, 0, regStat+1, regStat, 4, &statInitFuncdef, 0); /* Implementation of the following: @@ -1230,16 +1227,20 @@ static void analyzeOneTable( #endif assert( regChng==(regStat+1) ); { - int j1, j2, j3; sqlite3VdbeAddFunctionCall(pParse, 1, regStat, regTemp, 2+IsStat4, &statPushFuncdef, 0); - j1 = sqlite3VdbeAddOp1(v, OP_IsNull, regTemp); - j2 = sqlite3VdbeAddOp1(v, OP_If, regTemp); - j3 = sqlite3VdbeAddOp4Int(v, OP_SeekGT, iIdxCur, 0, regPrev, 1); - sqlite3VdbeJumpHere(v, j1); - sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); - sqlite3VdbeJumpHere(v, j2); - sqlite3VdbeJumpHere(v, j3); + if( db->nAnalysisLimit ){ + int j1, j2, j3; + j1 = sqlite3VdbeAddOp1(v, OP_IsNull, regTemp); VdbeCoverage(v); + j2 = sqlite3VdbeAddOp1(v, OP_If, regTemp); VdbeCoverage(v); + j3 = sqlite3VdbeAddOp4Int(v, OP_SeekGT, iIdxCur, 0, regPrev, 1); + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, j2); + sqlite3VdbeJumpHere(v, j3); + }else{ + sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); + } } /* Add the entry to the stat1 table. */ diff --git a/src/pragma.c b/src/pragma.c index e2c625549e..7105f75a97 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2176,6 +2176,25 @@ void sqlite3Pragma( break; } + /* + ** PRAGMA analysis_limit + ** PRAGMA analysis_limit = N + ** + ** Configure the maximum number of rows that ANALYZE will examine + ** in each index that it looks at. Return the new limit. + */ + case PragTyp_ANALYSIS_LIMIT: { + sqlite3_int64 N; + if( zRight + && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK + && N>=0 + ){ + db->nAnalysisLimit = (int)(N&0x7fffffff); + } + returnSingleInt(v, db->nAnalysisLimit); + break; + } + #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Report the current state of file logs for all databases diff --git a/src/pragma.h b/src/pragma.h index 7046695a5b..0d7cae0e1d 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -6,49 +6,50 @@ /* The various pragma types */ #define PragTyp_ACTIVATE_EXTENSIONS 0 -#define PragTyp_HEADER_VALUE 1 -#define PragTyp_AUTO_VACUUM 2 -#define PragTyp_FLAG 3 -#define PragTyp_BUSY_TIMEOUT 4 -#define PragTyp_CACHE_SIZE 5 -#define PragTyp_CACHE_SPILL 6 -#define PragTyp_CASE_SENSITIVE_LIKE 7 -#define PragTyp_COLLATION_LIST 8 -#define PragTyp_COMPILE_OPTIONS 9 -#define PragTyp_DATA_STORE_DIRECTORY 10 -#define PragTyp_DATABASE_LIST 11 -#define PragTyp_DEFAULT_CACHE_SIZE 12 -#define PragTyp_ENCODING 13 -#define PragTyp_FOREIGN_KEY_CHECK 14 -#define PragTyp_FOREIGN_KEY_LIST 15 -#define PragTyp_FUNCTION_LIST 16 -#define PragTyp_HARD_HEAP_LIMIT 17 -#define PragTyp_INCREMENTAL_VACUUM 18 -#define PragTyp_INDEX_INFO 19 -#define PragTyp_INDEX_LIST 20 -#define PragTyp_INTEGRITY_CHECK 21 -#define PragTyp_JOURNAL_MODE 22 -#define PragTyp_JOURNAL_SIZE_LIMIT 23 -#define PragTyp_LOCK_PROXY_FILE 24 -#define PragTyp_LOCKING_MODE 25 -#define PragTyp_PAGE_COUNT 26 -#define PragTyp_MMAP_SIZE 27 -#define PragTyp_MODULE_LIST 28 -#define PragTyp_OPTIMIZE 29 -#define PragTyp_PAGE_SIZE 30 -#define PragTyp_PRAGMA_LIST 31 -#define PragTyp_SECURE_DELETE 32 -#define PragTyp_SHRINK_MEMORY 33 -#define PragTyp_SOFT_HEAP_LIMIT 34 -#define PragTyp_SYNCHRONOUS 35 -#define PragTyp_TABLE_INFO 36 -#define PragTyp_TEMP_STORE 37 -#define PragTyp_TEMP_STORE_DIRECTORY 38 -#define PragTyp_THREADS 39 -#define PragTyp_WAL_AUTOCHECKPOINT 40 -#define PragTyp_WAL_CHECKPOINT 41 -#define PragTyp_LOCK_STATUS 42 -#define PragTyp_STATS 43 +#define PragTyp_ANALYSIS_LIMIT 1 +#define PragTyp_HEADER_VALUE 2 +#define PragTyp_AUTO_VACUUM 3 +#define PragTyp_FLAG 4 +#define PragTyp_BUSY_TIMEOUT 5 +#define PragTyp_CACHE_SIZE 6 +#define PragTyp_CACHE_SPILL 7 +#define PragTyp_CASE_SENSITIVE_LIKE 8 +#define PragTyp_COLLATION_LIST 9 +#define PragTyp_COMPILE_OPTIONS 10 +#define PragTyp_DATA_STORE_DIRECTORY 11 +#define PragTyp_DATABASE_LIST 12 +#define PragTyp_DEFAULT_CACHE_SIZE 13 +#define PragTyp_ENCODING 14 +#define PragTyp_FOREIGN_KEY_CHECK 15 +#define PragTyp_FOREIGN_KEY_LIST 16 +#define PragTyp_FUNCTION_LIST 17 +#define PragTyp_HARD_HEAP_LIMIT 18 +#define PragTyp_INCREMENTAL_VACUUM 19 +#define PragTyp_INDEX_INFO 20 +#define PragTyp_INDEX_LIST 21 +#define PragTyp_INTEGRITY_CHECK 22 +#define PragTyp_JOURNAL_MODE 23 +#define PragTyp_JOURNAL_SIZE_LIMIT 24 +#define PragTyp_LOCK_PROXY_FILE 25 +#define PragTyp_LOCKING_MODE 26 +#define PragTyp_PAGE_COUNT 27 +#define PragTyp_MMAP_SIZE 28 +#define PragTyp_MODULE_LIST 29 +#define PragTyp_OPTIMIZE 30 +#define PragTyp_PAGE_SIZE 31 +#define PragTyp_PRAGMA_LIST 32 +#define PragTyp_SECURE_DELETE 33 +#define PragTyp_SHRINK_MEMORY 34 +#define PragTyp_SOFT_HEAP_LIMIT 35 +#define PragTyp_SYNCHRONOUS 36 +#define PragTyp_TABLE_INFO 37 +#define PragTyp_TEMP_STORE 38 +#define PragTyp_TEMP_STORE_DIRECTORY 39 +#define PragTyp_THREADS 40 +#define PragTyp_WAL_AUTOCHECKPOINT 41 +#define PragTyp_WAL_CHECKPOINT 42 +#define PragTyp_LOCK_STATUS 43 +#define PragTyp_STATS 44 /* Property flags associated with various pragma. */ #define PragFlg_NeedSchema 0x01 /* Force schema load before running */ @@ -139,6 +140,11 @@ static const PragmaName aPragmaName[] = { /* ColNames: */ 0, 0, /* iArg: */ 0 }, #endif + {/* zName: */ "analysis_limit", + /* ePragTyp: */ PragTyp_ANALYSIS_LIMIT, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) {/* zName: */ "application_id", /* ePragTyp: */ PragTyp_HEADER_VALUE, @@ -639,4 +645,4 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_WriteSchema|SQLITE_NoSchemaError }, #endif }; -/* Number of pragmas: 66 on by default, 76 total. */ +/* Number of pragmas: 67 on by default, 77 total. */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d08c7ce1e7..464ab91175 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1539,6 +1539,7 @@ struct sqlite3 { BusyHandler busyHandler; /* Busy callback */ Db aDbStatic[2]; /* Static space for the 2 default backends */ Savepoint *pSavepoint; /* List of active savepoints */ + int nAnalysisLimit; /* Number of index rows to ANALYZE */ int busyTimeout; /* Busy handler timeout, in msec */ int nSavepoint; /* Number of non-transaction savepoints */ int nStatement; /* Number of nested statement-transactions */ diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 7c8ffd3530..c33f20fd98 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -382,6 +382,9 @@ set pragma_def { NAME: threads FLAG: Result0 + NAME: analysis_limit + FLAG: Result0 + NAME: optimize FLAG: Result1 NeedSchema From 31e3744ecb0b679ffbdf3b9b5ac32c360877571c Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Apr 2020 01:15:16 +0000 Subject: [PATCH 8/9] Simple fixes to PRAGMA analysis_limit. FossilOrigin-Name: c20d4fdee21409ebc9c65c9540af8ac48d1f4425499a6674ef9319655c192612 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 10 +++------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index b3aa1c4102..2f08512a89 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\s"PRAGMA\sanalysis_limit=N;"\sto\slimit\sthe\snumber\sof\srows\svisited\sby\nANALYZE\swhen\sN\sis\spositive.\s\sPositive\sN\salso\sdisables\scollecting\sstat4. -D 2020-03-31T20:57:06.787 +C Simple\sfixes\sto\sPRAGMA\sanalysis_limit. +D 2020-04-01T01:15:16.332 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -466,7 +466,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c f48a4423c8f198d7f1ae4940f74b606707d05384ac79fb219be8e3323af2a2de -F src/analyze.c 4b139fa045d17192ba11ee97e6123cc011efbae690ec302f5d1eb53fd52d5ded +F src/analyze.c 05c99006dfc373a056b05a96539d86482684715b8840c097a6bbc83616c7cb50 F src/attach.c ff2daea0fe62080192e3f262670e4f61f5a86c1e7bea9cec34e960fe79852aa1 F src/auth.c a3d5bfdba83d25abed1013a8c7a5f204e2e29b0c25242a56bc02bb0c07bf1e06 F src/backup.c 5e617c087f1c2d6005c2ec694ce80d6e16bc68d906e1b1c556d7c7c2228b636b @@ -1860,7 +1860,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c705ce266ad25af71791035590875f0ea9f2c72826b3eda17f065d2bf091de92 -R 89edd918806457e430236657da896c8f +P a279b151c1623807774daf4975175c62ea252eefb71f9820ced6773769b392c5 +R 29bf358993a7c679676ef1389a2bc787 U drh -Z bd47b9db8446ad6b70c7b3bf3d82180a +Z 5cabae488dd95fb06d50ea84062f02c6 diff --git a/manifest.uuid b/manifest.uuid index 788d4b8710..a36af84637 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a279b151c1623807774daf4975175c62ea252eefb71f9820ced6773769b392c5 \ No newline at end of file +c20d4fdee21409ebc9c65c9540af8ac48d1f4425499a6674ef9319655c192612 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 2cad81817b..baca0efad0 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -444,10 +444,6 @@ static void statInit( p->nEst = sqlite3_value_int64(argv[2]); p->nRow = 0; p->nLimit = sqlite3_value_int64(argv[3]); -#ifdef SQLITE_ENABLE_STAT4 - /* Disable STAT4 statistics gathering if there is a analysis_limit set */ - if( p->nLimit>0 ) p->mxSample = 0; -#endif p->nCol = nCol; p->nKeyCol = nKeyCol; p->nSkipAhead = 0; @@ -455,7 +451,7 @@ static void statInit( p->current.anEq = &p->current.anDLt[nColUp]; #ifdef SQLITE_ENABLE_STAT4 - p->mxSample = mxSample; + p->mxSample = p->nLimit==0 ? mxSample : 0; if( mxSample ){ u8 *pSpace; /* Allocated space not yet assigned */ int i; /* Used to iterate through p->aSample[] */ @@ -489,7 +485,7 @@ static void statInit( sqlite3_result_blob(context, p, sizeof(*p), statAccumDestructor); } static const FuncDef statInitFuncdef = { - 2+IsStat4, /* nArg */ + 4, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ @@ -1256,7 +1252,7 @@ static void analyzeOneTable( /* Add the entries to the stat4 table. */ #ifdef SQLITE_ENABLE_STAT4 - if( OptimizationEnabled(db, SQLITE_Stat4) ){ + if( OptimizationEnabled(db, SQLITE_Stat4) && db->nAnalysisLimit==0 ){ int regEq = regStat1; int regLt = regStat1+1; int regDLt = regStat1+2; From f72981f2d94bc9c3a17a7fa5abf821d49bad6aa0 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 2 May 2020 03:29:21 +0000 Subject: [PATCH 9/9] Add a missing VdbeCoverage() macro. FossilOrigin-Name: 77a55c394d2c313a5710229bee9262457dcfc6620e6500f9f526c5f6acf87cef --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 324e12ab15..ee0f01f5ce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\strunk\senhancements. -D 2020-05-01T15:04:13.668 +C Add\sa\smissing\sVdbeCoverage()\smacro. +D 2020-05-02T03:29:21.137 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -468,7 +468,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c fa2c3be9b0ebecfafb7062072a0ae6eda126d3e5a9fd51b2eded5acd95dc783c -F src/analyze.c 05c99006dfc373a056b05a96539d86482684715b8840c097a6bbc83616c7cb50 +F src/analyze.c f71fb976b1bc8e62320e4932f979e56f535237b1579e07b392b22f9a1ccbba42 F src/attach.c ff2daea0fe62080192e3f262670e4f61f5a86c1e7bea9cec34e960fe79852aa1 F src/auth.c a3d5bfdba83d25abed1013a8c7a5f204e2e29b0c25242a56bc02bb0c07bf1e06 F src/backup.c 5e617c087f1c2d6005c2ec694ce80d6e16bc68d906e1b1c556d7c7c2228b636b @@ -1862,7 +1862,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 17901ea6a1951b8c55671408841901c6660e3f8099378204f080b171a684d718 853703cd6d44d6dd48ef5eda6523e374b8ebdf7c338ddaad31c15a40a8b3fd9b -R d39b23d1b282daf000e818b2d8c99f0c +P 2100b2c8f339e9778723fa0c91e479bab8675cf6fbea1664b6af49f40db6d27b +R 90bb2d279dfaf51c661d9433e0108281 U drh -Z 7176e56ef832c90b125106619a6d3ee2 +Z fa6729d909d64e1004093dfe525b2a80 diff --git a/manifest.uuid b/manifest.uuid index 755246313d..ba532de872 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2100b2c8f339e9778723fa0c91e479bab8675cf6fbea1664b6af49f40db6d27b \ No newline at end of file +77a55c394d2c313a5710229bee9262457dcfc6620e6500f9f526c5f6acf87cef \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index baca0efad0..53bd62a822 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1230,6 +1230,7 @@ static void analyzeOneTable( j1 = sqlite3VdbeAddOp1(v, OP_IsNull, regTemp); VdbeCoverage(v); j2 = sqlite3VdbeAddOp1(v, OP_If, regTemp); VdbeCoverage(v); j3 = sqlite3VdbeAddOp4Int(v, OP_SeekGT, iIdxCur, 0, regPrev, 1); + VdbeCoverage(v); sqlite3VdbeJumpHere(v, j1); sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); sqlite3VdbeJumpHere(v, j2);