The sqlite_stat2.cnt field is parsed if it is present. But it is not yet

used.  A large comment added to analyze.c to explain the format of the
ANALYZE system tables.

FossilOrigin-Name: 6d1e2372fe73f4c04561108aac6bc8c95f6e7a1a
This commit is contained in:
drh
2011-08-06 19:48:53 +00:00
parent ed6185ccde
commit ad6dc0ccd7
6 changed files with 214 additions and 75 deletions
+9 -9
View File
@@ -1,5 +1,5 @@
C Merge\stogether\sthe\sfork\sin\sthe\squery-planner-tweaks\sbranch.
D 2011-08-06T02:03:10.239
C The\ssqlite_stat2.cnt\sfield\sis\sparsed\sif\sit\sis\spresent.\s\sBut\sit\sis\snot\syet\nused.\s\sA\slarge\scomment\sadded\sto\sanalyze.c\sto\sexplain\sthe\sformat\sof\sthe\nANALYZE\ssystem\stables.
D 2011-08-06T19:48:53.187
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 1e6988b3c11dee9bd5edc0c804bd4468d74a9cdc
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -118,7 +118,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b
F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc
F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad
F src/alter.c ac80a0f31189f8b4a524ebf661e47e84536ee7f5
F src/analyze.c 19894783b6f833337ae7e56001c96dc3795d2a83
F src/analyze.c 8a5343a20ae69fd27247249cf40262b0021e379e
F src/attach.c 12c6957996908edc31c96d7c68d4942c2474405f
F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34
F src/backup.c 986c15232757f2873dff35ee3b35cbf935fc573c
@@ -183,7 +183,7 @@ F src/select.c d219c4b68d603cc734b6f9b1e2780fee12a1fa0d
F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd
F src/sqlite.h.in 0b3cab7b2ea51f58396e8871fa5f349cfece5330
F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93
F src/sqliteInt.h be1ab8bc2c295a1bf0ddc9241ea879f455b72df6
F src/sqliteInt.h a01882eb98520566f039017232290dde2d0cbeed
F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf
F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
@@ -250,7 +250,7 @@ F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582
F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9
F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a
F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
F src/where.c b8ebb50b82578761e5e764c5c16d7693049a523f
F src/where.c 67ad221f87aa1b12123444cfd9b338e91a50ed7a
F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823
F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87
@@ -267,7 +267,7 @@ F test/analyze4.test 757b37875cf9bb528d46f74497bc789c88365045
F test/analyze5.test f6c250012b173f9b10ae5d577e9592767b376b1c
F test/analyze6.test c125622a813325bba1b4999040ddc213773c2290
F test/analyze7.test ed672a2c09c4b5a5b528ca38951318af463bb17e
F test/analyze8.test ff7a3b4631c896a51e7404268796c61d74117e9f
F test/analyze8.test 8a9c83bf735311daf812c9f762d79234f9069404
F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b
F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b
F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7
@@ -956,7 +956,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5
F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262
P 7e914aa999d9f3f7be58a4494d33a7876af56603 b9d41c3490bf325915227a32b612e6772fb62fa9
R c43dfa67e269269636fb7d00fce4d038
P 2daab6bd42b34b32de46db513437cc4d6ca17975
R cec8d2f4d5cd3f2eecb5e9f36d91b967
U drh
Z 2a4dca5bf1206d663b012fe85305a05e
Z deea39c8beb81cdf79c7b9366ab4e7a9
+1 -1
View File
@@ -1 +1 @@
2daab6bd42b34b32de46db513437cc4d6ca17975
6d1e2372fe73f4c04561108aac6bc8c95f6e7a1a
+156 -32
View File
@@ -10,6 +10,99 @@
**
*************************************************************************
** This file contains code associated with the ANALYZE command.
**
** The ANALYZE command gather statistics about the content of tables
** and indices. These statistics are made available to the query planner
** to help it make better decisions about the best way to implement a
** query.
**
** Two system tables are created as follows:
**
** CREATE TABLE sqlite_stat1(tbl, idx, stat);
** CREATE TABLE sqlite_stat2(tbl, idx, sampleno, sample, cnt);
**
** Additional tables might be added in future releases of SQLite.
** The sqlite_stat2 table is only created and used if SQLite is
** compiled with SQLITE_ENABLE_STAT2. Older versions of SQLite
** omit the sqlite_stat2.cnt column. Newer versions of SQLite are
** able to use older versions of the stat2 table that lack the cnt
** column.
**
** Format of sqlite_stat1:
**
** There is normally one row per index, with the index identified by the
** name in the idx column. The tbl column is the name of the table to
** which the index belongs. In each such row, the stat column will be
** a string consisting of a list of integers. The first integer in this
** list is the number of rows in the index and in the table. The second
** integer is the average number of rows in the index that have the same
** value in the first column of the index. The third integer is the average
** number of rows in the index that have the same value for the first two
** columns. The N-th integer (for N>1) is the average number of rows in
** the index which have the same value for the first N-1 columns. For
** a K-column index, there will be K+1 integers in the stat column. If
** the index is unique, then the last integer will be 1.
**
** The list of integers in the stat column can optionally be followed
** by the keyword "unordered". The "unordered" keyword, if it is present,
** must be separated from the last integer by a single space. If the
** "unordered" keyword is present, then the query planner assumes that
** the index is unordered and will not use the index for a range query.
**
** If the sqlite_stat1.idx column is NULL, then the sqlite_stat1.stat
** column contains a single integer which is the (estimated) number of
** rows in the table identified by sqlite_stat1.tbl.
**
** Format of sqlite_stat2:
**
** The sqlite_stat2 is only created and is only used if SQLite is compiled
** with SQLITE_ENABLE_STAT2. The "stat2" table contains additional information
** about the key distribution within an index. The index is identified by
** the "idx" column and the "tbl" column is the name of the table to which
** the index belongs. There are usually multiple rows in the sqlite_stat2
** table for each index.
**
** The sqlite_stat2 entires for an index that have sampleno>=0 are
** sampled key values for the first column of the index taken at
** intervals along the index. The sqlite_stat2.sample column holds
** the value of the key in the left-most column of the index.
**
** The samples are numbered from 0 to S-1
** where S is 10 by default. The number of samples created by the
** ANALYZE command can be adjusted at compile-time using the
** SQLITE_INDEX_SAMPLES macro. The maximum number of samples is
** SQLITE_MAX_SAMPLES, currently set to 100. There are places in the
** code that use an unsigned character to count samples, so an upper
** bound on SQLITE_MAX_SAMPLES is 255.
**
** Suppose the index contains C rows. And let the number
** of samples be S. SQLite assumes that the samples are taken from the
** following rows for i between 0 and S-1:
**
** rownumber = (i*C*2 + C)/(S*2)
**
** Conceptually, the index is divided into S bins and the sample is
** taken from the middle of each bin. The ANALYZE will not attempt
** to populate sqlite_stat2 for an index that holds fewer than S*2
** entries.
**
** If the key value for a sample (the sqlite_stat2.sample column) is a
** large string or blob, SQLite will only use the first 255 bytes of
** that string or blob.
**
** The sqlite_stat2.cnt column contains the number of entries in the
** index for which sqlite_stat2.sample matches the left-most column
** of the index. In other words, sqlite_stat2.cnt holds the number of
** times the sqlite_stat2.sample value appears in the index.. Many
** older versions of SQLite omit the sqlite_stat2.cnt column.
**
** If the sqlite_stat2.sampleno value is -1, then that row holds a first-
** column key that is a frequently used key in the index. The
** sqlite_stat2.cnt column will hold the number of occurrances of that key.
** This information is useful to the query planner in cases where a
** large percentage of the rows in indexed field have one of a small
** handful of value but the balance of the rows in the index have
** distinct or nearly distinct keys.
*/
#ifndef SQLITE_OMIT_ANALYZE
#include "sqliteInt.h"
@@ -582,33 +675,68 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){
return 0;
}
#if SQLITE_ENABLE_STAT2
/*
** If the Index.aSample variable is not NULL, delete the aSample[] array
** and its contents.
** Delete an array of IndexSample objects
*/
static void deleteIndexSampleArray(
sqlite3 *db, /* The database connection */
IndexSampleArray *pArray /* Array of IndexSample objects */
){
int j;
if( pArray->a==0 ) return;
for(j=0; j<pArray->n; j++){
IndexSample *p = &pArray->a[j];
if( p->eType==SQLITE_TEXT || p->eType==SQLITE_BLOB ){
sqlite3_free(p->u.z);
}
}
sqlite3_free(pArray->a);
memset(pArray, 0, sizeof(*pArray));
}
#endif
/*
** Delete the sample and common-key arrays from the index.
*/
void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){
#ifdef SQLITE_ENABLE_STAT2
if( pIdx->aSample ){
int j;
for(j=0; j<pIdx->nSample; j++){
IndexSample *p = &pIdx->aSample[j];
if( p->eType==SQLITE_TEXT || p->eType==SQLITE_BLOB ){
sqlite3DbFree(db, p->u.z);
}
}
sqlite3DbFree(db, pIdx->aSample);
}
deleteIndexSampleArray(db, &pIdx->sample);
deleteIndexSampleArray(db, &pIdx->comkey);
#else
UNUSED_PARAMETER(db);
UNUSED_PARAMETER(pIdx);
#endif
}
#ifdef SQLITE_ENABLE_STAT2
/*
** Enlarge an array of IndexSample objects.
*/
static IndexSample *allocIndexSample(
sqlite3 *db, /* Database connection to malloc against */
IndexSampleArray *pArray, /* The array to enlarge */
int i /* Return this element */
){
IndexSample *p;
if( i>=pArray->nAlloc ){
int szNew = i+1;
p = (IndexSample*)sqlite3_realloc(pArray->a, szNew*sizeof(IndexSample));
if( p==0 ) return 0;
pArray->a = p;
memset(&pArray->a[pArray->n], 0, (szNew-(pArray->n))*sizeof(IndexSample));
pArray->nAlloc = szNew;
}
if( i>=pArray->n ) pArray->n = i+1;
return &pArray->a[i];
}
#endif
/*
** Load the content of the sqlite_stat1 and sqlite_stat2 tables. The
** contents of sqlite_stat1 are used to populate the Index.aiRowEst[]
** arrays. The contents of sqlite_stat2 are used to populate the
** Index.aSample[] arrays.
** Index.sample and Index.comkey arrays.
**
** If the sqlite_stat1 table is not present in the database, SQLITE_ERROR
** is returned. In this case, even if SQLITE_ENABLE_STAT2 was defined
@@ -629,6 +757,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
HashElem *i;
char *zSql;
int rc;
Table *pTab; /* Stat1 or Stat2 table */
assert( iDb>=0 && iDb<db->nDb );
assert( db->aDb[iDb].pBt!=0 );
@@ -639,14 +768,12 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
Index *pIdx = sqliteHashData(i);
sqlite3DefaultRowEst(pIdx);
sqlite3DeleteIndexSamples(db, pIdx);
pIdx->aSample = 0;
pIdx->nSample = 0;
}
/* Check to make sure the sqlite_stat1 table exists */
sInfo.db = db;
sInfo.zDatabase = db->aDb[iDb].zName;
if( sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)==0 ){
if( (pTab=sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase))==0 ){
return SQLITE_ERROR;
}
@@ -663,15 +790,17 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
/* Load the statistics from the sqlite_stat2 table. */
#ifdef SQLITE_ENABLE_STAT2
if( rc==SQLITE_OK && !sqlite3FindTable(db, "sqlite_stat2", sInfo.zDatabase) ){
if( rc==SQLITE_OK
&& (pTab=sqlite3FindTable(db, "sqlite_stat2", sInfo.zDatabase))==0 ){
rc = SQLITE_ERROR;
}
if( rc==SQLITE_OK ){
sqlite3_stmt *pStmt = 0;
zSql = sqlite3MPrintf(db,
"SELECT idx, sampleno, sample FROM %Q.sqlite_stat2"
" ORDER BY rowid DESC", sInfo.zDatabase);
"SELECT idx, sampleno, sample, %s FROM %Q.sqlite_stat2"
" ORDER BY rowid DESC",
pTab->nCol>=5 ? "cnt" : "0", sInfo.zDatabase);
if( !zSql ){
rc = SQLITE_NOMEM;
}else{
@@ -692,21 +821,16 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
pIdx = sqlite3FindIndex(db, zIndex, sInfo.zDatabase);
if( pIdx==0 ) continue;
iSample = sqlite3_column_int(pStmt, 1);
if( iSample>=SQLITE_MAX_SAMPLES || iSample<0 ) continue;
if( pIdx->nSample<=iSample ){
IndexSample *pNew;
int sz = sizeof(IndexSample)*(iSample+1);
pNew = (IndexSample*)sqlite3Realloc(pIdx->aSample, sz);
if( pNew==0 ){
db->mallocFailed = 1;
break;
}
pIdx->aSample = pNew;
pIdx->nSample = iSample+1;
if( iSample>=SQLITE_MAX_SAMPLES ) continue;
if( iSample<0 ){
pSample = allocIndexSample(db, &pIdx->comkey, pIdx->comkey.n);
}else{
pSample = allocIndexSample(db, &pIdx->sample, iSample);
}
if( pSample==0 ) break;
eType = sqlite3_column_type(pStmt, 2);
pSample = &pIdx->aSample[iSample];
pSample->eType = (u8)eType;
pSample->nCopy = sqlite3_column_int(pStmt, 4);
if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){
pSample->u.r = sqlite3_column_double(pStmt, 2);
}else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){
@@ -716,7 +840,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
sqlite3_column_text(pStmt, 2)
);
int n = sqlite3_column_bytes(pStmt, 2);
if( n>24 ) n = 24;
if( n>255 ) n = 255;
pSample->nByte = (u8)n;
if( n < 1){
pSample->u.z = 0;
+28 -15
View File
@@ -615,6 +615,7 @@ typedef struct FuncDefHash FuncDefHash;
typedef struct IdList IdList;
typedef struct Index Index;
typedef struct IndexSample IndexSample;
typedef struct IndexSampleArray IndexSampleArray;
typedef struct KeyClass KeyClass;
typedef struct KeyInfo KeyInfo;
typedef struct Lookaside Lookaside;
@@ -1450,6 +1451,29 @@ struct UnpackedRecord {
#define UNPACKED_PREFIX_MATCH 0x0010 /* A prefix match is considered OK */
#define UNPACKED_PREFIX_SEARCH 0x0020 /* A prefix match is considered OK */
/*
** Each sample stored in the sqlite_stat2 table is represented in memory
** using a structure of this type.
*/
struct IndexSample {
union {
char *z; /* Value if eType is SQLITE_TEXT or SQLITE_BLOB */
double r; /* Value if eType is SQLITE_FLOAT or SQLITE_INTEGER */
} u;
u8 eType; /* SQLITE_NULL, SQLITE_INTEGER ... etc. */
u8 nByte; /* Size in byte of text or blob. */
u32 nCopy; /* How many copies of this sample are in the database */
};
/*
** An array of IndexSample elements is as follows:
*/
struct IndexSampleArray {
u16 n; /* Number of elements in the array */
u16 nAlloc; /* Space allocated to a[] */
IndexSample *a; /* The samples */
};
/*
** Each SQL index is represented in memory by an
** instance of the following structure.
@@ -1486,26 +1510,15 @@ struct Index {
u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */
u8 autoIndex; /* True if is automatically created (ex: by UNIQUE) */
u8 bUnordered; /* Use this index for == or IN queries only */
u8 nSample; /* Number of slots in aSample[] */
char *zColAff; /* String defining the affinity of each column */
Index *pNext; /* The next index associated with the same table */
Schema *pSchema; /* Schema containing this index */
u8 *aSortOrder; /* Array of size Index.nColumn. True==DESC, False==ASC */
char **azColl; /* Array of collation sequence names for index */
IndexSample *aSample; /* Array of SQLITE_INDEX_SAMPLES samples */
};
/*
** Each sample stored in the sqlite_stat2 table is represented in memory
** using a structure of this type.
*/
struct IndexSample {
union {
char *z; /* Value if eType is SQLITE_TEXT or SQLITE_BLOB */
double r; /* Value if eType is SQLITE_FLOAT or SQLITE_INTEGER */
} u;
u8 eType; /* SQLITE_NULL, SQLITE_INTEGER ... etc. */
u8 nByte; /* Size in byte of text or blob. */
#ifdef SQLITE_ENABLE_STAT2
IndexSampleArray sample; /* Sampled histogram for the first column */
IndexSampleArray comkey; /* The most common keys */
#endif
};
/*
+18 -18
View File
@@ -2422,12 +2422,12 @@ static void bestVirtualIndex(
/*
** Argument pIdx is a pointer to an index structure that has an array of
** pIdx->nSample evenly spaced samples of the first indexed column
** stored in Index.aSample. These samples divide the domain of values stored
** the index into (pIdx->nSample+1) regions.
** pIdx->sample.n evenly spaced samples of the first indexed column
** stored in Index.sample. These samples divide the domain of values stored
** the index into (pIdx->sample.n+1) regions.
** Region 0 contains all values less than the first sample value. Region
** 1 contains values between the first and second samples. Region 2 contains
** values between samples 2 and 3. And so on. Region pIdx->nSample
** values between samples 2 and 3. And so on. Region pIdx->sample.n
** contains values larger than the last sample.
**
** If the index contains many duplicates of a single value, then it is
@@ -2452,8 +2452,8 @@ static int whereRangeRegion(
){
assert( roundUp==0 || roundUp==1 );
if( ALWAYS(pVal) ){
IndexSample *aSample = pIdx->aSample;
int nSample = pIdx->nSample;
IndexSample *aSample = pIdx->sample.a;
int nSample = pIdx->sample.n;
int i = 0;
int eType = sqlite3_value_type(pVal);
@@ -2528,7 +2528,7 @@ static int whereRangeRegion(
}
}
assert( i>=0 && i<=pIdx->nSample );
assert( i>=0 && i<=pIdx->sample.n );
*piRegion = i;
}
return SQLITE_OK;
@@ -2623,13 +2623,13 @@ static int whereRangeScanEst(
#ifdef SQLITE_ENABLE_STAT2
if( nEq==0 && p->aSample ){
if( nEq==0 && p->sample.a ){
sqlite3_value *pLowerVal = 0;
sqlite3_value *pUpperVal = 0;
int iEst;
int iLower = 0;
int nSample = p->nSample;
int iUpper = p->nSample;
int nSample = p->sample.n;
int iUpper = p->sample.n;
int roundUpUpper = 0;
int roundUpLower = 0;
u8 aff = p->pTable->aCol[p->aiColumn[0]].affinity;
@@ -2656,7 +2656,7 @@ static int whereRangeScanEst(
if( pLower ) iLower = iUpper/2;
}else if( pUpperVal==0 ){
rc = whereRangeRegion(pParse, p, pLowerVal, roundUpLower, &iLower);
if( pUpper ) iUpper = (iLower + p->nSample + 1)/2;
if( pUpper ) iUpper = (iLower + p->sample.n + 1)/2;
}else{
rc = whereRangeRegion(pParse, p, pUpperVal, roundUpUpper, &iUpper);
if( rc==SQLITE_OK ){
@@ -2721,8 +2721,8 @@ static int whereEqualScanEst(
int rc; /* Subfunction return code */
double nRowEst; /* New estimate of the number of rows */
assert( p->aSample!=0 );
assert( p->nSample>0 );
assert( p->sample.a!=0 );
assert( p->sample.n>0 );
aff = p->pTable->aCol[p->aiColumn[0]].affinity;
if( pExpr ){
rc = valueFromExpr(pParse, pExpr, aff, &pRhs);
@@ -2737,10 +2737,10 @@ static int whereEqualScanEst(
if( rc ) goto whereEqualScanEst_cancel;
WHERETRACE(("equality scan regions: %d..%d\n", iLower, iUpper));
if( iLower>=iUpper ){
nRowEst = p->aiRowEst[0]/(p->nSample*3);
nRowEst = p->aiRowEst[0]/(p->sample.n*3);
if( nRowEst<*pnRow ) *pnRow = nRowEst;
}else{
nRowEst = (iUpper-iLower)*p->aiRowEst[0]/p->nSample;
nRowEst = (iUpper-iLower)*p->aiRowEst[0]/p->sample.n;
*pnRow = nRowEst;
}
@@ -2782,11 +2782,11 @@ static int whereInScanEst(
int nSingle = 0; /* Histogram regions hit by a single value */
int nNotFound = 0; /* Count of values that are not constants */
int i; /* Loop counter */
int nSample = p->nSample; /* Number of samples */
int nSample = p->sample.n; /* Number of samples */
u8 aSpan[SQLITE_MAX_SAMPLES+1]; /* Histogram regions that are spanned */
u8 aSingle[SQLITE_MAX_SAMPLES+1]; /* Histogram regions hit once */
assert( p->aSample!=0 );
assert( p->sample.a!=0 );
assert( nSample>0 );
aff = p->pTable->aCol[p->aiColumn[0]].affinity;
memset(aSpan, 0, nSample+1);
@@ -3036,7 +3036,7 @@ static void bestBtreeIndex(
wsFlags |= WHERE_COLUMN_NULL;
}
#ifdef SQLITE_ENABLE_STAT2
if( nEq==0 && pProbe->aSample ) pFirstTerm = pTerm;
if( nEq==0 && pProbe->sample.a ) pFirstTerm = pTerm;
#endif
used |= pTerm->prereqRight;
}
+2
View File
@@ -161,3 +161,5 @@ do_test analyze8-3.6 {
SELECT sample, cnt FROM sqlite_stat2 WHERE idx='t1all' ORDER BY sampleno;
}
} {2 2 20 38 20 38 20 38 20 38 20 38 20 38 20 38 20 38 20 38}
finish_test