paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add an option to generate stat1 data based on a subset of the user database

Browse Source 
table contents to sqlite3_expert.

FossilOrigin-Name: c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
This commit is contained in:
dan
2017-04-20 09:54:04 +00:00
parent a6ed5a4f39
commit e53b4f9774
5 changed files with 175 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ext/expert/expert.c
+9 -1
View File
@@ -36,6 +36,7 @@ static void usage(char **argv){
fprintf(stderr, " -sql SQL (analyze SQL statements passed as argument)\n");
fprintf(stderr, " -file FILE (read SQL statements from file FILE)\n");
fprintf(stderr, " -verbose LEVEL (integer verbosity level. default 1)\n");
fprintf(stderr, " -sample PERCENT (percent of db to sample. default 100)\n");
exit(-1);
}
@@ -97,11 +98,18 @@ int main(int argc, char **argv){
rc = readSqlFromFile(p, argv[i], &zErr);
}
else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){
else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){
if( ++i==(argc-1) ) option_requires_argument("-sql");
rc = sqlite3_expert_sql(p, argv[i], &zErr);
}
else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sample", nArg) ){
int iSample;
if( ++i==(argc-1) ) option_requires_argument("-sample");
iSample = option_integer_arg(argv[i]);
sqlite3_expert_config(p, EXPERT_CONFIG_SAMPLE, iSample);
}
else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-verbose", nArg) ){
if( ++i==(argc-1) ) option_requires_argument("-verbose");
iVerbose = option_integer_arg(argv[i]);
ext/expert/sqlite3expert.c
+124 -9
View File
@@ -28,6 +28,9 @@ typedef struct IdxStatement IdxStatement;
typedef struct IdxTable IdxTable;
typedef struct IdxWrite IdxWrite;
#define UNIQUE_TABLE_NAME "t592690916721053953805701627921227776"
/*
** A single constraint. Equivalent to either "col = ?" or "col < ?" (or
** any other type of single-ended range constraint on a column).
@@ -127,6 +130,7 @@ struct IdxHash {
** sqlite3expert object.
*/
struct sqlite3expert {
int iSample; /* Percentage of tables to sample for stat1 */
sqlite3 *db; /* User database */
sqlite3 *dbm; /* In-memory db for this analysis */
sqlite3 *dbv; /* Vtab schema for this analysis */
@@ -1080,8 +1084,8 @@ static int idxProcessOneTrigger(
IdxWrite *pWrite,
char **pzErr
){
static const char *zInt = "t592690916721053953805701627921227776";
static const char *zDrop = "DROP TABLE t592690916721053953805701627921227776";
static const char *zInt = UNIQUE_TABLE_NAME;
static const char *zDrop = "DROP TABLE " UNIQUE_TABLE_NAME;
IdxTable *pTab = pWrite->pTab;
const char *zTab = pTab->zName;
const char *zSql =
@@ -1235,6 +1239,38 @@ static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
return rc;
}
struct IdxSampleCtx {
int iTarget;
double target; /* Target nRet/nRow value */
double nRow; /* Number of rows seen */
double nRet; /* Number of rows returned */
};
static void idxSampleFunc(
sqlite3_context *pCtx,
int argc,
sqlite3_value **argv
){
struct IdxSampleCtx *p = (struct IdxSampleCtx*)sqlite3_user_data(pCtx);
int bRet;
assert( argc==0 );
if( p->nRow==0.0 ){
bRet = 1;
}else{
bRet = (p->nRet / p->nRow) <= p->target;
if( bRet==0 ){
unsigned short rnd;
sqlite3_randomness(2, (void*)&rnd);
bRet = ((int)rnd % 100) <= p->iTarget;
}
}
sqlite3_result_int(pCtx, bRet);
p->nRow += 1.0;
p->nRet += (double)bRet;
}
struct IdxRemCtx {
int nSlot;
struct IdxRemSlot {
@@ -1360,6 +1396,8 @@ static int idxPopulateOneStat1(
int *aStat = 0;
int rc = SQLITE_OK;
assert( p->iSample>0 );
/* Formulate the query text */
sqlite3_bind_text(pIndexXInfo, 1, zIdx, -1, SQLITE_STATIC);
while( SQLITE_OK==rc && SQLITE_ROW==sqlite3_step(pIndexXInfo) ){
@@ -1372,9 +1410,15 @@ static int idxPopulateOneStat1(
zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol);
}
if( rc==SQLITE_OK ){
zQuery = sqlite3_mprintf(
"SELECT %s FROM %Q x ORDER BY %s", zCols, zTab, zOrder
);
if( p->iSample==100 ){
zQuery = sqlite3_mprintf(
"SELECT %s FROM %Q x ORDER BY %s", zCols, zTab, zOrder
);
}else{
zQuery = sqlite3_mprintf(
"SELECT %s FROM temp."UNIQUE_TABLE_NAME" x ORDER BY %s", zCols, zOrder
);
}
}
sqlite3_free(zCols);
sqlite3_free(zOrder);
@@ -1433,6 +1477,25 @@ static int idxPopulateOneStat1(
return rc;
}
static int idxBuildSampleTable(sqlite3expert *p, const char *zTab){
int rc;
char *zSql;
rc = sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
if( rc!=SQLITE_OK ) return rc;
zSql = sqlite3_mprintf(
"CREATE TABLE temp." UNIQUE_TABLE_NAME
" AS SELECT * FROM %Q WHERE sample()"
, zTab
);
if( zSql==0 ) return SQLITE_NOMEM;
rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
sqlite3_free(zSql);
return rc;
}
/*
** This function is called as part of sqlite3_expert_analyze(). Candidate
** indexes have already been created in database sqlite3expert.dbm, this
@@ -1444,13 +1507,15 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
int rc = SQLITE_OK;
int nMax =0;
struct IdxRemCtx *pCtx = 0;
struct IdxSampleCtx samplectx;
int i;
i64 iPrev = -100000;
sqlite3_stmt *pAllIndex = 0;
sqlite3_stmt *pIndexXInfo = 0;
sqlite3_stmt *pWrite = 0;
const char *zAllIndex =
"SELECT s.name, l.name FROM "
const char *zAllIndex =
"SELECT s.rowid, s.name, l.name FROM "
" sqlite_master AS s, "
" pragma_index_list(s.name) AS l "
"WHERE s.type = 'table'";
@@ -1458,6 +1523,9 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
"SELECT name, coll FROM pragma_index_xinfo(?) WHERE key";
const char *zWrite = "INSERT INTO sqlite_stat1 VALUES(?, ?, ?)";
/* If iSample==0, no sqlite_stat1 data is required. */
if( p->iSample==0 ) return SQLITE_OK;
rc = idxLargestIndex(p->dbm, &nMax, pzErr);
if( nMax<=0 || rc!=SQLITE_OK ) return rc;
@@ -1473,6 +1541,11 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
p->db, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
);
}
if( rc==SQLITE_OK ){
rc = sqlite3_create_function(
p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0
);
}
if( rc==SQLITE_OK ){
pCtx->nSlot = nMax+1;
@@ -1486,9 +1559,24 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
}
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pAllIndex) ){
const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 0);
const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 1);
i64 iRowid = sqlite3_column_int64(pAllIndex, 0);
const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 1);
const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 2);
if( p->iSample<100 && iPrev!=iRowid ){
samplectx.target = (double)p->iSample / 100.0;
samplectx.iTarget = p->iSample;
samplectx.nRow = 0.0;
samplectx.nRet = 0.0;
rc = idxBuildSampleTable(p, zTab);
if( rc!=SQLITE_OK ) break;
}
rc = idxPopulateOneStat1(p, pIndexXInfo, pWrite, zTab, zIdx, pzErr);
iPrev = iRowid;
}
if( p->iSample<100 ){
rc = sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp." UNIQUE_TABLE_NAME,
0,0,0
);
}
idxFinalize(&rc, pAllIndex);
@@ -1503,6 +1591,8 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
if( rc==SQLITE_OK ){
rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_master", 0, 0, 0);
}
sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
return rc;
}
@@ -1523,6 +1613,7 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
*/
if( rc==SQLITE_OK ){
pNew->db = db;
pNew->iSample = 100;
rc = sqlite3_open(":memory:", &pNew->dbv);
}
if( rc==SQLITE_OK ){
@@ -1565,6 +1656,30 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
return pNew;
}
/*
** Configure an sqlite3expert object.
*/
int sqlite3_expert_config(sqlite3expert *p, int op, ...){
int rc = SQLITE_OK;
va_list ap;
va_start(ap, op);
switch( op ){
case EXPERT_CONFIG_SAMPLE: {
int iVal = va_arg(ap, int);
if( iVal<0 ) iVal = 0;
if( iVal>100 ) iVal = 100;
p->iSample = iVal;
break;
}
default:
rc = SQLITE_NOTFOUND;
break;
}
va_end(ap);
return rc;
}
/*
** Add an SQL statement to the analysis.
*/
ext/expert/sqlite3expert.h
+33
View File
@@ -27,6 +27,38 @@ typedef struct sqlite3expert sqlite3expert;
*/
sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr);
/*
** Configure an sqlite3expert object.
**
** EXPERT_CONFIG_SAMPLE:
** By default, sqlite3_expert_analyze() generates sqlite_stat1 data for
** each candidate index. This involves scanning and sorting the entire
** contents of each user database table once for each candidate index
** associated with the table. For large databases, this can be
** prohibitively slow. This option allows the sqlite3expert object to
** be configured so that sqlite_stat1 data is instead generated based on a
** subset of each table, or so that no sqlite_stat1 data is used at all.
**
** A single integer argument is passed to this option. If the value is less
** than or equal to zero, then no sqlite_stat1 data is generated or used by
** the analysis - indexes are recommended based on the database schema only.
** Or, if the value is 100 or greater, complete sqlite_stat1 data is
** generated for each candidate index (this is the default). Finally, if the
** value falls between 0 and 100, then it represents the percentage of user
** table rows that should be considered when generating sqlite_stat1 data.
**
** Examples:
**
** // Do not generate any sqlite_stat1 data
** sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 0);
**
** // Generate sqlite_stat1 data based on 10% of the rows in each table.
** sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 10);
*/
int sqlite3_expert_config(sqlite3expert *p, int op, ...);
#define EXPERT_CONFIG_SAMPLE 1 /* int */
/*
** Specify zero or more SQL statements to be included in the analysis.
**
@@ -54,6 +86,7 @@ int sqlite3_expert_sql(
char **pzErr /* OUT: Error message (if any) */
);
/*
** This function is called after the sqlite3expert object has been configured
** with all SQL statements using sqlite3_expert_sql() to actually perform
manifest
+8 -8
View File
@@ -1,5 +1,5 @@
C Have\ssqlite3_expert_analyze()\spopulate\sthe\ssqlite_stat1\stable\sbefore\srunning\nqueries\sthrough\sthe\splanner\sfor\sthe\ssecond\stime.
D 2017-04-18T20:10:16.786
C Add\san\soption\sto\sgenerate\sstat1\sdata\sbased\son\sa\ssubset\sof\sthe\suser\sdatabase\ntable\scontents\sto\ssqlite3_expert.
D 2017-04-20T09:54:04.700
F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -41,10 +41,10 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
F ext/expert/expert.c 22d2dd096d479049bc332506fc8c0294bf53b7ebfe60af99635d8c87839bb40b
F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
F ext/expert/sqlite3expert.c 713388c6c440c6759a1e0898c7936a014dc9791237e62780412229e4a79b0035
F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
F ext/expert/sqlite3expert.c af3b336f83bcd2a586f6119d4040ac36ccf45162c48e3780ed63ab119fd04fe1
F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
F ext/expert/test_expert.c b01a5115f9444a9b416582c985138f5dfdb279848ce8b7452be383530be27f01
F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
@@ -1579,7 +1579,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P ff4976da667872614331d88e68fb67d347874f164a1c7950dd738c7c2320b954
R d621eb0093377628a3811751fb5daa84
P a157fcfde5afc27ae38e7cf4669fcc8e60e23d9d301ffe2e541dd69f895b493b
R 2a75643c384b85cff6c5d9d5615ac2a2
U dan
Z 02b917b4563ed84ab0581f0a345c83b3
Z 7876a034c88841c26664b8a83eb3817f
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
a157fcfde5afc27ae38e7cf4669fcc8e60e23d9d301ffe2e541dd69f895b493b
c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a