paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implementation of enhancement request [695a1a53dea6b240]: The query planner

Browse Source 
has improved awareness of when indexes on expressions are covering and adjusts
their costs accordingly.

FossilOrigin-Name: e3474d79b27298e96e7686e5a6f9a8e736b8a6568a0a08fb1abe5bcca038ca63
This commit is contained in:
drh
2022-11-28 19:42:48 +00:00
parent 1b3d13e65e cbb9585805
commit 57ba01de4a
5 changed files with 203 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
manifest
+9 -9
View File
@@ -1,5 +1,5 @@
C Add\sthe\sSQLITE_FCNTL_RESET_CACHE\sverb.\sUse\sit\sto\sensure\sthat\sthe\spage\scache\sis\spurged\sbefore\sand\safter\sa\sthe\srecovery\sextension\sis\srun.
D 2022-11-28T18:41:41.586
C Implementation\sof\senhancement\srequest\s[695a1a53dea6b240]:\sThe\squery\splanner\nhas\simproved\sawareness\sof\swhen\sindexes\son\sexpressions\sare\scovering\sand\sadjusts\ntheir\scosts\saccordingly.
D 2022-11-28T19:42:48.433
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -731,8 +731,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
F src/where.c ea0f518df9e00aa44013a1d384090b4b3a499ee11d4daa0a7d99c4eb9f7ab4ba
F src/whereInt.h df0c79388c0b71b4a91f480d02791679fe0345d40410435c541c8893e95a4d3f
F src/where.c 81422870d17fdf3880fdb9d3b3965e1aa5efb7a72c6933cec3f1bc8081c463db
F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c
F src/wherecode.c 133a94f82858787217d073143617df19e4a6a7d0b771a1519f957608109ad5a5
F src/whereexpr.c 05295b44b54eea76d1ba766f0908928d0e20e990c249344c9521454d3d09c7ae
F src/window.c 14836767adb26573b50f528eb37f8b1336f2c430ab38de7cead1e5c546bb4d8c
@@ -1213,7 +1213,7 @@ F test/index7.test b238344318e0b4e42126717f6554f0e7dfd0b39cecad4b736039b43e1e3b6
F test/index8.test caa097735c91dbc23d8a402f5e63a2a03c83840ba3928733ed7f9a03f8a912a3
F test/index9.test 2ac891806a4136ef3e91280477e23114e67575207dc331e6797fa0ed9379f997
F test/indexedby.test f21eca4f7a6ffe14c8500a7ad6cd53166666c99e5ccd311842a28bc94a195fe0
F test/indexexpr1.test 3360c2a29a8844e7c4b13293567025281257f9e13a31854cfff6959cede11502
F test/indexexpr1.test 1cd460113e3999a1477754fd0e3ac0d7bcbbbc742b427b418945f28af708c695
F test/indexexpr2.test 2c7abe3c48f8aaa5a448615ab4d13df3662185d28419c00999670834a3f0b484
F test/indexfault.test 98d78a8ff1f5335628b62f886a1cb7c7dac1ef6d48fa39c51ec871c87dce9811
F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7
@@ -2064,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P b44ab10c49bc2895483a9d40813be3798710ee713cc4bf04e449dce55a68452a
R 013988a2db147f7833ada18759e80a5f
U dan
Z 0974da01907b06c2d097fc0b34ec52e3
P 6db0bc4bc0d272b610bef2aeeae43f539ed6e7cc0a9cc767d5af85ecb0019d5f c022c0152ad61a4f56e36f4062609073e2273fbf6f826c20652159be229c2d46
R cf4309a5723ca7c0ed03679edb5024d5
U drh
Z 287040c8f243d0f300388e5a2dad7f24
# Remove this line to create a well-formed Fossil manifest.
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
6db0bc4bc0d272b610bef2aeeae43f539ed6e7cc0a9cc767d5af85ecb0019d5f
e3474d79b27298e96e7686e5a6f9a8e736b8a6568a0a08fb1abe5bcca038ca63
src/where.c
+120 -41
View File
@@ -3094,7 +3094,7 @@ static int whereLoopAddBtreeIndex(
assert( pSrc->pTab->szTabRow>0 );
rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow;
pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx);
if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){
if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){
pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16);
}
ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult);
@@ -3247,16 +3247,39 @@ static int whereUsablePartialIndex(
return 0;
}
/*
** pIdx is an index containing expressions. Check it see if any of the
** expressions in the index match the pExpr expression.
*/
static int exprIsCoveredByIndex(
const Expr *pExpr,
const Index *pIdx,
int iTabCur
){
int i;
for(i=0; i<pIdx->nColumn; i++){
if( pIdx->aiColumn[i]==XN_EXPR
&& sqlite3ExprCompare(0, pExpr, pIdx->aColExpr->a[i].pExpr, iTabCur)==0
){
return 1;
}
}
return 0;
}
/*
** Structure passed to the whereIsCoveringIndex Walker callback.
*/
typedef struct CoveringIndexCheck CoveringIndexCheck;
struct CoveringIndexCheck {
Index *pIdx; /* The index */
int iTabCur; /* Cursor number for the corresponding table */
u8 bExpr; /* Uses an indexed expression */
u8 bUnidx; /* Uses an unindexed column not within an indexed expr */
};
/*
** Information passed in is pWalk->u.pCovIdxCk. Call is pCk.
** Information passed in is pWalk->u.pCovIdxCk. Call it pCk.
**
** If the Expr node references the table with cursor pCk->iTabCur, then
** make sure that column is covered by the index pCk->pIdx. We know that
@@ -3268,71 +3291,103 @@ struct CoveringIndexCheck {
**
** If this node does not disprove that the index can be a covering index,
** then just return WRC_Continue, to continue the search.
**
** If pCk->pIdx contains indexed expressions and one of those expressions
** matches pExpr, then prune the search.
*/
static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){
int i; /* Loop counter */
const Index *pIdx; /* The index of interest */
const i16 *aiColumn; /* Columns contained in the index */
u16 nColumn; /* Number of columns in the index */
if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_AGG_COLUMN ) return WRC_Continue;
if( pExpr->iColumn<(BMS-1) ) return WRC_Continue;
if( pExpr->iTable!=pWalk->u.pCovIdxCk->iTabCur ) return WRC_Continue;
pIdx = pWalk->u.pCovIdxCk->pIdx;
aiColumn = pIdx->aiColumn;
nColumn = pIdx->nColumn;
for(i=0; i<nColumn; i++){
if( aiColumn[i]==pExpr->iColumn ) return WRC_Continue;
int i; /* Loop counter */
const Index *pIdx; /* The index of interest */
const i16 *aiColumn; /* Columns contained in the index */
u16 nColumn; /* Number of columns in the index */
CoveringIndexCheck *pCk; /* Info about this search */
pCk = pWalk->u.pCovIdxCk;
pIdx = pCk->pIdx;
if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){
/* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/
if( pExpr->iTable!=pCk->iTabCur ) return WRC_Continue;
pIdx = pWalk->u.pCovIdxCk->pIdx;
aiColumn = pIdx->aiColumn;
nColumn = pIdx->nColumn;
for(i=0; i<nColumn; i++){
if( aiColumn[i]==pExpr->iColumn ) return WRC_Continue;
}
pCk->bUnidx = 1;
return WRC_Abort;
}else if( pIdx->bHasExpr
&& exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){
pCk->bExpr = 1;
return WRC_Prune;
}
pWalk->eCode = 1;
return WRC_Abort;
return WRC_Continue;
}
/*
** pIdx is an index that covers all of the low-number columns used by
** pWInfo->pSelect (columns from 0 through 62). But there are columns
** in pWInfo->pSelect beyond 62. This routine tries to answer the question
** of whether pIdx covers *all* columns in the query.
** pWInfo->pSelect (columns from 0 through 62) or an index that has
** expressions terms. Hence, we cannot determine whether or not it is
** a covering index by using the colUsed bitmasks. We have to do a search
** to see if the index is covering. This routine does that search.
**
** Return 0 if pIdx is a covering index. Return non-zero if pIdx is
** not a covering index or if we are unable to determine if pIdx is a
** covering index.
** The return value is one of these:
**
** This routine is an optimization. It is always safe to return non-zero.
** But returning zero when non-zero should have been returned can lead to
** incorrect bytecode and assertion faults.
** 0 The index is definitely not a covering index
**
** WHERE_IDX_ONLY The index is definitely a covering index
**
** WHERE_EXPRIDX The index is likely a covering index, but it is
** difficult to determine precisely because of the
** expressions that are indexed. Score it as a
** covering index, but still keep the main table open
** just in case we need it.
**
** This routine is an optimization. It is always safe to return zero.
** But returning one of the other two values when zero should have been
** returned can lead to incorrect bytecode and assertion faults.
*/
static SQLITE_NOINLINE u32 whereIsCoveringIndex(
WhereInfo *pWInfo, /* The WHERE clause context */
Index *pIdx, /* Index that is being tested */
int iTabCur /* Cursor for the table being indexed */
){
int i;
int i, rc;
struct CoveringIndexCheck ck;
Walker w;
if( pWInfo->pSelect==0 ){
/* We don't have access to the full query, so we cannot check to see
** if pIdx is covering. Assume it is not. */
return 1;
return 0;
}
for(i=0; i<pIdx->nColumn; i++){
if( pIdx->aiColumn[i]>=BMS-1 ) break;
}
if( i>=pIdx->nColumn ){
/* pIdx does not index any columns greater than 62, but we know from
** colMask that columns greater than 62 are used, so this is not a
** covering index */
return 1;
if( pIdx->bHasExpr==0 ){
for(i=0; i<pIdx->nColumn; i++){
if( pIdx->aiColumn[i]>=BMS-1 ) break;
}
if( i>=pIdx->nColumn ){
/* pIdx does not index any columns greater than 62, but we know from
** colMask that columns greater than 62 are used, so this is not a
** covering index */
return 0;
}
}
ck.pIdx = pIdx;
ck.iTabCur = iTabCur;
ck.bExpr = 0;
ck.bUnidx = 0;
memset(&w, 0, sizeof(w));
w.xExprCallback = whereIsCoveringIndexWalkCallback;
w.xSelectCallback = sqlite3SelectWalkNoop;
w.u.pCovIdxCk = &ck;
w.eCode = 0;
sqlite3WalkSelect(&w, pWInfo->pSelect);
return w.eCode;
if( ck.bUnidx ){
rc = 0;
}else if( ck.bExpr ){
rc = WHERE_EXPRIDX;
}else{
rc = WHERE_IDX_ONLY;
}
return rc;
}
/*
@@ -3548,14 +3603,38 @@ static int whereLoopAddBtree(
}else{
Bitmask m;
if( pProbe->isCovering ){
pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
m = 0;
pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
}else{
m = pSrc->colUsed & pProbe->colNotIdxed;
if( m==TOPBIT ){
m = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor);
pNew->wsFlags = WHERE_INDEXED;
if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){
u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor);
if( isCov==0 ){
WHERETRACE(0xff,
("-> %s is not a covering index"
" according to whereIsCoveringIndex()\n", pProbe->zName));
assert( m!=0 );
}else{
m = 0;
pNew->wsFlags |= isCov;
if( isCov & WHERE_IDX_ONLY ){
WHERETRACE(0xff,
("-> %s is a covering expression index"
" according to whereIsCoveringIndex()\n", pProbe->zName));
}else{
assert( isCov==WHERE_EXPRIDX );
WHERETRACE(0xff,
("-> %s might be a covering expression index"
" according to whereIsCoveringIndex()\n", pProbe->zName));
}
}
}else if( m==0 ){
WHERETRACE(0xff,
("-> %s a covering index according to bitmasks\n",
pProbe->zName, m==0 ? "is" : "is not"));
pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
}
pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED;
}
/* Full scan via index */
src/whereInt.h
+1
View File
@@ -634,5 +634,6 @@ void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*);
#define WHERE_SELFCULL 0x00800000 /* nOut reduced by extra WHERE terms */
#define WHERE_OMIT_OFFSET 0x01000000 /* Set offset counter to zero */
#define WHERE_VIEWSCAN 0x02000000 /* A full-scan of a VIEW or subquery */
#define WHERE_EXPRIDX 0x04000000 /* Uses an index-on-expressions */
#endif /* !defined(SQLITE_WHEREINT_H) */
test/indexexpr1.test
+72 -11
View File
@@ -420,7 +420,7 @@ do_execsql_test indexexpr1-1500 {
# 2018-01-03 OSSFuzz discovers another test case for the same problem
# above.
#
do_execsql_test indexexpr-1510 {
do_execsql_test indexexpr1-1510 {
DROP TABLE IF EXISTS t1;
CREATE TABLE t1(a PRIMARY KEY,b UNIQUE);
REPLACE INTO t1 VALUES(2, 1);
@@ -434,18 +434,18 @@ do_execsql_test indexexpr-1510 {
# a numeric table column, trouble can arise since there are multiple
# string that can map to the same numeric value. (Ex: 123, 0123, 000123).
#
do_execsql_test indexexpr-1600 {
do_execsql_test indexexpr1-1600 {
DROP TABLE IF EXISTS t1;
CREATE TABLE t1 (a INTEGER, b);
CREATE INDEX idx1 ON t1 (lower(a));
INSERT INTO t1 VALUES('0001234',3);
PRAGMA integrity_check;
} {ok}
do_execsql_test indexexpr-1610 {
do_execsql_test indexexpr1-1610 {
INSERT INTO t1 VALUES('1234',0),('001234',2),('01234',1);
SELECT b FROM t1 WHERE lower(a)='1234' ORDER BY +b;
} {0 1 2 3}
do_execsql_test indexexpr-1620 {
do_execsql_test indexexpr1-1620 {
SELECT b FROM t1 WHERE lower(a)='01234' ORDER BY +b;
} {}
@@ -453,7 +453,7 @@ do_execsql_test indexexpr-1620 {
# ExprImpliesExpr theorem prover bug:
# "(NULL IS FALSE) IS FALSE" does not imply "NULL IS NULL"
#
do_execsql_test indexexpr-1700 {
do_execsql_test indexexpr1-1700 {
DROP TABLE IF EXISTS t0;
CREATE TABLE t0(c0);
INSERT INTO t0(c0) VALUES (0);
@@ -468,17 +468,17 @@ do_execsql_test indexexpr-1700 {
# computing the expression.
#
ifcapable like_match_blobs {
do_execsql_test indexexpr-1800 {
do_execsql_test indexexpr1-1800 {
DROP TABLE IF EXISTS t0;
CREATE TABLE t0(c0 REAL, c1 TEXT);
CREATE INDEX i0 ON t0(+c0, c0);
INSERT INTO t0(c0) VALUES(0);
SELECT CAST(+ t0.c0 AS BLOB) LIKE 0 FROM t0;
} {0}
do_execsql_test indexexpr-1810 {
do_execsql_test indexexpr1-1810 {
SELECT CAST(+ t0.c0 AS BLOB) LIKE '0.0' FROM t0;
} {1}
do_execsql_test indexexpr-1820 {
do_execsql_test indexexpr1-1820 {
DROP TABLE IF EXISTS t1;
CREATE TABLE t1(x REAL);
CREATE INDEX t1x ON t1(x, +x);
@@ -491,19 +491,80 @@ ifcapable like_match_blobs {
# Assertion fault during a DELETE INDEXED BY.
#
reset_db
do_execsql_test indexexpr-1900 {
do_execsql_test indexexpr1-1900 {
CREATE TABLE t1(x TEXT PRIMARY KEY, y TEXT, z INT);
INSERT INTO t1(x,y,z) VALUES('alpha','ALPHA',1),('bravo','charlie',1);
CREATE INDEX i1 ON t1(+y COLLATE NOCASE);
SELECT * FROM t1;
} {alpha ALPHA 1 bravo charlie 1}
do_execsql_test indexexpr-1910 {
do_execsql_test indexexpr1-1910 {
DELETE FROM t1 INDEXED BY i1
WHERE x IS +y COLLATE NOCASE IN (SELECT z FROM t1)
RETURNING *;
} {alpha ALPHA 1}
do_execsql_test indexexpr-1920 {
do_execsql_test indexexpr1-1920 {
SELECT * FROM t1;
} {bravo charlie 1}
# 2022-11-28 Ticket 695a1a53de
# Improved ability to recognize that an index on an expression is a
# covering index.
#
reset_db
do_execsql_test indexexpr1-2000 {
CREATE TABLE t1(a INT, b TEXT);
INSERT INTO t1(a,b) VALUES
(10, '{"one":5,"two":6}'),
(10, '{"one":50,"two":60}'),
(10, '{"three":99}'),
(11, '{"one":100,"two":200}');
CREATE INDEX t1_one ON t1(a, b->>'one');
CREATE INDEX t1_two ON t1(a, b->>'two');
}
do_execsql_test indexexpr1-2010 {
EXPLAIN QUERY PLAN
SELECT sum(b->>'one') FROM t1 WHERE a=10; /* Query AA */
} {/.* t1_one .*/}
do_execsql_test indexexpr1-2011 {
SELECT sum(b->>'one') FROM t1 WHERE a=10; /* Query AA */
} {55}
do_execsql_test indexexpr1-2020 {
EXPLAIN QUERY PLAN
SELECT sum(b->>'two') FROM t1 WHERE a=10; /* Query BB */
} {/.* t1_two .*/}
do_execsql_test indexexpr1-2021 {
SELECT sum(b->>'two') FROM t1 WHERE a=10; /* Query BB */
} {66}
do_execsql_test indexexpr1-2030 {
DROP TABLE t1;
CREATE TABLE t1(a INT, b TEXT, c INT, d INT);
INSERT INTO t1(a,b,c,d) VALUES
(1, '{"x":1}', 12, 3),
(1, '{"x":2}', 4, 5),
(1, '{"x":1}', 6, 11),
(2, '{"x":1}', 22, 3),
(2, '{"x":2}', 4, 5),
(3, '{"x":1}', 6, 7);
CREATE INDEX t1x ON t1(d, a, b->>'x', c);
}
do_execsql_test indexexpr1-2030 {
SELECT a,
SUM(1) AS t1,
SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
SUM(c) AS t3,
SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
FROM t1;
} {1 6 4 54 46}
do_execsql_test indexexpr1-2030 {
explain query plan
SELECT a,
SUM(1) AS t1,
SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
SUM(c) AS t3,
SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
FROM t1;
} {/.*SCAN t1 USING INDEX t1x.*/}
finish_test