paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Experimental optimization for distinct aggregates (e.g. "SELECT count(DISTINCT <expr) FROM ...").

Browse Source 
FossilOrigin-Name: eb919611fd2f255e4ad1fe7db633363793169f6cf99c650eaefa48c022eb5d22
This commit is contained in:
dan
2021-03-09 16:06:25 +00:00
parent 52a0753152
commit 4fcb30b7d7
5 changed files with 115 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
manifest
+11 -8
View File
@@ -1,5 +1,5 @@
C Add\stest\scases\sfor\sALTER\sTABLE.
D 2021-03-08T17:22:01.712
C Experimental\soptimization\sfor\sdistinct\saggregates\s(e.g.\s"SELECT\scount(DISTINCT\s<expr)\sFROM\s...").
D 2021-03-09T16:06:25.224
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -542,12 +542,12 @@ F src/printf.c 2b03a80d7c11bb422115dca175a18bf430e9c9dbaa0eee63b758f0c022f8f34f
F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
F src/resolve.c 688070848f0a0c41bcc545a4b4b052921d9abc29ba3102985d3d6f7595d9637c
F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
F src/select.c fb1851222c3ec1e0e1b4c8197a76a4595008956220ddebd7a0a898ad15705acc
F src/select.c b51fbd795e5de19bd8448816eb37b098ef679f49b2572a5d570db73aaa2494a6
F src/shell.c.in af18a2e980aabe739a8188266464866fe7947b100674e07480e7ba3e37595947
F src/sqlite.h.in 3426a080ea1f222a73e3bd91e7eacbd30570a0117c03d42c6dde606f33e5e318
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 61b38c073d5e1e96a3d45271b257aef27d0d13da2bea5347692ae579475cd95e
F src/sqliteInt.h 7c4679b3b068149f497fe50203a4b04be6b17b8aba581cdbc75da45329cad915
F src/sqliteInt.h 75d9504d22cc08a355a51e7027aa955c3423121ce7768a65331a4082aafd66ed
F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -629,7 +629,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c 69e770e96fd56cc21608992bf2c6f1f3dc5cf2572d0495c6a643b06c3a679f14
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c d42d6c80ea363ef689a462e65eefcfe87deab924c50de5baa37ecb6af7d7ddaa
F src/where.c 10d06b16670a1d2a992d52a9f08e49426d38a08fb0a7ae5f7f62fd023d560e1e
F src/where.c 6fcbfab409b4a60dd92abd4c5f3fca44f5df3f8ea24f689d2ea620f05da89d96
F src/whereInt.h 446e5e8018f83358ef917cf32d8e6a86dc8430113d0b17e720f1839d3faa44c4
F src/wherecode.c e57a8690311a75d06e723e8d379f9831de04aba300e07174d236e32a7f9c7a13
F src/whereexpr.c 53452fe2fb07be2f4cb17f55cc721416fae0092c00717f106faf289c990b6494
@@ -1910,7 +1910,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P f13e0d12f137cd6b4e83e92bd36652431d8416e6eed4797313e061f2e390d39b
R f9de7ca9f057059fa9533f91ff672e89
P 9c9ba36e859e330c50ed40ede4b93eeb0a5c3337240465d953a7be9115a81390
R 05e155a5881f070d054d078349eab572
T *branch * distinct-agg-opt
T *sym-distinct-agg-opt *
T -sym-trunk *
U dan
Z e860451ea7454ca8ef2d0c3618d92e27
Z a27a5de08cdd8303a3cf21c478413fb4
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
9c9ba36e859e330c50ed40ede4b93eeb0a5c3337240465d953a7be9115a81390
eb919611fd2f255e4ad1fe7db633363793169f6cf99c650eaefa48c022eb5d22
src/select.c
+97 -61
View File
@@ -740,6 +740,7 @@ static void codeOffset(
}
}
#if 0
/*
** Add code that will check to make sure the N registers starting at iMem
** form a distinct entry. iTab is a sorting index that holds previously
@@ -767,6 +768,78 @@ static void codeDistinct(
sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
sqlite3ReleaseTempReg(pParse, r1);
}
#endif
static void codeDistinct2(
Parse *pParse, /* Parsing and code generating context */
int eTnctType, /* WHERE_DISTINCT_* value */
int iTab, /* A sorting index used to test for distinctness */
int iTabAddr, /* Address of OP_OpenEphemeral instruction for iTab */
int addrRepeat, /* Jump to here if not distinct */
ExprList *pEList, /* Expression for each element */
int regElem /* First element */
){
int nResultCol = pEList->nExpr;
Vdbe *v = pParse->pVdbe;
switch( eTnctType ){
case WHERE_DISTINCT_ORDERED: {
int i;
VdbeOp *pOp; /* No longer required OpenEphemeral instr. */
int iJump; /* Jump destination */
int regPrev; /* Previous row content */
/* Allocate space for the previous row */
regPrev = pParse->nMem+1;
pParse->nMem += nResultCol;
/* Change the OP_OpenEphemeral coded earlier to an OP_Null
** sets the MEM_Cleared bit on the first register of the
** previous value. This will cause the OP_Ne below to always
** fail on the first iteration of the loop even if the first
** row is all NULLs. */
sqlite3VdbeChangeToNoop(v, iTabAddr);
pOp = sqlite3VdbeGetOp(v, iTabAddr);
pOp->opcode = OP_Null;
pOp->p1 = 1;
pOp->p2 = regPrev;
pOp = 0; /* Ensure pOp is not used after sqlite3VdbeAddOp() */
iJump = sqlite3VdbeCurrentAddr(v) + nResultCol;
for(i=0; i<nResultCol; i++){
CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[i].pExpr);
if( i<nResultCol-1 ){
sqlite3VdbeAddOp3(v, OP_Ne, regElem+i, iJump, regPrev+i);
VdbeCoverage(v);
}else{
sqlite3VdbeAddOp3(v, OP_Eq, regElem+i, addrRepeat, regPrev+i);
VdbeCoverage(v);
}
sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ);
sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
}
assert( sqlite3VdbeCurrentAddr(v)==iJump || pParse->db->mallocFailed );
sqlite3VdbeAddOp3(v, OP_Copy, regElem, regPrev, nResultCol-1);
break;
}
case WHERE_DISTINCT_UNIQUE: {
sqlite3VdbeChangeToNoop(v, iTabAddr);
break;
}
default: {
int r1 = sqlite3GetTempReg(pParse);
sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, regElem, nResultCol);
VdbeCoverage(v);
sqlite3VdbeAddOp3(v, OP_MakeRecord, regElem, nResultCol, r1);
sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r1, regElem, nResultCol);
sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
sqlite3ReleaseTempReg(pParse, r1);
break;
}
}
}
#ifdef SQLITE_ENABLE_SORTER_REFERENCES
/*
@@ -1013,59 +1086,9 @@ static void selectInnerLoop(
** part of the result.
*/
if( hasDistinct ){
switch( pDistinct->eTnctType ){
case WHERE_DISTINCT_ORDERED: {
VdbeOp *pOp; /* No longer required OpenEphemeral instr. */
int iJump; /* Jump destination */
int regPrev; /* Previous row content */
/* Allocate space for the previous row */
regPrev = pParse->nMem+1;
pParse->nMem += nResultCol;
/* Change the OP_OpenEphemeral coded earlier to an OP_Null
** sets the MEM_Cleared bit on the first register of the
** previous value. This will cause the OP_Ne below to always
** fail on the first iteration of the loop even if the first
** row is all NULLs.
*/
sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct);
pOp = sqlite3VdbeGetOp(v, pDistinct->addrTnct);
pOp->opcode = OP_Null;
pOp->p1 = 1;
pOp->p2 = regPrev;
pOp = 0; /* Ensure pOp is not used after sqlite3VdbeAddOp() */
iJump = sqlite3VdbeCurrentAddr(v) + nResultCol;
for(i=0; i<nResultCol; i++){
CollSeq *pColl = sqlite3ExprCollSeq(pParse, p->pEList->a[i].pExpr);
if( i<nResultCol-1 ){
sqlite3VdbeAddOp3(v, OP_Ne, regResult+i, iJump, regPrev+i);
VdbeCoverage(v);
}else{
sqlite3VdbeAddOp3(v, OP_Eq, regResult+i, iContinue, regPrev+i);
VdbeCoverage(v);
}
sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ);
sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
}
assert( sqlite3VdbeCurrentAddr(v)==iJump || pParse->db->mallocFailed );
sqlite3VdbeAddOp3(v, OP_Copy, regResult, regPrev, nResultCol-1);
break;
}
case WHERE_DISTINCT_UNIQUE: {
sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct);
break;
}
default: {
assert( pDistinct->eTnctType==WHERE_DISTINCT_UNORDERED );
codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol,
regResult);
break;
}
}
assert( nResultCol==p->pEList->nExpr );
codeDistinct2(pParse, pDistinct->eTnctType, pDistinct->tabTnct,
pDistinct->addrTnct, iContinue, p->pEList, regResult);
if( pSort==0 ){
codeOffset(v, p->iOffset, iContinue);
}
@@ -5635,8 +5658,8 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){
pFunc->iDistinct = -1;
}else{
KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pE->x.pList,0,0);
sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0,
(char*)pKeyInfo, P4_KEYINFO);
pFunc->iDistAddr = sqlite3VdbeAddOp4(v, OP_OpenEphemeral,
pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO);
}
}
}
@@ -5668,7 +5691,12 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){
** registers if register regAcc contains 0. The caller will take care
** of setting and clearing regAcc.
*/
static void updateAccumulator(Parse *pParse, int regAcc, AggInfo *pAggInfo){
static void updateAccumulator(
Parse *pParse,
int regAcc,
AggInfo *pAggInfo,
int eDistinctType
){
Vdbe *v = pParse->pVdbe;
int i;
int regHit = 0;
@@ -5714,13 +5742,14 @@ static void updateAccumulator(Parse *pParse, int regAcc, AggInfo *pAggInfo){
nArg = 0;
regAgg = 0;
}
if( pF->iDistinct>=0 ){
if( pF->iDistinct>=0 && pList ){
if( addrNext==0 ){
addrNext = sqlite3VdbeMakeLabel(pParse);
}
testcase( nArg==0 ); /* Error condition */
testcase( nArg>1 ); /* Also an error */
codeDistinct(pParse, pF->iDistinct, addrNext, 1, regAgg);
codeDistinct2(pParse, eDistinctType, pF->iDistinct, pF->iDistAddr,
addrNext, pList, regAgg);
}
if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){
CollSeq *pColl = 0;
@@ -6905,7 +6934,7 @@ int sqlite3Select(
** the current row
*/
sqlite3VdbeJumpHere(v, addr1);
updateAccumulator(pParse, iUseFlag, pAggInfo);
updateAccumulator(pParse, iUseFlag, pAggInfo, WHERE_DISTINCT_UNORDERED);
sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag);
VdbeComment((v, "indicate data in accumulator"));
@@ -7025,6 +7054,8 @@ int sqlite3Select(
explainSimpleCount(pParse, pTab, pBest);
}else{
int regAcc = 0; /* "populate accumulators" flag */
ExprList *pDistinct = 0;
u16 distFlag = 0;
/* If there are accumulator registers but no min() or max() functions
** without FILTER clauses, allocate register regAcc. Register regAcc
@@ -7048,6 +7079,9 @@ int sqlite3Select(
regAcc = ++pParse->nMem;
sqlite3VdbeAddOp2(v, OP_Integer, 0, regAcc);
}
}else if( pAggInfo->nFunc==1 && pAggInfo->aFunc[0].iDistinct>=0 ){
pDistinct = pAggInfo->aFunc[0].pFExpr->x.pList;
distFlag = pDistinct ? WHERE_WANT_DISTINCT : 0;
}
/* This case runs if the aggregate has no GROUP BY clause. The
@@ -7067,12 +7101,14 @@ int sqlite3Select(
SELECTTRACE(1,pParse,p,("WhereBegin\n"));
pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy,
0, minMaxFlag, 0);
pDistinct, minMaxFlag|distFlag, 0);
if( pWInfo==0 ){
goto select_end;
}
SELECTTRACE(1,pParse,p,("WhereBegin returns\n"));
updateAccumulator(pParse, regAcc, pAggInfo);
updateAccumulator(
pParse, regAcc, pAggInfo, sqlite3WhereIsDistinct(pWInfo)
);
if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc);
if( minMaxFlag ){
sqlite3WhereMinMaxOptEarlyOut(v, pWInfo);
src/sqliteInt.h
+1
View File
@@ -2598,6 +2598,7 @@ struct AggInfo {
FuncDef *pFunc; /* The aggregate function implementation */
int iMem; /* Memory location that acts as accumulator */
int iDistinct; /* Ephemeral table used to enforce DISTINCT */
int iDistAddr; /* Addres of OP_OpenEphemeral */
} *aFunc;
int nFunc; /* Number of entries in aFunc[] */
u32 selId; /* Select to which this AggInfo belongs */
src/where.c
+5 -4
View File
@@ -492,7 +492,7 @@ static int findIndexCol(
for(i=0; i<pList->nExpr; i++){
Expr *p = sqlite3ExprSkipCollateAndLikely(pList->a[i].pExpr);
if( ALWAYS(p!=0)
&& p->op==TK_COLUMN
&& (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN)
&& p->iColumn==pIdx->aiColumn[iCol]
&& p->iTable==iBase
){
@@ -557,7 +557,8 @@ static int isDistinctRedundant(
for(i=0; i<pDistinct->nExpr; i++){
Expr *p = sqlite3ExprSkipCollateAndLikely(pDistinct->a[i].pExpr);
if( NEVER(p==0) ) continue;
if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1;
if( p->op!=TK_COLUMN && p->op!=TK_AGG_COLUMN ) continue;
if( p->iTable==iBase && p->iColumn<0 ) return 1;
}
/* Loop through all indices on the table, checking each to see if it makes
@@ -3822,7 +3823,7 @@ static i8 wherePathSatisfiesOrderBy(
if( MASKBIT(i) & obSat ) continue;
pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr);
if( NEVER(pOBExpr==0) ) continue;
if( pOBExpr->op!=TK_COLUMN ) continue;
if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue;
if( pOBExpr->iTable!=iCur ) continue;
pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn,
~ready, eqOpMask, 0);
@@ -3951,7 +3952,7 @@ static i8 wherePathSatisfiesOrderBy(
if( NEVER(pOBExpr==0) ) continue;
if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0;
if( iColumn>=XN_ROWID ){
if( pOBExpr->op!=TK_COLUMN ) continue;
if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue;
if( pOBExpr->iTable!=iCur ) continue;
if( pOBExpr->iColumn!=iColumn ) continue;
}else{