diff --git a/manifest b/manifest index 64d80c2f59..76e28676a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\srestriction\s(9)\sto\sthe\spush-down\soptimization:\s\sIf\sthe\ssubquery\sis\na\scompound\sthen\sall\sarms\sof\sthe\scompound\smust\shave\sthe\ssame\saffinity.\ndbsqlfuzz\s3a548de406a50e896c1bf7142692d35d339d697f. -D 2022-11-25T15:52:00.241 +C Immprove\sthe\squery\splanner\ssuch\sthat\sit\sis\sable\sto\smake\suse\sof\sindexed\nexpressions\swithin\san\saggregate\squery\swith\sGROUP\sBY.\s\sThis\simplements\nenhancement\srequest\s[99378177930f87bd]. +D 2022-11-25T16:10:48.068 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c bc6527e3dff813c8102418e6e201870626a7fa5f69329ea7b082d602e7ed1cd9 +F src/expr.c 9e7fadc664b938c18f006be0d4f6669888f9302756ee204420c7eccaed5435a6 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d @@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c bafe6424e942aad558b2d4be8dbcf5e35ce427ce3cf66d7f0c0ac37c366b00c6 +F src/select.c 7dcab394efe24c6016c6ed7958d426bfcdef57f50c2520dcccda27d9df28fdb6 F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f -F src/sqliteInt.h 6a24230f2928b3d1d9b0fdbedb98c862b828a4b1a9170306108e74cd6277f476 +F src/sqliteInt.h 5dd5d3d47f40b6a12be4a5fc131673bfe00c00373ed266ff4c4ec05d1991e69f F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -1628,6 +1628,7 @@ F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 +F test/tkt-99378177930f87bd.test 0f932e85fa1d41f30532cb7be9718d82e491e953123b8c4c85cf025f36ffe34b F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 @@ -2059,8 +2060,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a2b6883ac2ef878f525ee847b170beb9f9ab9d1fa67557101be2cdae1e7f7a57 -R 9c0acbc69bb427be679e111e4ec839ec +P 1ad41840c5e0fa702ba2c0df77a3ea126bd695b910b5d1271fa3129c38c58f5f e3e1b453dc47884ddc9e51c4302fa2f4d40bca3d69ac7c13d8e2ae1adc81ac56 +R eb048b7302ca70762f9994708c895609 +T +closed e3e1b453dc47884ddc9e51c4302fa2f4d40bca3d69ac7c13d8e2ae1adc81ac56 U drh -Z ae1fb8eb841aaf444344209ddd2c7ea7 +Z 6fdb43be076439dd6a7f7f9247393df9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4e3e4aa9d9..ff7ccc702c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1ad41840c5e0fa702ba2c0df77a3ea126bd695b910b5d1271fa3129c38c58f5f \ No newline at end of file +b9190d3da70c41717eb188474fd225ee43d0b46646e1b03de5967bd332553870 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index fb718eb23e..d30ae97668 100644 --- a/src/expr.c +++ b/src/expr.c @@ -53,7 +53,7 @@ char sqlite3ExprAffinity(const Expr *pExpr){ } op = pExpr->op; if( op==TK_REGISTER ) op = pExpr->op2; - if( op==TK_COLUMN || op==TK_AGG_COLUMN ){ + if( op==TK_COLUMN || (op==TK_AGG_COLUMN && pExpr->y.pTab!=0) ){ assert( ExprUseYTab(pExpr) ); assert( pExpr->y.pTab!=0 ); return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); @@ -173,7 +173,9 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ while( p ){ int op = p->op; if( op==TK_REGISTER ) op = p->op2; - if( op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER ){ + if( (op==TK_AGG_COLUMN && p->y.pTab!=0) + || op==TK_COLUMN || op==TK_TRIGGER + ){ int j; assert( ExprUseYTab(p) ); assert( p->y.pTab!=0 ); @@ -4128,15 +4130,16 @@ expr_code_doover: assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); pCol = &pAggInfo->aCol[pExpr->iAgg]; if( !pAggInfo->directMode ){ - assert( pCol->iMem>0 ); - return pCol->iMem; + return AggInfoColumnReg(pAggInfo, pExpr->iAgg); }else if( pAggInfo->useSortingIdx ){ Table *pTab = pCol->pTab; sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pCol->iSorterColumn, target); - if( pCol->iColumn<0 ){ + if( pTab==0 ){ + /* No comment added */ + }else if( pCol->iColumn<0 ){ VdbeComment((v,"%s.rowid",pTab->zName)); - }else if( ALWAYS(pTab!=0) ){ + }else{ VdbeComment((v,"%s.%s", pTab->zName, pTab->aCol[pCol->iColumn].zCnName)); if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){ @@ -4144,6 +4147,11 @@ expr_code_doover: } } return target; + }else if( pExpr->y.pTab==0 ){ + /* This case happens when the argument to an aggregate function + ** is rewritten by aggregateConvertIndexedExprRefToColumn() */ + sqlite3VdbeAddOp3(v, OP_Column, pExpr->iTable, pExpr->iColumn, target); + return target; } /* Otherwise, fall thru into the TK_COLUMN case */ /* no break */ deliberate_fall_through @@ -4441,7 +4449,7 @@ expr_code_doover: assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3ErrorMsg(pParse, "misuse of aggregate: %#T()", pExpr); }else{ - return pInfo->aFunc[pExpr->iAgg].iMem; + return AggInfoFuncReg(pInfo, pExpr->iAgg); } break; } @@ -4730,7 +4738,7 @@ expr_code_doover: if( pAggInfo ){ assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); if( !pAggInfo->directMode ){ - inReg = pAggInfo->aCol[pExpr->iAgg].iMem; + inReg = AggInfoColumnReg(pAggInfo, pExpr->iAgg); break; } if( pExpr->pAggInfo->useSortingIdx ){ @@ -6242,6 +6250,72 @@ static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ return i; } +/* +** Search the AggInfo object for an aCol[] entry that has iTable and iColumn. +** Return the index in aCol[] of the entry that describes that column. +** +** If no prior entry is found, create a new one and return -1. The +** new column will have an idex of pAggInfo->nColumn-1. +*/ +static void findOrCreateAggInfoColumn( + Parse *pParse, /* Parsing context */ + AggInfo *pAggInfo, /* The AggInfo object to search and/or modify */ + Expr *pExpr /* Expr describing the column to find or insert */ +){ + struct AggInfo_col *pCol; + int k; + + assert( pAggInfo->iFirstReg==0 ); + pCol = pAggInfo->aCol; + for(k=0; knColumn; k++, pCol++){ + if( pCol->iTable==pExpr->iTable + && pCol->iColumn==pExpr->iColumn + && pExpr->op!=TK_IF_NULL_ROW + ){ + goto fix_up_expr; + } + } + k = addAggInfoColumn(pParse->db, pAggInfo); + if( k<0 ){ + /* OOM on resize */ + assert( pParse->db->mallocFailed ); + return; + } + pCol = &pAggInfo->aCol[k]; + assert( ExprUseYTab(pExpr) ); + pCol->pTab = pExpr->y.pTab; + pCol->iTable = pExpr->iTable; + pCol->iColumn = pExpr->iColumn; + pCol->iSorterColumn = -1; + pCol->pCExpr = pExpr; + if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ + int j, n; + ExprList *pGB = pAggInfo->pGroupBy; + struct ExprList_item *pTerm = pGB->a; + n = pGB->nExpr; + for(j=0; jpExpr; + if( pE->op==TK_COLUMN + && pE->iTable==pExpr->iTable + && pE->iColumn==pExpr->iColumn + ){ + pCol->iSorterColumn = j; + break; + } + } + } + if( pCol->iSorterColumn<0 ){ + pCol->iSorterColumn = pAggInfo->nSortingColumn++; + } +fix_up_expr: + ExprSetVVAProperty(pExpr, EP_NoReduce); + pExpr->pAggInfo = pAggInfo; + if( pExpr->op==TK_COLUMN ){ + pExpr->op = TK_AGG_COLUMN; + } + pExpr->iAgg = (i16)k; +} + /* ** This is the xExprCallback for a tree walker. It is used to ** implement sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates @@ -6255,7 +6329,36 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ AggInfo *pAggInfo = pNC->uNC.pAggInfo; assert( pNC->ncFlags & NC_UAggInfo ); + assert( pAggInfo->iFirstReg==0 ); switch( pExpr->op ){ + default: { + IndexedExpr *pIEpr; + Expr tmp; + assert( pParse->iSelfTab==0 ); + if( (pNC->ncFlags & NC_InAggFunc)==0 ) break; + if( pParse->pIdxEpr==0 ) break; + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + int iDataCur = pIEpr->iDataCur; + if( iDataCur<0 ) continue; + if( sqlite3ExprCompare(0, pExpr, pIEpr->pExpr, iDataCur)==0 ) break; + } + if( pIEpr==0 ) break; + if( NEVER(!ExprUseYTab(pExpr)) ) break; + + /* If we reach this point, it means that expression pExpr can be + ** translated into a reference to an index column as described by + ** pIEpr. + */ + memset(&tmp, 0, sizeof(tmp)); + tmp.op = TK_AGG_COLUMN; + tmp.iTable = pIEpr->iIdxCur; + tmp.iColumn = pIEpr->iIdxCol; + findOrCreateAggInfoColumn(pParse, pAggInfo, &tmp); + pAggInfo->aCol[tmp.iAgg].pCExpr = pExpr; + pExpr->pAggInfo = pAggInfo; + pExpr->iAgg = tmp.iAgg; + return WRC_Prune; + } case TK_IF_NULL_ROW: case TK_AGG_COLUMN: case TK_COLUMN: { @@ -6267,67 +6370,9 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ if( ALWAYS(pSrcList!=0) ){ SrcItem *pItem = pSrcList->a; for(i=0; inSrc; i++, pItem++){ - struct AggInfo_col *pCol; assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); if( pExpr->iTable==pItem->iCursor ){ - /* If we reach this point, it means that pExpr refers to a table - ** that is in the FROM clause of the aggregate query. - ** - ** Make an entry for the column in pAggInfo->aCol[] if there - ** is not an entry there already. - */ - int k; - pCol = pAggInfo->aCol; - for(k=0; knColumn; k++, pCol++){ - if( pCol->iTable==pExpr->iTable - && pCol->iColumn==pExpr->iColumn - && pExpr->op!=TK_IF_NULL_ROW - ){ - break; - } - } - if( (k>=pAggInfo->nColumn) - && (k = addAggInfoColumn(pParse->db, pAggInfo))>=0 - ){ - pCol = &pAggInfo->aCol[k]; - assert( ExprUseYTab(pExpr) ); - pCol->pTab = pExpr->y.pTab; - pCol->iTable = pExpr->iTable; - pCol->iColumn = pExpr->iColumn; - pCol->iMem = ++pParse->nMem; - pCol->iSorterColumn = -1; - pCol->pCExpr = pExpr; - if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ - int j, n; - ExprList *pGB = pAggInfo->pGroupBy; - struct ExprList_item *pTerm = pGB->a; - n = pGB->nExpr; - for(j=0; jpExpr; - if( pE->op==TK_COLUMN - && pE->iTable==pExpr->iTable - && pE->iColumn==pExpr->iColumn - ){ - pCol->iSorterColumn = j; - break; - } - } - } - if( pCol->iSorterColumn<0 ){ - pCol->iSorterColumn = pAggInfo->nSortingColumn++; - } - } - /* There is now an entry for pExpr in pAggInfo->aCol[] (either - ** because it was there before or because we just created it). - ** Convert the pExpr to be a TK_AGG_COLUMN referring to that - ** pAggInfo->aCol[] entry. - */ - ExprSetVVAProperty(pExpr, EP_NoReduce); - pExpr->pAggInfo = pAggInfo; - if( pExpr->op==TK_COLUMN ){ - pExpr->op = TK_AGG_COLUMN; - } - pExpr->iAgg = (i16)k; + findOrCreateAggInfoColumn(pParse, pAggInfo, pExpr); break; } /* endif pExpr->iTable==pItem->iCursor */ } /* end loop over pSrcList */ @@ -6357,7 +6402,6 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); pItem = &pAggInfo->aFunc[i]; pItem->pFExpr = pExpr; - pItem->iMem = ++pParse->nMem; assert( ExprUseUToken(pExpr) ); pItem->pFunc = sqlite3FindFunction(pParse->db, pExpr->u.zToken, diff --git a/src/select.c b/src/select.c index 65a0a06c33..067c98b4a3 100644 --- a/src/select.c +++ b/src/select.c @@ -6229,6 +6229,172 @@ void sqlite3SelectPrep( sqlite3SelectAddTypeInfo(pParse, p); } +#if TREETRACE_ENABLED +/* +** Display all information about an AggInfo object +*/ +static void printAggInfo(AggInfo *pAggInfo){ + int ii; + for(ii=0; iinColumn; ii++){ + struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; + sqlite3DebugPrintf( + "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" + " iSorterColumn=%d %s\n", + ii, pCol->pTab ? pCol->pTab->zName : "NULL", + pCol->iTable, pCol->iColumn, AggInfoColumnReg(pAggInfo,ii), + pCol->iSorterColumn, + ii>=pAggInfo->nAccumulator ? "" : " Accumulator"); + sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); + } + for(ii=0; iinFunc; ii++){ + sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", + ii, AggInfoFuncReg(pAggInfo,ii)); + sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); + } +} +#endif /* TREETRACE_ENABLED */ + +/* +** Analyze the arguments to aggregate functions. Create new pAggInfo->aCol[] +** entries for columns that are arguments to aggregate functions but which +** are not otherwise used. +** +** The aCol[] entries in AggInfo prior to nAccumulator are columns that +** are referenced outside of aggregate functions. These might be columns +** that are part of the GROUP by clause, for example. Other database engines +** would throw an error if there is a column reference that is not in the +** GROUP BY clause and that is not part of an aggregate function argument. +** But SQLite allows this. +** +** The aCol[] entries beginning with the aCol[nAccumulator] and following +** are column references that are used exclusively as arguments to +** aggregate functions. This routine is responsible for computing +** (or recomputing) those aCol[] entries. +*/ +static void analyzeAggFuncArgs( + Parse *pParse, + AggInfo *pAggInfo, + NameContext *pNC +){ + int i; + assert( pAggInfo!=0 ); + assert( pAggInfo->iFirstReg==0 ); + pNC->ncFlags |= NC_InAggFunc; + for(i=0; inFunc; i++){ + Expr *pExpr = pAggInfo->aFunc[i].pFExpr; + assert( ExprUseXList(pExpr) ); + sqlite3ExprAnalyzeAggList(pNC, pExpr->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( !IsWindowFunc(pExpr) ); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ExprAnalyzeAggregates(pNC, pExpr->y.pWin->pFilter); + } +#endif + } + pNC->ncFlags &= ~NC_InAggFunc; +} + +/* +** An index on expressions is being used in the inner loop of an +** aggregate query with a GROUP BY clause. This routine attempts +** to adjust the AggInfo object to take advantage of index and to +** perhaps use the index as a covering index. +** +*/ +static void optimizeAggregateUseOfIndexedExpr( + Parse *pParse, /* Parsing context */ + Select *pSelect, /* The SELECT statement being processed */ + AggInfo *pAggInfo, /* The aggregate info */ + NameContext *pNC /* Name context used to resolve agg-func args */ +){ + assert( pAggInfo->iFirstReg==0 ); + pAggInfo->nColumn = pAggInfo->nAccumulator; + if( ALWAYS(pAggInfo->nSortingColumn>0) ){ + if( pAggInfo->nColumn==0 ){ + pAggInfo->nSortingColumn = 0; + }else{ + pAggInfo->nSortingColumn = + pAggInfo->aCol[pAggInfo->nColumn-1].iSorterColumn+1; + } + } + analyzeAggFuncArgs(pParse, pAggInfo, pNC); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + IndexedExpr *pIEpr; + TREETRACE(0x20, pParse, pSelect, + ("AggInfo (possibly) adjusted for Indexed Exprs\n")); + sqlite3TreeViewSelect(0, pSelect, 0); + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + printf("data-cursor=%d index={%d,%d}\n", + pIEpr->iDataCur, pIEpr->iIdxCur, pIEpr->iIdxCol); + sqlite3TreeViewExpr(0, pIEpr->pExpr, 0); + } + printAggInfo(pAggInfo); + } +#else + (void)pSelect; /* Suppress unused-parameter warnings */ +#endif +} + +/* +** Walker callback for aggregateConvertIndexedExprRefToColumn(). +*/ +static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ + AggInfo *pAggInfo; + struct AggInfo_col *pCol; + if( pExpr->pAggInfo==0 ) return WRC_Continue; + if( pExpr->op==TK_AGG_COLUMN ) return WRC_Continue; + if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue; + if( pExpr->op==TK_IF_NULL_ROW ) return WRC_Continue; + pAggInfo = pExpr->pAggInfo; + assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + pCol = &pAggInfo->aCol[pExpr->iAgg]; + pExpr->op = TK_AGG_COLUMN; + pExpr->iTable = pCol->iTable; + pExpr->iColumn = pCol->iColumn; + return WRC_Prune; +} + +/* +** Convert every pAggInfo->aFunc[].pExpr such that any node within +** those expressions that has pAppInfo set is changed into a TK_AGG_COLUMN +** opcode. +*/ +static void aggregateConvertIndexedExprRefToColumn(AggInfo *pAggInfo){ + int i; + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = aggregateIdxEprRefToColCallback; + for(i=0; inFunc; i++){ + sqlite3WalkExpr(&w, pAggInfo->aFunc[i].pFExpr); + } +} + + +/* +** Allocate a block of registers so that there is one register for each +** pAggInfo->aCol[] and pAggInfo->aFunc[] entry in pAggInfo. The first +** register in this block is stored in pAggInfo->iFirstReg. +** +** This routine may only be called once for each AggInfo object. Prior +** to calling this routine: +** +** * The aCol[] and aFunc[] arrays may be modified +** * The AggInfoColumnReg() and AggInfoFuncReg() macros may not be used +** +** After clling this routine: +** +** * The aCol[] and aFunc[] arrays are fixed +** * The AggInfoColumnReg() and AggInfoFuncReg() macros may be used +** +*/ +static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ + assert( pAggInfo!=0 ); + assert( pAggInfo->iFirstReg==0 ); + pAggInfo->iFirstReg = pParse->nMem + 1; + pParse->nMem += pAggInfo->nColumn + pAggInfo->nFunc; +} + /* ** Reset the aggregate accumulator. ** @@ -6242,24 +6408,13 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ int i; struct AggInfo_func *pFunc; int nReg = pAggInfo->nFunc + pAggInfo->nColumn; + assert( pAggInfo->iFirstReg>0 ); assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); if( nReg==0 ) return; if( pParse->nErr ) return; -#ifdef SQLITE_DEBUG - /* Verify that all AggInfo registers are within the range specified by - ** AggInfo.mnReg..AggInfo.mxReg */ - assert( nReg==pAggInfo->mxReg-pAggInfo->mnReg+1 ); - for(i=0; inColumn; i++){ - assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg - && pAggInfo->aCol[i].iMem<=pAggInfo->mxReg ); - } - for(i=0; inFunc; i++){ - assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg - && pAggInfo->aFunc[i].iMem<=pAggInfo->mxReg ); - } -#endif - sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mxReg); + sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->iFirstReg, + pAggInfo->iFirstReg+nReg-1); for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ if( pFunc->iDistinct>=0 ){ Expr *pE = pFunc->pFExpr; @@ -6291,15 +6446,16 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); pList = pF->pFExpr->x.pList; - sqlite3VdbeAddOp2(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0); + sqlite3VdbeAddOp2(v, OP_AggFinal, AggInfoFuncReg(pAggInfo,i), + pList ? pList->nExpr : 0); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); } } /* -** Update the accumulator memory cells for an aggregate based on -** the current cursor position. +** Generate code that will update the accumulator memory cells for an +** aggregate based on the current cursor position. ** ** If regAcc is non-zero and there are no min() or max() aggregates ** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator @@ -6319,6 +6475,8 @@ static void updateAccumulator( struct AggInfo_func *pF; struct AggInfo_col *pC; + assert( pAggInfo->iFirstReg>0 ); + if( pParse->nErr ) return; pAggInfo->directMode = 1; for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ int nArg; @@ -6379,7 +6537,7 @@ static void updateAccumulator( if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, pF->iMem); + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); sqlite3ReleaseTempRange(pParse, regAgg, nArg); @@ -6394,7 +6552,7 @@ static void updateAccumulator( addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v); } for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ - sqlite3ExprCode(pParse, pC->pCExpr, pC->iMem); + sqlite3ExprCode(pParse, pC->pCExpr, AggInfoColumnReg(pAggInfo,i)); } pAggInfo->directMode = 0; @@ -6654,30 +6812,6 @@ static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){ return 0; } -#if TREETRACE_ENABLED -/* -** Display all information about an AggInfo object -*/ -static void printAggInfo(AggInfo *pAggInfo){ - int ii; - for(ii=0; iinColumn; ii++){ - struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; - sqlite3DebugPrintf( - "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" - " iSorterColumn=%d\n", - ii, pCol->pTab ? pCol->pTab->zName : "NULL", - pCol->iTable, pCol->iColumn, pCol->iMem, - pCol->iSorterColumn); - sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); - } - for(ii=0; iinFunc; ii++){ - sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", - ii, pAggInfo->aFunc[ii].iMem); - sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); - } -} -#endif /* TREETRACE_ENABLED */ - /* ** Generate code for the SELECT statement given in the p argument. ** @@ -7439,7 +7573,6 @@ int sqlite3Select( sNC.pSrcList = pTabList; sNC.uNC.pAggInfo = pAggInfo; VVA_ONLY( sNC.ncFlags = NC_UAggInfo; ) - pAggInfo->mnReg = pParse->nMem+1; pAggInfo->nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; pAggInfo->pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); @@ -7460,20 +7593,7 @@ int sqlite3Select( }else{ minMaxFlag = WHERE_ORDERBY_NORMAL; } - for(i=0; inFunc; i++){ - Expr *pExpr = pAggInfo->aFunc[i].pFExpr; - assert( ExprUseXList(pExpr) ); - sNC.ncFlags |= NC_InAggFunc; - sqlite3ExprAnalyzeAggList(&sNC, pExpr->x.pList); -#ifndef SQLITE_OMIT_WINDOWFUNC - assert( !IsWindowFunc(pExpr) ); - if( ExprHasProperty(pExpr, EP_WinFunc) ){ - sqlite3ExprAnalyzeAggregates(&sNC, pExpr->y.pWin->pFilter); - } -#endif - sNC.ncFlags &= ~NC_InAggFunc; - } - pAggInfo->mxReg = pParse->nMem; + analyzeAggFuncArgs(pParse, pAggInfo, &sNC); if( db->mallocFailed ) goto select_end; #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x20 ){ @@ -7561,6 +7681,10 @@ int sqlite3Select( sqlite3ExprListDelete(db, pDistinct); goto select_end; } + if( pParse->pIdxEpr ){ + optimizeAggregateUseOfIndexedExpr(pParse, p, pAggInfo, &sNC); + } + assignAggregateRegisters(pParse, pAggInfo); eDist = sqlite3WhereIsDistinct(pWInfo); TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ @@ -7621,6 +7745,23 @@ int sqlite3Select( pAggInfo->useSortingIdx = 1; } + /* If there entries in pAgggInfo->aFunc[] that contain subexpressions + ** that are indexed (and that were previously identified and tagged + ** in optimizeAggregateUseOfIndexedExpr()) then those subexpressions + ** must now be converted into a TK_AGG_COLUMN node so that the value + ** is correctly pulled from the index rather than being recomputed. */ + if( pParse->pIdxEpr ){ + aggregateConvertIndexedExprRefToColumn(pAggInfo); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20, pParse, p, + ("AggInfo function expressions converted to reference index\n")); + sqlite3TreeViewSelect(0, p, 0); + printAggInfo(pAggInfo); + } +#endif + } + /* If the index or temporary table used by the GROUP BY sort ** will naturally deliver rows in the order required by the ORDER BY ** clause, cancel the ephemeral table open coded earlier. @@ -7799,7 +7940,8 @@ int sqlite3Select( if( pKeyInfo ){ sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO); } - sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem); + assignAggregateRegisters(pParse, pAggInfo); + sqlite3VdbeAddOp2(v, OP_Count, iCsr, AggInfoFuncReg(pAggInfo,0)); sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); }else{ @@ -7835,6 +7977,7 @@ int sqlite3Select( pDistinct = pAggInfo->aFunc[0].pFExpr->x.pList; distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0; } + assignAggregateRegisters(pParse, pAggInfo); /* This case runs if the aggregate has no GROUP BY clause. The ** processing is much simpler since there is only a single row @@ -7918,7 +8061,7 @@ select_end: if( pAggInfo && !db->mallocFailed ){ for(i=0; inColumn; i++){ Expr *pExpr = pAggInfo->aCol[i].pCExpr; - assert( pExpr!=0 ); + if( pExpr==0 ) continue; assert( pExpr->pAggInfo==pAggInfo ); assert( pExpr->iAgg==i ); } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 86f33dec9e..db65082bbc 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2739,16 +2739,15 @@ struct AggInfo { ** from source tables rather than from accumulators */ u8 useSortingIdx; /* In direct mode, reference the sorting index rather ** than the source table */ + u16 nSortingColumn; /* Number of columns in the sorting index */ int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ - int nSortingColumn; /* Number of columns in the sorting index */ - int mnReg, mxReg; /* Range of registers allocated for aCol and aFunc */ + int iFirstReg; /* First register in range for aCol[] and aFunc[] */ ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ Expr *pCExpr; /* The original expression */ int iTable; /* Cursor number of the source table */ - int iMem; /* Memory location that acts as accumulator */ i16 iColumn; /* Column number within the source table */ i16 iSorterColumn; /* Column number in the sorting index */ } *aCol; @@ -2759,7 +2758,6 @@ struct AggInfo { struct AggInfo_func { /* For each aggregate function */ Expr *pFExpr; /* Expression encoding the function */ FuncDef *pFunc; /* The aggregate function implementation */ - int iMem; /* Memory location that acts as accumulator */ int iDistinct; /* Ephemeral table used to enforce DISTINCT */ int iDistAddr; /* Address of OP_OpenEphemeral */ } *aFunc; @@ -2767,6 +2765,17 @@ struct AggInfo { u32 selId; /* Select to which this AggInfo belongs */ }; +/* +** Macros to compute aCol[] and aFunc[] register numbers. +** +** These macros should not be used prior to the call to +** assignAggregateRegisters() that computes the value of pAggInfo->iFirstReg. +** The assert()s that are part of this macro verify that constraint. +*/ +#define AggInfoColumnReg(A,I) (assert((A)->iFirstReg),(A)->iFirstReg+(I)) +#define AggInfoFuncReg(A,I) \ + (assert((A)->iFirstReg),(A)->iFirstReg+(A)->nColumn+(I)) + /* ** The datatype ynVar is a signed integer, either 16-bit or 32-bit. ** Usually it is 16-bits. But if SQLITE_MAX_VARIABLE_NUMBER is greater diff --git a/test/tkt-99378177930f87bd.test b/test/tkt-99378177930f87bd.test new file mode 100644 index 0000000000..b21be3f67f --- /dev/null +++ b/test/tkt-99378177930f87bd.test @@ -0,0 +1,154 @@ +# 2022-11-23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file implements tests to verify that the enhancement +# request documented by ticket 99378177930f87bd is working. +# +# The enhancement is that if an aggregate query with a GROUP BY clause +# uses subexpressions in the arguments to aggregate functions that are +# also columns of an index, then the values are pulled from the index +# rather than being recomputed. This has the potential to make some +# indexed queries works as if the index were covering. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test tkt-99378-100 { + CREATE TABLE t1(a INT, b TEXT, c INT, d INT); + INSERT INTO t1(a,b,c,d) VALUES + (1, '{"x":1}', 12, 3), + (1, '{"x":2}', 4, 5), + (1, '{"x":1}', 6, 11), + (2, '{"x":1}', 22, 3), + (2, '{"x":2}', 4, 5), + (3, '{"x":1}', 6, 7); + CREATE INDEX t1x ON t1(d, a, b->>'x', c); +} {} +do_execsql_test tkt-99378-110 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} + +# The proof that the index on the expression is being used is in the +# fact that the byte code contains no "Function" opcodes. In other words, +# the ->> operator (which is implemented by a function) is never invoked. +# Instead, the b->>'x' value is pulled out of the index. +# +do_execsql_test tkt-99378-120 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} + + +do_execsql_test tkt-99378-130 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY +a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} +do_execsql_test tkt-99378-140 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY +a; +} {~/Function/} + +do_execsql_test tkt-99378-200 { + DROP INDEX t1x; + CREATE INDEX t1x ON t1(a, d, b->>'x', c); +} +do_execsql_test tkt-99378-210 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} +do_execsql_test tkt-99378-220 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} +do_execsql_test tkt-99378-230 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} +do_execsql_test tkt-99378-240 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} + + + + +finish_test