From d737feeacfdbf6ea5d44e383647ea6ced390c3ab Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 20 Apr 2024 15:14:06 +0000 Subject: [PATCH 1/3] Fix a problem where an expression like (a, b) IN (SELECT ...) might not use an index on (a, b) if the affinities and collation sequences of "a" and "b" are not identical. FossilOrigin-Name: 4d870fd8b5450047a7486fc023d1ac9439642e8ed91eadfd5026c4cda7cc9179 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/where.c | 38 ++++++++++++++++++++++++++++++++++---- 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 98a7d3bada..5c55caac4b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correct\shandling\sof\sOUTER\sJOIN\swhen\son\sor\sthe\sother\soperand\sis\sa\ssubquery\nimplemented\susing\sthe\sVALUES-as-coroutine\soptimization.\ndbsqlfuzz\sbde3bf80aedf25afa56e2997a0545a314765d3f8. -D 2024-04-18T16:11:01.833 +C Fix\sa\sproblem\swhere\san\sexpression\slike\s(a,\sb)\sIN\s(SELECT\s...)\smight\snot\suse\san\sindex\son\s(a,\sb)\sif\sthe\saffinities\sand\scollation\ssequences\sof\s"a"\sand\s"b"\sare\snot\sidentical. +D 2024-04-20T15:14:06.554 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -835,7 +835,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c 40ee94edd4cf31e6dc3e4c9ac814777b8cf4dc8ee985d87eb5f77bd4c42e69da +F src/where.c b4eb20a2e5f74e299cb19095454dd257292ad2a23a90265a9eecd2390fd0552a F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 F src/wherecode.c 4b5909be0c36030eec18ea0e7a64c60254180a21e626c9d2e0e9be3007ad47c1 F src/whereexpr.c 7b64295f1d82ad0928df435925dd7bbd5997b44a026153113eace0d9e71ff435 @@ -2184,8 +2184,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6b21cccdeec92db9f6ce3dd7ea5e61b8b46650cc1e550271aa51bdc619f55b11 -R 742e51db40c512171314564bf9760205 -U drh -Z 2a179895cc5d13f95288c795c253907b +P 8c0f69e0e4ae0a446838cc193bfd4395fd251f3c7659b35ac388e5a0a7650a66 +R 76aaae3efae57f02d5fc2e79c67a8826 +T *branch * vector-in-fix +T *sym-vector-in-fix * +T -sym-trunk * +U dan +Z 8e2ed7e86b55b55a9e6d2d789c1dac73 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2e5a76fc98..c480de9800 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8c0f69e0e4ae0a446838cc193bfd4395fd251f3c7659b35ac388e5a0a7650a66 \ No newline at end of file +4d870fd8b5450047a7486fc023d1ac9439642e8ed91eadfd5026c4cda7cc9179 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 93984347b7..f6b0990e67 100644 --- a/src/where.c +++ b/src/where.c @@ -302,6 +302,28 @@ static Expr *whereRightSubexprIsColumn(Expr *p){ return 0; } +static int SQLITE_NOINLINE indexInAffinityOk( + Parse *pParse, + WhereTerm *pTerm, + u8 idxaff, + CollSeq **ppColl +){ + Expr *pX = pTerm->pExpr; + Expr inexpr; + + if( sqlite3ExprIsVector(pX->pLeft) ){ + int iField = pTerm->u.x.iField - 1; + inexpr.op = TK_EQ; + inexpr.pLeft = pX->pLeft->x.pList->a[iField].pExpr; + assert( ExprUseXSelect(pX) ); + inexpr.pRight = pX->x.pSelect->pEList->a[iField].pExpr; + pX = &inexpr; + } + + *ppColl = sqlite3ExprCompareCollSeq(pParse, pX); + return sqlite3IndexAffinityOk(pX, idxaff); +} + /* ** Advance to the next WhereTerm that matches according to the criteria ** established when the pScan object was initialized by whereScanInit(). @@ -355,11 +377,19 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ CollSeq *pColl; Parse *pParse = pWC->pWInfo->pParse; pX = pTerm->pExpr; - if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ - continue; + + if( (pTerm->eOperator & WO_IN) ){ + if( !indexInAffinityOk(pParse, pTerm, pScan->idxaff, &pColl) ){ + continue; + } + }else{ + if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ + continue; + } + assert(pX->pLeft); + pColl = sqlite3ExprCompareCollSeq(pParse, pX); } - assert(pX->pLeft); - pColl = sqlite3ExprCompareCollSeq(pParse, pX); + if( pColl==0 ) pColl = pParse->db->pDfltColl; if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ continue; From 5749e1ee4386a2a6c436dfeb45770f8352abdb65 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 20 Apr 2024 19:19:30 +0000 Subject: [PATCH 2/3] Slight performance improvement for the new code on this branch. FossilOrigin-Name: 500c67f1341fe2a7e7333d525c90df201cc73a683b943ad5c1e41d4a4f639043 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/where.c | 35 ++++++++++++++++++++++++----------- test/in5.test | 22 ++++++++++++++++++++++ 4 files changed, 54 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index 5c55caac4b..d652ae551d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swhere\san\sexpression\slike\s(a,\sb)\sIN\s(SELECT\s...)\smight\snot\suse\san\sindex\son\s(a,\sb)\sif\sthe\saffinities\sand\scollation\ssequences\sof\s"a"\sand\s"b"\sare\snot\sidentical. -D 2024-04-20T15:14:06.554 +C Slight\sperformance\simprovement\sfor\sthe\snew\scode\son\sthis\sbranch. +D 2024-04-20T19:19:30.713 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -835,7 +835,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c b4eb20a2e5f74e299cb19095454dd257292ad2a23a90265a9eecd2390fd0552a +F src/where.c 79c673bf23b3b7fcc5e9cfa0adcd88a97645cd47920d2365dd6aec8cab7b222a F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 F src/wherecode.c 4b5909be0c36030eec18ea0e7a64c60254180a21e626c9d2e0e9be3007ad47c1 F src/whereexpr.c 7b64295f1d82ad0928df435925dd7bbd5997b44a026153113eace0d9e71ff435 @@ -1274,7 +1274,7 @@ F test/in.test d1cad4ededd425568b2e39fb0c31fa9a3772311dd595801ff13ba3912b69bba6 F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test bb767ec1cfd1730256f0a83219f0acda36bc251b63f8b8bb7d8c7cff17875a4f -F test/in5.test b32ce7f4a93f44c5dee94af16886d922cc16ebe33c8e1765c73d4049d0f4b40f +F test/in5.test 4fd79c70dfa0681313e8cdca07f5ff0400bdc0e20f808a5c59eaef1e4b48082a F test/in6.test f5f40d6816a8bb7c784424b58a10ac38efb76ab29127a2c17399e0cbeeda0e4b F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822 F test/incrblob2.test a494c9e848560039a23974b9119cfc2cf3ad3bd15cc2694ee6367ae537ef8f1f @@ -2184,11 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8c0f69e0e4ae0a446838cc193bfd4395fd251f3c7659b35ac388e5a0a7650a66 -R 76aaae3efae57f02d5fc2e79c67a8826 -T *branch * vector-in-fix -T *sym-vector-in-fix * -T -sym-trunk * +P 4d870fd8b5450047a7486fc023d1ac9439642e8ed91eadfd5026c4cda7cc9179 +R 6023edc800b34da7492bfeaa8c433f44 U dan -Z 8e2ed7e86b55b55a9e6d2d789c1dac73 +Z caab3df5b31f1752021c68136ff6d06a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c480de9800..6bb6214d88 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d870fd8b5450047a7486fc023d1ac9439642e8ed91eadfd5026c4cda7cc9179 \ No newline at end of file +500c67f1341fe2a7e7333d525c90df201cc73a683b943ad5c1e41d4a4f639043 \ No newline at end of file diff --git a/src/where.c b/src/where.c index f6b0990e67..96a6df9deb 100644 --- a/src/where.c +++ b/src/where.c @@ -302,15 +302,25 @@ static Expr *whereRightSubexprIsColumn(Expr *p){ return 0; } -static int SQLITE_NOINLINE indexInAffinityOk( +/* +** Term pTerm is guaranteed to be a WO_IN term. It may be a component term +** of a vector IN expression of the form "(x, y, ...) IN (SELECT ...)". +** This function checks to see if the term is compatible with an index +** column with affinity idxaff (one of the SQLITE_AFF_XYZ values). If so, +** it returns a pointer to the name of the collation sequence (e.g. "BINARY" +** or "NOCASE") used by the comparison in pTerm. If it is not compatible +** with affinity idxaff, NULL is returned. +*/ +static SQLITE_NOINLINE const char *indexInAffinityOk( Parse *pParse, WhereTerm *pTerm, - u8 idxaff, - CollSeq **ppColl + u8 idxaff ){ Expr *pX = pTerm->pExpr; Expr inexpr; + assert( pTerm->eOperator & WO_IN ); + if( sqlite3ExprIsVector(pX->pLeft) ){ int iField = pTerm->u.x.iField - 1; inexpr.op = TK_EQ; @@ -320,8 +330,11 @@ static int SQLITE_NOINLINE indexInAffinityOk( pX = &inexpr; } - *ppColl = sqlite3ExprCompareCollSeq(pParse, pX); - return sqlite3IndexAffinityOk(pX, idxaff); + if( sqlite3IndexAffinityOk(pX, idxaff) ){ + CollSeq *pRet = sqlite3ExprCompareCollSeq(pParse, pX); + return pRet ? pRet->zName : sqlite3StrBINARY; + } + return 0; } /* @@ -374,24 +387,24 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ if( (pTerm->eOperator & pScan->opMask)!=0 ){ /* Verify the affinity and collating sequence match */ if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ - CollSeq *pColl; + const char *zCollName; Parse *pParse = pWC->pWInfo->pParse; pX = pTerm->pExpr; if( (pTerm->eOperator & WO_IN) ){ - if( !indexInAffinityOk(pParse, pTerm, pScan->idxaff, &pColl) ){ - continue; - } + zCollName = indexInAffinityOk(pParse, pTerm, pScan->idxaff); + if( !zCollName ) continue; }else{ + CollSeq *pColl; if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ continue; } assert(pX->pLeft); pColl = sqlite3ExprCompareCollSeq(pParse, pX); + zCollName = pColl ? pColl->zName : sqlite3StrBINARY; } - if( pColl==0 ) pColl = pParse->db->pDfltColl; - if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ + if( sqlite3StrICmp(zCollName, pScan->zCollName) ){ continue; } } diff --git a/test/in5.test b/test/in5.test index 6680641ff3..933eb90026 100644 --- a/test/in5.test +++ b/test/in5.test @@ -266,4 +266,26 @@ do_execsql_test 9.2 { SELECT lower('1e500') FROM t0 WHERE rowid != lower('1e500'); } {1e500} +#------------------------------------------------------------------------- +# +reset_db + +do_execsql_test 10.0 { + CREATE TABLE t1(a, b TEXT COLLATE NOCASE); + INSERT INTO t1 VALUES('abc', 'def'); + INSERT INTO t1 VALUES('ghi', 'jkl'); +} + +do_execsql_test 10.1 { + SELECT rowid FROM t1 WHERE (a, b) IN ( VALUES('abc', 'def'), ('ghi', 'JKL') ); +} {1 2} + +do_execsql_test 10.2 { + CREATE INDEX i1 ON t1(a, b COLLATE BINARY); +} + +do_execsql_test 10.3 { + SELECT rowid FROM t1 WHERE (a, b) IN ( VALUES('abc', 'def'), ('ghi', 'JKL') ); +} {1 2} + finish_test From 0309fb919aef74ee2df3b87871c43da025cefeb4 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 23 Apr 2024 11:56:20 +0000 Subject: [PATCH 3/3] Fix an uninitialized variable problem in the new code on this branch. FossilOrigin-Name: 85625b38718c6e4cf7aa97fa3c52153bd25c810b68f19e7924e1f4b709c6a05d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index d652ae551d..1581ca907f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Slight\sperformance\simprovement\sfor\sthe\snew\scode\son\sthis\sbranch. -D 2024-04-20T19:19:30.713 +C Fix\san\suninitialized\svariable\sproblem\sin\sthe\snew\scode\son\sthis\sbranch. +D 2024-04-23T11:56:20.612 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -835,7 +835,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c 79c673bf23b3b7fcc5e9cfa0adcd88a97645cd47920d2365dd6aec8cab7b222a +F src/where.c 09dc0b63e1c18c380179f5ca15dd25ec4b796fc142bfe202b6a1a4e6c4fa8100 F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 F src/wherecode.c 4b5909be0c36030eec18ea0e7a64c60254180a21e626c9d2e0e9be3007ad47c1 F src/whereexpr.c 7b64295f1d82ad0928df435925dd7bbd5997b44a026153113eace0d9e71ff435 @@ -2184,8 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4d870fd8b5450047a7486fc023d1ac9439642e8ed91eadfd5026c4cda7cc9179 -R 6023edc800b34da7492bfeaa8c433f44 +P 500c67f1341fe2a7e7333d525c90df201cc73a683b943ad5c1e41d4a4f639043 +R 842a99b3ce378b1a997aef4384eecd35 U dan -Z caab3df5b31f1752021c68136ff6d06a +Z 6e531b1f6828a7a325bb114af8c59a31 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6bb6214d88..03f943f25c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -500c67f1341fe2a7e7333d525c90df201cc73a683b943ad5c1e41d4a4f639043 \ No newline at end of file +85625b38718c6e4cf7aa97fa3c52153bd25c810b68f19e7924e1f4b709c6a05d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 96a6df9deb..38cf6b7287 100644 --- a/src/where.c +++ b/src/where.c @@ -323,6 +323,7 @@ static SQLITE_NOINLINE const char *indexInAffinityOk( if( sqlite3ExprIsVector(pX->pLeft) ){ int iField = pTerm->u.x.iField - 1; + inexpr.flags = 0; inexpr.op = TK_EQ; inexpr.pLeft = pX->pLeft->x.pList->a[iField].pExpr; assert( ExprUseXSelect(pX) );