Generalize the indexCellCompare() so that works on any index page, not just

the current page that a cursor is pointing to.

FossilOrigin-Name: eda518028f60efc7f879b8c10c900b4b1d4861b62f69a076051e98e61e5b1fee

ext/expert/expert1.test

@@ -77,6 +77,7 @@ foreach {tn setup} {
       $expert destroy
 
       set tst [subst -nocommands {set {} [squish [join {$result}]]}]
+      set tst [regsub {, OR SCAN [ a-z0-9]+} $tst {}]
       uplevel [list do_test $tn $tst [string trim [squish $res]]]
     }
   }
@@ -90,7 +91,7 @@ foreach {tn setup} {
     proc do_rec_test {tn sql res} {
       set res [squish [string trim $res]]
       set tst [subst -nocommands { 
-        squish [string trim [exec $::CLI test.db ".expert" {$sql;}]]
+        regsub {, OR SCAN [ a-z0-9]+} [squish [string trim [exec $::CLI test.db ".expert" {$sql;}]]] {}
       }]
       uplevel [list do_test $tn $tst $res]
     }

manifest

@@ -1,5 +1,5 @@
-C Have\ssqlite3_setlk_timeout()\stake\sthe\sdatabase\shandle\smutex.\sThis\sfixes\san\sassert()\sfailure\sthat\scould\soccur\sif\ssqlite3_setlk_timeout()\swere\scalled\son\sa\sthreadsafe\shandle.
-D 2025-06-12T07:35:38.259
+C Generalize\sthe\sindexCellCompare()\sso\sthat\sworks\son\sany\sindex\spage,\snot\sjust\nthe\scurrent\spage\sthat\sa\scursor\sis\spointing\sto.
+D 2025-06-14T18:02:12.057
 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -71,7 +71,7 @@ F doc/wal-lock.md 7db0cd61e2000b545b78ce89b0c2a9a8dd8d64c097839258ac10d7c5c4156e
 F ext/README.md 6eb1ac267d917767952ed0ef63f55de003b6a5da433ce1fa389e1a9532e73132
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4
-F ext/expert/expert1.test 1d2da6606623b57bb47064e02140823ce1daecd4cacbf402c73ad3473d7f000c
+F ext/expert/expert1.test c700f0fac91571191a97f4c027d5f10f3bff1aa2ddbccf320a1b14bdbb428c45
 F ext/expert/sqlite3expert.c cf4b1e5584862f486a4c6014ddb081831f1c512065dcf35644638d57179979d6
 F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaedabd158ea8969b
 F ext/expert/test_expert.c c395134bd6d4efa594a7d26578a1cb624c4027b79b4b5fcd44736c5ef1f5f725
@@ -726,8 +726,8 @@ F src/auth.c 54ab9c6c5803b47c0d45b76ce27eff22a03b4b1f767c5945a3a4eb13aa4c78dc
 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523
 F src/bitvec.c e242d4496774dfc88fa278177dd23b607dce369ccafb3f61b41638eea2c9b399
 F src/btmutex.c 30dada73a819a1ef5b7583786370dce1842e12e1ad941e4d05ac29695528daea
-F src/btree.c 53a9c7b243e94c992853d90f1dac0959028433b4b0d27e04409cee04e001b508
-F src/btree.h 18e5e7b2124c23426a283523e5f31a4bff029131b795bb82391f9d2f3136fc50
+F src/btree.c 062be9340c7a61e51130c1cf84821bdde19d0c6c03d9fd8630f15ce0f53d5685
+F src/btree.h 70f5f4a2c4767fcada25a94c8ac1fbada4b416e3b18be48a4a090c3a5314505c
 F src/btreeInt.h 9c0f9ea5c9b5f4dcaea18111d43efe95f2ac276cd86d770dce10fd99ccc93886
 F src/build.c 67c1db4c5e89a8519fe9b6dafc287f6bc3627696b5b8536dc5e06db570d8c05f
 F src/callback.c acae8c8dddda41ee85cfdf19b926eefe830f371069f8aadca3aa39adf5b1c859
@@ -736,10 +736,10 @@ F src/date.c 9db4d604e699a73e10b8e85a44db074a1f04c0591a77e2abfd77703f50dce1e9
 F src/dbpage.c b3e218f8ed74fcbb7fa805df8ca669a3718d397617b3d8a8aac3307dc315c4d6
 F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c
 F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42
-F src/expr.c f16fa5cbd849991462edf1d31bb7def5b970bb9611afcb4ea21c77e88e52a220
+F src/expr.c 083f478563cfa0163a8b7a2f5c8d74a45197e02f95cf3d61f0cde67578aa8996
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f
-F src/func.c de47a8295503aa130baae5e6d9868ecf4f7c4dbffa65d83ad1f70bdbac0ee2d6
+F src/func.c adbb40155039b23cf6238a023eb1896920c65ed3a3535669f6a73c6914ac597d
 F src/global.c a19e4b1ca1335f560e9560e590fc13081e21f670643367f99cb9e8f9dc7d615b
 F src/hash.c 73934a7f7ab1cb110614a9388cb516893b0cf5b7b69e4fd1a0780ac4ce166be7
 F src/hash.h 46b92795a95bfefb210f52f0c316e9d7cdbcdd7e7fcfb0d8be796d3a5767cddf
@@ -786,11 +786,11 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c d40fe18d7c2fd0339f5846ffcf7d6809866e380acdf14c76fb2af87e9fe13f64
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
 F src/select.c 882d739e0d5e6c7a8b46a3cca3ada37fe1a56301f1360d6b141312c666bbe482
-F src/shell.c.in 4ae9ff8e8b23d9a309e50d5b5ef2768b9bb18f748ece2acc2641a3c23b71e2b9
+F src/shell.c.in 51341dda27332052ed6c777994933f0d11ba53e550ac3cfd8768344621765722
 F src/sqlite.h.in 22882ddd3a70751aa8864c81993ee4562ed54c2c508b6270f75e223ffee38e1b
 F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479
 F src/sqlite3ext.h 0bfd049bb2088cc44c2ad54f2079d1c6e43091a4e1ce8868779b75f6c1484f1e
-F src/sqliteInt.h 5f190640c1939960f993c07d0e626fb2acc40d42ac12b21811dd6c069fb71db4
+F src/sqliteInt.h c6e4eb12ecde8bb7b0efdfd30223d8d3bd716f6854d8f5c021f3b921e9c0ba14
 F src/sqliteLimit.h 6d817c28a8f19af95e6f4921933b7fbbca48a962bce0eb0ec81e8bb3ef38e68b
 F src/status.c 0e72e4f6be6ccfde2488eb63210297e75f569f3ce9920f6c3d77590ec6ce5ffd
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -852,7 +852,7 @@ F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1
 F src/utf.c 7267c3fb9e2467020507601af3354c2446c61f444387e094c779dccd5ca62165
 F src/util.c 36fb1150062957280777655976f3f9a75db236cb8207a0770ceae8d5ec17fcd3
 F src/vacuum.c 1bacdd0a81d2b5dc1c508fbf0d938c89fa78dd8d5b46ec92686d44030d4f4789
-F src/vdbe.c 714fab7aa7c516edbcf5e4f653ae8f548a3e24c0ed19086d7383bb5851983992
+F src/vdbe.c d826706a03b6b72514abae2c39d7eafa02d3272c65c1220e03d24c465979fdab
 F src/vdbe.h 93761ed7c6b8bc19524912fd9b9b587d41bf4f1d0ade650a00dadc10518d8958
 F src/vdbeInt.h 0bc581a9763be385e3af715e8c0a503ba8422c2b7074922faf4bb0d6ae31b15e
 F src/vdbeapi.c 613a6f29efacd6ed83e886b6e52db0fe52ba80a596b0a137608db1948bad90a9
@@ -867,9 +867,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c b0f848cfba8dd057f77073493cdd542f9125b4cf87941f53e9d0db21604155c8
 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452
 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014
-F src/where.c 45a3b496248a0b36d91ce34da3278d54f8fa20e9d3fbd36d45a42051d1118137
-F src/whereInt.h ecdbfb5551cf394f04ec7f0bc7ad963146d80eee3071405ac29aa84950128b8e
-F src/wherecode.c 9710e62379c000189476404f923d4d1b192d0def222fdd287b820cc085a0d555
+F src/where.c 2a7ac07e5c288d15898f7910b810d556731818e32810778369261f8f94dd5722
+F src/whereInt.h b0c5487551dc7821426cac213fd5c194a558cab9dfd4e130d71f7afa6416463c
+F src/wherecode.c 45adf7743c305a89647f8278a71f3eba27e1d1f002c99cc3685254d7ae9e958f
 F src/whereexpr.c 2415c8eee5ff89a8b709d7d83d71c1ff986cd720d0520057e1d8a5371339012a
 F src/window.c d01227141f622f24fbe36ca105fbe6ef023f9fd98f1ccd65da95f88886565db5
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -932,7 +932,7 @@ F test/autoanalyze1.test b9cc3f32a990fa56669b668d237c6d53e983554ae80c0604992e188
 F test/autoinc.test 9df9930966dbe92c55ef37a4d89112cfd537be0d0596d397177c12db9e581be0
 F test/autoindex1.test 65931519206bbec71948b11e125af0656435a0937973fe5fed70d776a712911f
 F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df
-F test/autoindex3.test ca502c8050166ac6107a7b4fe4e951f4d3270a23a958af02b14f1b962b83c4b6
+F test/autoindex3.test 0d78b07de5519f0d415f24481b7de998db997b5054036608a09ca241bb078b15
 F test/autoindex4.test 3c2105e9172920e26f950ba3c5823e4972190e022c1e6f260ba476b0af24c593
 F test/autoindex5.test 3fb938cbf4e7f3896563ce04e2a24b0bc653fc6245b4bf3268cd7b20f441d87f
 F test/autovacuum.test 00671369bbf96c6a49989a9425f5b78b94075d6a4b031e5e00000c2c32f365df
@@ -1051,7 +1051,7 @@ F test/corruptK.test ac13504593d89d69690d45479547616ed12644d42b5cb7eeb2e759a76fc
 F test/corruptL.test 652fc8ac0763a6fd3eb28b951d481924167b2d9936083bcc68253b2274a0c8fe
 F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c3e067
 F test/corruptN.test a034bb217bebd8d007625dfb078e76ec3d53515052dbceb68bd47b2c27674d5c
-F test/cost.test cc434a026b1e9d0d98137a147e24e5daf1b1ad09e9ff7da63b34c83ddd136d92
+F test/cost.test 10a7020c9789be27f5cfac90c2c36a39c0282276e1461b824ba1f8eccdb6e01c
 F test/count.test cd4bd531066e8d77ef8fe1e3fc8253d042072e117ccab214b290cf83f1602249
 F test/countofview.test 4088e461a10ee33e69803c177a69aa1d7bba81a9ffc2df66d76465a22ca7fdfc
 F test/coveridxscan.test f35c7208dedc4f98e471c569df64c0f95a49f6e072d8dc7c8f99bdee2697de1b
@@ -1405,7 +1405,7 @@ F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200
 F test/lemon-test01.y 70110eff607ab137ccc851edb2bc7e14a6d4f246b5d2d25f82a60b69d87a9ff2
 F test/like.test 0036f8fe548fceabd1496361bfa262f35cf5ed17c794bd15e22e9f3de12e0eb0
 F test/like2.test d3be15fefee3e02fc88942a9b98f26c5339bbdef7783c90023c092c4955fe3d3
-F test/like3.test 3c9be7a0122908d8ead6aa25e47c2b9787eea88e8062926078ea7b3e95c71d99
+F test/like3.test 3eb98719a3b12e1f712915e712cd9214ac8e88c91ddb711f4905cb0af6d96625
 F test/limit.test 350f5d03c29e7dff9a2cde016f84f8d368d40bcd02fa2b2a52fa10c4bf3cbfaf
 F test/limit2.test 621188fc3e5c3b8d2ef9827e05fa8313792ae563579073136efd25cb65325f1b
 F test/literal.test a65dca9fef86e51b8e45544268e37abbd4bb94ba35fd65f6fdcab2f288cd8f79
@@ -1694,7 +1694,7 @@ F test/spellfix4.test 51c7c26514ade169855c66bcf130bd5acfb4d7fd090cc624645ab275ae
 F test/sqldiff1.test 1b7ab4f312442c5cc6b3a5f299fa8ca051416d1dd173cb1126fd51bf64f2c3fb
 F test/sqllimits1.test 408131e4975d61868711c83f101a56d4602313cc5cae88d3eee81c1da364fd89
 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a
-F test/starschema1.test f5388cd32527ab18d3f98f9e3402ec780f6a186e04e0d9c8531d7568ee734e11
+F test/starschema1.test d3b22a6eef7a61d24e5f7beb531aa54554355fe72566df7cbe729be179ea13c4
 F test/startup.c 1beb5ca66fcc0fce95c3444db9d1674f90fc605499a574ae2434dcfc10d22805
 F test/stat.test 123212a20ceb496893d5254a5f6c76442ce549fdc08d1702d8288a2bbaac8408
 F test/statfault.test 064f43379e4992b5221b7d9ac887c313b3191f85cce605d78e416fc4045da64e
@@ -1895,7 +1895,7 @@ F test/tkt3992.test f3e7d548ac26f763b47bc0f750da3d03c81071da
 F test/tkt3997.test a335fa41ca3985660a139df7b734a26ef53284bd
 F test/tkt4018.test 18dbc6617f7a4b90e938d1bd6d26ad18daafaf08
 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7
-F test/tpch01.test 4479008f85f6f8f25f7ab2cb305d665752b4727fa28a8df3d8e0ad46520c62ff
+F test/tpch01.test c4a0ed2404412b61a99bbe6695761e540b5fbc0b9cc7a0702c2f873a87af6492
 F test/trace.test a659a9862957f4789e37a92b3bf6d2caf5c86b02cdeefc41e850ae53acf6992a
 F test/trace2.test f5cb67ad3bc09e0c58e8cca78dfd0b5639259983
 F test/trace3.test 2deeac66359c9f007f0fc9fb6336994a5d68fc1a65129f322a9e9546fd537d0a
@@ -2063,12 +2063,12 @@ F test/whereJ.test fc05e374cc9f2dc204148d6c06822c380ad388895fe97a6d335b94a26a08a
 F test/whereK.test 0270ab7f04ba5436fb9156d31d642a1c82727f4c4bfe5ba90d435c78cf44684a
 F test/whereL.test cb115604cc9bd61acbc99a1f1df0eb1ea7a7875a77fef25ba9282f01d10283e1
 F test/whereM.test 0dbc9998783458ddcf3cc078ca7c2951d8b2677d472ecf0028f449ed327c0250
-F test/whereN.test 63a3584b71acfb6963416de82f26c6b1644abc5ca6080c76546b9246734c8803
+F test/whereN.test c611fb6ded3e8f23604045bb11e160e0854d3e71d2f66505e77f63ab50b54139
 F test/wherefault.test 6cf2a9c5712952d463d3f45ebee7f6caf400984df51a195d884cfb7eb0e837a7
 F test/wherelfault.test 9012e4ef5259058b771606616bd007af5d154e64cc25fa9fd4170f6411db44e3
 F test/wherelimit.test afb46397c6d7e964e6e294ba3569864a0c570fe3807afc634236c2b752372f31
 F test/wherelimit2.test b9e4bfe7b4d7c2f85f99cf2bd2c51369378d04b1f3d1b60557423752003bfd90
-F test/wherelimit3.test 22d73e046870cf8bbe15573eda6b432b07ebe64a88711f9f849c6b3667c1fae6
+F test/wherelimit3.test d8830bd51149b20922457ada26c13ce2dd9b0d1f5f76369d658bbd52da786b2c
 F test/widetab1.test c296a98e123762de79917350e45fa33fdf88577a2571eb3a64c8bf7e44ef74d1
 F test/win32heap.test 1ec2ce646aee491ec23bfcdfd005b33c79f13bf91467966f374a76ffe7c7e85f
 F test/win32lock.test e56d7a9b6cf9d5f3867c2dd19ff36c5326881e4038c6867610ecb3a9868ea4eb
@@ -2099,7 +2099,7 @@ F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c
 F test/windowpushd.test c420e2265f0e09a0e798d0513a660d71b51602088d81b3dbd038918ee1339dcc
 F test/with1.test 1ee171d7c306ab8b0771f3511d870f56c735607729836585bbceb1fc2f47e0b1
 F test/with2.test 181674a6cc86a601ca2ac052741cdfad5b529e07e870435d2f6cdb92d589ff17
-F test/with3.test e30369ea27aa27eb1bda4c5e510c8a9f782c8afd2ab99d1a02b8a7f25a5d3e65
+F test/with3.test 4fca9df31cff0b09844aa06274a46cc4d4adaa18477ad108e85422fafc5d0e6f
 F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205
 F test/with5.test 6248213c41fab36290b5b73aa3f937309dfba337004d9d8434c3fabc8c7d4be8
 F test/with6.test 281e4861b5e517f6c3c2f08517a520c1e2ee7c11966545d3901f258a4fe8ef76
@@ -2208,8 +2208,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh 1ad0169b022b280bcaaf94a7fa231591be96b514230ab5c98fbf15cd7df842dd
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P badf3014bd1620fd3d4b8013f641fd820b249649fb93cc75b7b8df9dfd6f32a6
-R 7fdf393495e0e2303824909fb18b2ec1
-U dan
-Z 94b5809cfbb11f003ae05c3735568843
+P 711608e49b0ec2ed885f8b3d0d770ec0590841fc4aaf2c331e4627c64ad7d069
+R 0f91953f10cf45f8030c38e629ad41c9
+U drh
+Z f4b14aa877610f8cc0135cf0804977a5
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-a95d126e1330e1b83f42b51f97c4c216622cf38062f3b5d72ccb76313187e850
+eda518028f60efc7f879b8c10c900b4b1d4861b62f69a076051e98e61e5b1fee

src/btree.c

@@ -5882,8 +5882,8 @@ moveto_table_finish:
 }
 
 /*
-** Compare the "idx"-th cell on the page the cursor pCur is currently
-** pointing to to pIdxKey using xRecordCompare.  Return negative or
+** Compare the "idx"-th cell on the page pPage against the key
+** pointing to by pIdxKey using xRecordCompare.  Return negative or
 ** zero if the cell is less than or equal pIdxKey.  Return positive
 ** if unknown.
 **
@@ -5898,12 +5898,11 @@ moveto_table_finish:
 ** a positive value as that will cause the optimization to be skipped.
 */
 static int indexCellCompare(
-  BtCursor *pCur,
+  MemPage *pPage,
   int idx,
   UnpackedRecord *pIdxKey,
   RecordCompare xRecordCompare
 ){
-  MemPage *pPage = pCur->pPage;
   int c;
   int nCell;  /* Size of the pCell cell in bytes */
   u8 *pCell = findCellPastPtr(pPage, idx);
@@ -6012,14 +6011,14 @@ int sqlite3BtreeIndexMoveto(
   ){
     int c;
     if( pCur->ix==pCur->pPage->nCell-1
-     && (c = indexCellCompare(pCur, pCur->ix, pIdxKey, xRecordCompare))<=0
+     && (c = indexCellCompare(pCur->pPage,pCur->ix,pIdxKey,xRecordCompare))<=0
      && pIdxKey->errCode==SQLITE_OK
     ){
       *pRes = c;
       return SQLITE_OK;  /* Cursor already pointing at the correct spot */
     }
     if( pCur->iPage>0
-     && indexCellCompare(pCur, 0, pIdxKey, xRecordCompare)<=0
+     && indexCellCompare(pCur->pPage, 0, pIdxKey, xRecordCompare)<=0
      && pIdxKey->errCode==SQLITE_OK
     ){
       pCur->curFlags &= ~(BTCF_ValidOvfl|BTCF_AtLast);
@@ -6236,11 +6235,41 @@ i64 sqlite3BtreeRowCountEst(BtCursor *pCur){
 
   n = pCur->pPage->nCell;
   for(i=0; i<pCur->iPage; i++){
-    n *= pCur->apPage[i]->nCell;
+    n *= pCur->apPage[i]->nCell+1;
   }
   return n;
 }
 
+/*
+** Return an estimate for the number of rows in the table that pCur is
+** pointing to that are to the left of, or that come before, or that are
+** less than pCur.  Return a negative number if no estimate is currently
+** available.
+*/
+int sqlite3BtreeEstimatedPosition(BtCursor *pCur){
+  i64 nAll;    /* Estimated total number of rows */
+  i64 nLeft;   /* Estimated number of rows to the left */
+  int i;       /* Btree stack counter */
+
+  assert( cursorOwnsBtShared(pCur) );
+  assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) );
+
+  if( pCur->eState!=CURSOR_VALID ) return -1;
+
+  nAll = pCur->pPage->nCell;
+  nLeft = pCur->ix;
+  for(i=pCur->iPage-1; i>=0; i--){
+    nLeft = nLeft + nAll*pCur->aiIdx[i];
+    nAll *= pCur->apPage[i]->nCell+1;
+  }
+  if( NEVER(nAll==0) ) return 0;
+  if( nAll<10000000 ){
+    return (nLeft*100)/nAll;
+  }else{
+    return nLeft/(nAll/100);
+  }
+}
+
 /*
 ** Advance the cursor to the next entry in the database.
 ** Return value:

src/btree.h

@@ -342,6 +342,7 @@ int sqlite3BtreeIntegrityCheck(
 );
 struct Pager *sqlite3BtreePager(Btree*);
 i64 sqlite3BtreeRowCountEst(BtCursor*);
+int sqlite3BtreeEstimatedPosition(BtCursor*);
 
 #ifndef SQLITE_OMIT_INCRBLOB
 int sqlite3BtreePayloadChecked(BtCursor*, u32 offset, u32 amt, void*);

src/expr.c

@@ -4545,6 +4545,23 @@ static int exprCodeInlineFunction(
               (aff<=SQLITE_AFF_NONE) ? "none" : azAff[aff-SQLITE_AFF_BLOB]);
       break;
     }
+
+    case INLINEFUNC_est_rank: {
+      /* The EST_RANK(x) function returns an integer between 0 and 100 to
+      ** indicate how far along in the table or index currently being used
+      ** to access column x is the cursor used to do that access.  If the
+      ** argument is not a simple table column reference, the value is NULL.
+      */
+      Expr *pExpr = pFarg->a[0].pExpr;
+      Vdbe *v = pParse->pVdbe;
+      if( pExpr->op==TK_COLUMN ){
+        sqlite3ExprCodeTarget(pParse, pExpr, target);
+        sqlite3VdbeAddOp2(v, OP_EstPos, 0, target);
+      }else{
+        sqlite3VdbeAddOp2(v, OP_Null, 0, target);
+      }
+      break;
+    }
 #endif /* !defined(SQLITE_UNTESTABLE) */
   }
   return target;

src/func.c

@@ -2917,6 +2917,7 @@ void sqlite3RegisterBuiltinFunctions(void){
     INLINE_FUNC(coalesce,       -4, INLINEFUNC_coalesce, 0 ),
     INLINE_FUNC(iif,            -4, INLINEFUNC_iif,      0 ),
     INLINE_FUNC(if,             -4, INLINEFUNC_iif,      0 ),
+    INLINE_FUNC(est_rank,        1, INLINEFUNC_est_rank, 0 ),
   };
 #ifndef SQLITE_OMIT_ALTERTABLE
   sqlite3AlterFunctions();

src/shell.c.in

@@ -11709,6 +11709,7 @@ static int do_meta_command(char *zLine, ShellState *p){
             { 0x08000000, 1, "OnePass" },
             { 0x10000000, 1, "OrderBySubq" },
             { 0x20000000, 1, "StarQuery" },
+            { 0x40000000, 1, "FlexSearch" },
             { 0xffffffff, 0, "All" },
           };
           unsigned int curOpt;

src/sqliteInt.h

@@ -1924,6 +1924,7 @@ struct sqlite3 {
 #define SQLITE_OnePass        0x08000000 /* Single-pass DELETE and UPDATE */
 #define SQLITE_OrderBySubq    0x10000000 /* ORDER BY in subquery helps outer */
 #define SQLITE_StarQuery      0x20000000 /* Heurists for star queries */
+#define SQLITE_FlexSearch     0x40000000 /* Maybe scan instead of use index */
 #define SQLITE_AllOpts        0xffffffff /* All optimizations */
 
 /*
@@ -2053,6 +2054,7 @@ struct FuncDestructor {
 #define INLINEFUNC_affinity             4
 #define INLINEFUNC_iif                  5
 #define INLINEFUNC_sqlite_offset        6
+#define INLINEFUNC_est_rank             7
 #define INLINEFUNC_unlikely            99  /* Default case */
 
 /*

src/vdbe.c

@@ -3757,6 +3757,68 @@ case OP_Count: {         /* out2 */
   goto check_for_interrupt;
 }
 
+/* Opcode: EstPos * P2 * * *
+**
+** If the immediately preceding opcode is an OP_Column using
+** cursor C, then write into r[P2] a number between 0 and 100
+** (inclusive) that is an estimate of the percentage of the
+** table or index associated with C that comes before the entry
+** that C is currently pointing to.
+**
+** If the previous opcode is not an OP_Column then r[P2] is set
+** to NULL.
+**
+** The opcode mnemonic is "ESTimated POSition".
+*/
+case OP_EstPos: {        /* out2 */
+  int iRes;
+  VdbeCursor *pCur;
+  BtCursor *pCrsr;
+
+  pOut = out2Prerelease(p, pOp);
+  pOut->flags = MEM_Null;
+  if( pOp[-1].opcode==OP_Column ){
+    pCur = p->apCsr[pOp[-1].p1];
+    if( pCur->eCurType==CURTYPE_BTREE ){
+      pCrsr = p->apCsr[pOp[-1].p1]->uc.pCursor;
+      iRes = sqlite3BtreeEstimatedPosition(pCrsr);
+      if( iRes>=0 ){
+        pOut->flags = MEM_Int;
+        pOut->u.i = iRes;
+      }
+    }
+  }
+  break;
+}
+
+/* Opcode: IfUseIndex P1 P2 P3 P4 *
+**
+** This opcode decides whether it will be faster to use the index
+** in cursor P1 or to do a full table scan on the original table.
+** If the index on cursor P1 should be used, a jump is made to P2.
+** (That is the usual case).  If it seems like it might be faster to
+** do a full table scan, then fall through.
+**
+** The P3 and P4 arguments define a key in the same format as OP_SeekGT
+** and similar.  That key is the last key in the range of index entries
+** that match the join condition.  This opcode works by estimating the
+** number of rows in the index P1 between the current row and the last
+** row that matches P3,P4.  If the number of rows that match is a significant
+** fraction of the total number of rows in the index, then it might be
+** faster to do a scan of the original table, and so this opcode falls.
+** If the estimated number of matching rows is small compared to the
+** total number of rows in the index, then the jump to P2 is taken
+** and cursor P1 is unchanged.
+*/
+case OP_IfUseIndex: {    /* jump */
+  /**** Temporary Hack:
+  ***** Take the jump if Tuning(1) is 0.  Fall through if non-zero.
+  ****/
+  if( Tuning(1)==0 ) goto jump_to_p2;
+  break;
+}
+
+
 /* Opcode: Savepoint P1 * * P4 *
 **
 ** Open, release or rollback the savepoint named by parameter P4, depending
@@ -4865,15 +4927,19 @@ case OP_SeekGT: {       /* jump0, in3, group, ncycle */
     ** with the same key.
     */
     if( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ) ){
-      eqOnly = 1;
+#ifdef SQLITE_DEBUG
+      VdbeOp *pNext = &pOp[1];
+      if( pNext->opcode==OP_IfUseIndex ) pNext = &p->aOp[pNext->p2];
       assert( pOp->opcode==OP_SeekGE || pOp->opcode==OP_SeekLE );
-      assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT );
-      assert( pOp->opcode==OP_SeekGE || pOp[1].opcode==OP_IdxLT );
-      assert( pOp->opcode==OP_SeekLE || pOp[1].opcode==OP_IdxGT );
-      assert( pOp[1].p1==pOp[0].p1 );
-      assert( pOp[1].p2==pOp[0].p2 );
-      assert( pOp[1].p3==pOp[0].p3 );
-      assert( pOp[1].p4.i==pOp[0].p4.i );
+      assert( pNext->opcode==OP_IdxLT || pNext->opcode==OP_IdxGT );
+      assert( pOp->opcode==OP_SeekGE || pNext->opcode==OP_IdxLT );
+      assert( pOp->opcode==OP_SeekLE || pNext->opcode==OP_IdxGT );
+      assert( pNext->p1==pOp[0].p1 );
+      assert( pNext->p2==pOp[0].p2 );
+      assert( pNext->p3==pOp[0].p3 );
+      assert( pNext->p4.i==pOp[0].p4.i );
+#endif /* SQLITE_DEBUG */
+      eqOnly = 1;
     }
 
     nField = pOp->p4.i;
@@ -4958,7 +5024,7 @@ seek_not_found:
   VdbeBranchTaken(res!=0,2);
   if( res ){
     goto jump_to_p2;
-  }else if( eqOnly ){
+  }else if( eqOnly && pOp[1].opcode!=OP_IfUseIndex ){
     assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT );
     pOp++; /* Skip the OP_IdxLt or OP_IdxGT that follows */
   }

src/where.c

@@ -7158,6 +7158,14 @@ WhereInfo *sqlite3WhereBegin(
           wherePartIdxExpr(
               pParse, pIx, pIx->pPartIdxWhere, 0, iIndexCur, pTabItem
           );
+        }else if( pIx->aiRowLogEst[pLoop->u.btree.nEq]>65
+               && 0==(pLoop->wsFlags & (WHERE_AUTO_INDEX
+                                       |WHERE_IDX_ONLY
+                                       |WHERE_EXPRIDX))
+               && 0!=(pLoop->wsFlags & (WHERE_BOTH_LIMIT|WHERE_CONSTRAINT))
+               && OptimizationEnabled(db, SQLITE_FlexSearch)
+        ){
+          pLoop->wsFlags |= WHERE_FLEX_SEARCH;
         }
       }
       pLevel->iIdxCur = iIndexCur;
@@ -7549,7 +7557,12 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
     if( pIdx
      && !db->mallocFailed
     ){
-      if( pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable) ){
+      if( pLoop->wsFlags & WHERE_FLEX_SEARCH ){
+        last = pLevel->addrBody + 1;
+        /* ^---- Setting last to the start of the loop body disables
+        ** table-to-index translation, while preserving the pIdx->bHasExpr
+        ** adjustments */
+      }else if( pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable) ){
         last = iEnd;
       }else{
         last = pWInfo->iEndWhere;

src/whereInt.h

@@ -660,5 +660,6 @@ void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*);
 #define WHERE_COROUTINE    0x02000000  /* Implemented by co-routine.
                                        ** NB: False-negatives are possible */
 #define WHERE_EXPRIDX      0x04000000  /* Uses an index-on-expressions */
+#define WHERE_FLEX_SEARCH  0x08000000  /* Maybe decide to SCAN at runtime */
 
 #endif /* !defined(SQLITE_WHEREINT_H) */

src/wherecode.c

@@ -175,6 +175,9 @@ void sqlite3WhereAddExplainText(
         sqlite3_str_appendf(&str, zFmt, pIdx->zName);
         explainIndexRange(&str, pLoop);
       }
+      if( (flags & WHERE_FLEX_SEARCH)!=0 && isSearch ){
+        sqlite3_str_appendf(&str, ", OR SCAN %S", pItem);
+      }
     }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){
       char cRangeOp;
 #if 0  /* Better output, but breaks many tests */
@@ -1898,6 +1901,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     int omitTable;               /* True if we use the index only */
     int regBignull = 0;          /* big-null flag register */
     int addrSeekScan = 0;        /* Opcode of the OP_SeekScan, if any */
+    int addrGosub = 0;           /* Address of first OP_Gosub */
 
     pIdx = pLoop->u.btree.pIndex;
     iIdxCur = pLevel->iIdxCur;
@@ -2139,6 +2143,25 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     if( zStartAff ) sqlite3DbNNFreeNN(db, zStartAff);
     if( zEndAff ) sqlite3DbNNFreeNN(db, zEndAff);
 
+    if( pLoop->wsFlags & WHERE_FLEX_SEARCH ){
+      int jIfIdx, ii;
+      int labelScanCont = sqlite3VdbeMakeLabel(pParse);
+      jIfIdx = sqlite3VdbeAddOp4Int(v, OP_IfUseIndex, iIdxCur, 0,
+                                    regBase, nConstraint);
+      sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur);
+      for(ii=0; ii<pLoop->nLTerm; ii++){
+        if( pLoop->aLTerm[ii] ){
+          sqlite3ExprIfFalse(pParse, pLoop->aLTerm[ii]->pExpr,
+                             labelScanCont, SQLITE_JUMPIFNULL);
+        }
+      }
+      addrGosub = sqlite3VdbeAddOp1(v, OP_Gosub, ++pParse->nMem);
+      sqlite3VdbeResolveLabel(v, labelScanCont);
+      sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, jIfIdx+2);
+      sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBrk);
+      sqlite3VdbeJumpHere(v, jIfIdx);
+    }
+
     /* Top of the loop body */
     pLevel->p2 = sqlite3VdbeCurrentAddr(v);
 
@@ -2236,6 +2259,27 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       assert( pLevel->p5==0 );
     }
     if( omitTable ) pIdx = 0;
+
+    if( addrGosub ){
+      /* This is a flex scan.
+      ** Generate a subroutine call to the both of this loop then terminate
+      ** this loop immediately.   Arrange for an OP_Return to occur after
+      ** the loop body.
+      */
+      int regRtnAddr = sqlite3VdbeGetOp(v, addrGosub)->p1;
+      int addrSub2 = sqlite3VdbeAddOp1(v, OP_Gosub, regRtnAddr);
+      sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3);
+      sqlite3VdbeChangeP5(v, pLevel->p5);
+      pLevel->op = OP_Return;
+      pLevel->p1 = regRtnAddr;
+      pLevel->p3 = 0;
+      pLevel->p5 = 0;
+      sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBrk);
+      sqlite3VdbeJumpHere(v, addrGosub);
+      sqlite3VdbeJumpHere(v, addrSub2);
+      pLevel->p2 = sqlite3VdbeCurrentAddr(v);
+    }
+
   }else
 
 #ifndef SQLITE_OMIT_OR_OPTIMIZATION

test/autoindex3.test

@@ -85,7 +85,7 @@ do_eqp_test 220 {
   select count(*) from u, v where u.b = v.b and v.e > 34;
 } {
   QUERY PLAN
-  |--SEARCH v USING INDEX ve (e>?)
+  |--SEARCH v USING INDEX ve (e>?), OR SCAN v
   |--BLOOM FILTER ON u (b=?)
   `--SEARCH u USING AUTOMATIC COVERING INDEX (b=?)
 }
@@ -119,10 +119,10 @@ do_eqp_test 310 {
   QUERY PLAN
   |--CO-ROUTINE children
   |  |--SETUP
-  |  |  `--SEARCH t2 USING INDEX x1 (pid=? AND rx=?)
+  |  |  `--SEARCH t2 USING INDEX x1 (pid=? AND rx=?), OR SCAN t2
   |  `--RECURSIVE STEP
   |     |--SCAN children
-  |     `--SEARCH t2 USING INDEX x1 (pid=? AND rx=?)
+  |     `--SEARCH t2 USING INDEX x1 (pid=? AND rx=?), OR SCAN t2
   `--SCAN children
 }  
 

test/cost.test

@@ -110,7 +110,7 @@ do_eqp_test 5.3 {
   SELECT * FROM t2 WHERE x BETWEEN ? AND ? ORDER BY rowid;
 } {
   QUERY PLAN
-  |--SEARCH t2 USING INDEX t2i1 (x>? AND x<?)
+  |--SEARCH t2 USING INDEX t2i1 (x>? AND x<?), OR SCAN t2
   `--USE TEMP B-TREE FOR ORDER BY
 }
 

test/like3.test

@@ -252,26 +252,26 @@ do_eqp_test like3-6.210 {
   SELECT * FROM t2 WHERE path LIKE 'a%';
 } {
   QUERY PLAN
-  `--SEARCH t2 USING INDEX t2path (path>? AND path<?)
+  `--SEARCH t2 USING INDEX t2path (path>? AND path<?), OR SCAN t2
 }
 do_eqp_test like3-6.220 {
   SELECT * FROM t2 WHERE path LIKE 'a%' ESCAPE '\';
 } {
   QUERY PLAN
-  `--SEARCH t2 USING INDEX t2path (path>? AND path<?)
+  `--SEARCH t2 USING INDEX t2path (path>? AND path<?), OR SCAN t2
 }
 db eval {PRAGMA case_sensitive_like=ON}
 do_eqp_test like3-6.230 {
   SELECT * FROM t2 WHERE path LIKE 'a%';
 } {
   QUERY PLAN
-  `--SEARCH t2 USING INDEX t2path2 (path>? AND path<?)
+  `--SEARCH t2 USING INDEX t2path2 (path>? AND path<?), OR SCAN t2
 }
 do_eqp_test like3-6.240 {
   SELECT * FROM t2 WHERE path LIKE 'a%' ESCAPE '\';
 } {
   QUERY PLAN
-  `--SEARCH t2 USING INDEX t2path2 (path>? AND path<?)
+  `--SEARCH t2 USING INDEX t2path2 (path>? AND path<?), OR SCAN t2
 }
 }
 

test/starschema1.test

@@ -530,7 +530,7 @@ do_execsql_test 1.7 {
     FROM t1, x01, x02, x03, x04, x05, x06, x07, x08
    WHERE a01=b01 AND a02=b02 AND a03=b03 AND a04=b04 AND a05=b05
      AND a06=b06 AND a07=b07 AND a08=b08;
-} {~/SCAN.*SCAN/}
+} {~/SCAN.*\{SCAN/}
 do_execsql_test 1.8 {
   EXPLAIN QUERY PLAN
   SELECT c01, c02, c03, c04, c05, c06, c07, c08,
@@ -547,7 +547,7 @@ do_execsql_test 1.8 {
      AND a19=b19 AND a20=b20 AND a21=b21 AND a22=b22 AND a23=b23 AND a24=b24
      AND a25=b25 AND a26=b26 AND a27=b27 AND a28=b28 AND a29=b29 AND a30=b30
      AND a31=b31 AND a32=b32;
-} {~/SCAN.*SCAN/}
+} {~/SCAN.*\{SCAN/}
 do_execsql_test 1.9 {
   EXPLAIN QUERY PLAN
   SELECT c01, c02, c03, c04, c05, c06, c07, c08,
@@ -577,7 +577,7 @@ do_execsql_test 1.9 {
      AND a49=b49 AND a50=b50 AND a51=b51 AND a52=b52 AND a53=b53 AND a54=b54
      AND a55=b55 AND a56=b56 AND a57=b57 AND a58=b58 AND a59=b59 AND a60=b60
      AND a61=b61 AND a62=b62 AND a63=b63;
-} {~/SCAN.*SCAN/}
+} {~/SCAN.*\{SCAN/}
 
 
 

test/tpch01.test

@@ -189,7 +189,7 @@ order by
     revenue desc;
 } {
   QUERY PLAN
-  |--SEARCH orders USING INDEX odi (O_ORDERDATE>? AND O_ORDERDATE<?)
+  |--SEARCH orders USING INDEX odi (O_ORDERDATE>? AND O_ORDERDATE<?), OR SCAN orders
   |--SEARCH customer USING INDEX cpki (C_CUSTKEY=?)
   |--SEARCH nation USING INDEX npki (N_NATIONKEY=?)
   |--SEARCH lineitem USING INDEX lpki (L_ORDERKEY=?)

test/whereN.test

@@ -93,7 +93,7 @@ do_eqp_test 1.1 {
   |--CO-ROUTINE (subquery-xxxxxx)
   |  |--SEARCH DS USING COVERING INDEX ds1 (name=?)
   |  |--SEARCH R USING COVERING INDEX rule2 (dsid=?)
-  |  |--SEARCH V USING INDEX v1 (rid=?)
+  |  |--SEARCH V USING INDEX v1 (rid=?), OR SCAN V
   |  `--USE TEMP B-TREE FOR ORDER BY
   `--SCAN (subquery-xxxxxx)
 }

test/wherelimit3.test

@@ -29,7 +29,7 @@ do_eqp_test 1.1 {
   SELECT * FROM t1 WHERE a>=100 AND a<300 ORDER BY b LIMIT 5;
 } {
   QUERY PLAN
-  |--SEARCH t1 USING INDEX t1a (a>? AND a<?)
+  |--SEARCH t1 USING INDEX t1a (a>? AND a<?), OR SCAN t1
   `--USE TEMP B-TREE FOR ORDER BY
 }
 ifcapable stat4 {
@@ -46,7 +46,7 @@ do_eqp_test 1.3 {
   SELECT * FROM t1 WHERE a>=100 AND a<300 ORDER BY b LIMIT $::N;
 } {
   QUERY PLAN
-  |--SEARCH t1 USING INDEX t1a (a>? AND a<?)
+  |--SEARCH t1 USING INDEX t1a (a>? AND a<?), OR SCAN t1
   `--USE TEMP B-TREE FOR ORDER BY
 }
 

test/with3.test

@@ -95,7 +95,7 @@ ifcapable analyze {
     |  `--RECURSIVE STEP
     |     `--SCAN cnt
     |--SCAN cnt
-    `--SEARCH y1 USING INDEX y1a (a=?)
+    `--SEARCH y1 USING INDEX y1a (a=?), OR SCAN y1
   }]
 
   do_eqp_test 3.1.3 {