Fix a buffer overread found by OSSFuzz that could occur if a WITHOUT ROWID table with many columns was NATURAL JOINed against itself.

FossilOrigin-Name: 807643c596b2315feed9e9c492dcdba1dc35d6eb81253a72f0bca320fcaa4fca

manifest

@@ -1,5 +1,5 @@
-C Version\s3.33.0
-D 2020-08-14T13:23:32.827
+C Fix\sa\sbuffer\soverread\sfound\sby\sOSSFuzz\sthat\scould\soccur\sif\sa\sWITHOUT\sROWID\stable\swith\smany\scolumns\swas\sNATURAL\sJOINed\sagainst\sitself.
+D 2020-09-16T16:49:09.845
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -476,10 +476,10 @@ F src/auth.c a3d5bfdba83d25abed1013a8c7a5f204e2e29b0c25242a56bc02bb0c07bf1e06
 F src/backup.c b1c90cd4110248c8e1273ff4578d3a84c0c34725e1b96dacd4a6294a908702de
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6
-F src/btree.c 1439fd9b45d4d1883c53752daef42af489adaa1a1508fa39dedbc9c80ea21a2f
+F src/btree.c 73a3e74e0f6116ba43175577d8fd5eee66794908ae30dde6a0dcf317d2abfd81
 F src/btree.h 7af72bbb4863c331c8f6753277ab40ee67d2a2125a63256d5c25489722ec162b
 F src/btreeInt.h 83166f6daeb91062b6ae9ee6247b3ad07e40eba58f3c05ba9e8dedad4ab1ea38
-F src/build.c dbdaee54ffef924a070eb6202017e10d6be56baab953ef0a8e714a6def683198
+F src/build.c e3e99e0a4d678390d84fc1851fdd83a61ce110c05b9e49254f99c90fe793551b
 F src/callback.c d0b853dd413255d2e337b34545e54d888ea02f20da5ad0e63585b389624c4a6c
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c e98518d2d3d4029a13c805e07313fb60c877be56db76e90dd5f3af73085d0ce6
@@ -487,7 +487,7 @@ F src/date.c dace306a10d9b02ee553d454c8e1cf8d3c9b932e137738a6b15b90253a9bfc10
 F src/dbpage.c 8a01e865bf8bc6d7b1844b4314443a6436c07c3efe1d488ed89e81719047833a
 F src/dbstat.c 3aa79fc3aed7ce906e4ea6c10e85d657299e304f6049861fe300053ac57de36c
 F src/delete.c 410c771c25afc113c273d9efad6ab6881bda28c75a1838b9d2c52ba20d1dc704
-F src/expr.c 58c06940d964c2cf455b979cf66a648499d294a5ee6dadcaeaed447257c1dc75
+F src/expr.c 4c8b9c2942adb896b9d1613794bd00cbf98c94e38c41cd5720b78bcdc21a965d
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 83372403298e6a7dd989a47aaacdbaa5b4307b5199dbd56e07d4896066b3de72
 F src/func.c 2333eb4277f55a5efdc12ef754e7d7ec9105d257b2fd00301d23ce1e8fa67dc0
@@ -499,7 +499,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 957254a2d0542597455d0d4c640e4e3f3eea8c6d78f04582df03dfc626f07925
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 436af4968c6954d304fce9efa12719367bd8f37b19b93b71d6ad607e85adbb47
-F src/main.c 09580279145f27f3db206ef44dcb3a8875a42644230f79c7e54aff35e71668f0
+F src/main.c 57cdcb542209252148d09e362841d2840e17564fc904aa18482c53b79aec5d2e
 F src/malloc.c 22d5bdd9fe88ae4fad1b91a1b9735104b82853ffef868f1f05517d60dc1875f5
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -519,7 +519,7 @@ F src/os.c 80e4cf3e5da06be03ca641661e331ce60eeeeabf0d7354dbb1c0e166d0eedbbe
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 9b1b860163fd2d4d7679b5260d384d1a9f88ef917a90f28963eca8acd472d8c8
+F src/os_unix.c 89f19aec76a049b4deb5f1fc16afd4786bd1d151d20467abc6570c822abbfb35
 F src/os_win.c a2149ff0a85c1c3f9cc102a46c673ce87e992396ba3411bfb53db66813b32f1d
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 3700a1c55427a3d4168ad1f1b8a8b0cb9ace1d107e4506e30a8f1e66d8a1195e
@@ -530,18 +530,18 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586
 F src/pcache1.c 6596e10baf3d8f84cc1585d226cf1ab26564a5f5caf85a15757a281ff977d51a
 F src/pragma.c bdb600be936f66b9fe69d26dfbba4528beaaf4f95c479c85b328a92484e0bf71
 F src/pragma.h 8dc78ab7e9ec6ce3ded8332810a2066f1ef6267e2e03cd7356ee00276125c6cf
-F src/prepare.c 3d5a761d026052bc888d1b803a06dd2bfe245e8e836d4689f927003549148b0f
+F src/prepare.c 885fd7a986f9e70227d0d2f16fcd22fc957835dfcd6b59b56e62c62276f6be9d
 F src/printf.c 9efcd4e984f22bcccb1ded37a1178cac98f6e3a0534e1e0629f64899971f8838
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
-F src/resolve.c d74715aceed2a8f493ba244d535646fa93132042a4400a29dfd26ec841514048
+F src/resolve.c 97b91fb25d86881ff20c9ad2ad98412c6c1bb5f7d6c9bb044db250cbc9cfcd4b
 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
-F src/select.c 510fdf819f218be3dac2683d3eaaf64e5080f548061a4dd12205590beda976bb
-F src/shell.c.in b9b819feede7b85585ab0826490a352e04e2ee46e8132c92597d29972b2be1d7
+F src/select.c 233e884d7da6601486c7b93aedb97fd29302ae5c03742d0e0eccb4790638bb77
+F src/shell.c.in 651e844efde4bb1b6a0d43502b6ee7ef0e5e229b52b4f1e47ef11e08d1442fc9
 F src/sqlite.h.in d2c03414a8ee5d4a6855c04dd7cd5998e45139b0fe66b65bae86d4223edd091f
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 2d1af80082edffd71c6f96f70ad1ce6a4fb46615ad10291fc77fe0dea9ff0197
-F src/sqliteInt.h a1aa5457ca881cbf5adb55933bf81d7d4889375afb9a0a5df382a451c058087d
-F src/sqliteLimit.h 95cb8479ca459496d9c1c6a9f76b38aee12203a56ce1092fe13e50ae2454c032
+F src/sqliteInt.h 51a6847d9ee559b8c2be294a3184e7b7f8a51bfbbfd265b521be77bf118ca7a7
+F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
 F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
 F src/tclsqlite.c 986b6391f02cd9b53c1d688be55899f6ffddeb8e8014cd83c1b73ff912579a71
@@ -605,7 +605,7 @@ F src/trigger.c ef67bde309a831515dc3c2173d792574309f2f42d45f8c078743fae9f7f98c75
 F src/update.c fb15bec5b54fd098f4b84f6abc83c7103b45ba8484011fff8edf5ae31656eab6
 F src/upsert.c 2920de71b20f04fe25eb00b655d086f0ba60ea133c59d7fa3325c49838818e78
 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
-F src/util.c c8bf30c4356b091bcc3b624d0e24b2b4d11b8be4d6c90d8e0705971e15cc819b
+F src/util.c c0c7977de7ef9b8cb10f6c85f2d0557889a658f817b0455909a49179ba4c8002
 F src/vacuum.c 1c4f8e2f39d950037f4cf946b6858c993d3a54c3101f78e05c76460a073afcf0
 F src/vdbe.c e9f7f818f128c8600058c0eabb6b3975974c95153a104d340f419adabbc15b9f
 F src/vdbe.h 83603854bfa5851af601fc0947671eb260f4363e62e960e8a994fb9bbcd2aaa1
@@ -614,7 +614,7 @@ F src/vdbeapi.c c5e7cb2ab89a24d7f723e87b508f21bfb1359a04db5277d8a99fd1e015c12eb9
 F src/vdbeaux.c 73854da7a9a4f12db72a855758214173c82f46a14be6cb19e63677ba02c97cae
 F src/vdbeblob.c 253ed82894924c362a7fa3079551d3554cd1cdace39aa833da77d3bc67e7c1b1
 F src/vdbemem.c 947f2a65910edb4014dc981d33e414a68c51f169f9df8c4c493a0ba840b6eb1f
-F src/vdbesort.c 2be76d26998ce2b3324cdcc9f6443728e54b6c7677c553ad909c7d7cfab587df
+F src/vdbesort.c f5b5e473a7cee44e47a94817b042fd7172cf3aa2c0a7928a8339d612bcfdec5a
 F src/vdbetrace.c fa3bf238002f0bbbdfb66cc8afb0cea284ff9f148d6439bc1f6f2b4c3b7143f0
 F src/vdbevtab.c f99b275366c5fc5e2d99f734729880994ab9500bdafde7fae3b02d562b9d323c
 F src/vtab.c 5f5fc793092f53bbdfde296c50f563fb7bda58cf48e9cf6a8bdfbc5abd409845
@@ -667,7 +667,7 @@ F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b
 F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7
 F test/async4.test 1787e3952128aa10238bf39945126de7ca23685a
 F test/async5.test 383ab533fdb9f7ad228cc99ee66e1acb34cc0dc0
-F test/atof1.test 1ccfc96a6888566597b83d882c81b3c04258dc39317e8c1cec89ba481eaa2fba
+F test/atof1.test 10049623e77006691c4c2978c1dc8a3f75276377a53417811aa85bda7493f963
 F test/atomic.test 065a453dde33c77ff586d91ccaa6ed419829d492dbb1a5694b8a09f3f9d7d061
 F test/atomic2.test b6863b4aa552543874f80b42fb3063f1c8c2e3d8e56b6562f00a3cc347b5c1da
 F test/atrc.c c388fac43dbba05c804432a7135ae688b32e8f25818e9994ffba4b64cf60c27c
@@ -728,7 +728,7 @@ F test/btree01.test e08b3613540145b353f20c81cb18ead54ff12e0f
 F test/btree02.test 7555a5440453d900410160a52554fe6478af4faf53098f7235f1f443d5a1d6cc
 F test/btreefault.test c2bcb542685eea44621275cfedbd8a13f65201e3
 F test/busy.test 510dc6daaad18bcbbc085bcc6217d6dc418def5e73f72ce1475eea0cb7834727
-F test/busy2.test 2499cb62c9e58e18335892602c158cb35639c411803adca6423401b31e46c503
+F test/busy2.test 26ae3b8b74235c851f27326807de94292af4d359476c629f80dc106f35c8293e
 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de
 F test/cacheflush.test af25bb1509df04c1da10e38d8f322d66eceedf61
 F test/cachespill.test 895997f84a25b323b166aecb69baab2d6380ea98f9e0bcc688c4493c535cfab9
@@ -1050,7 +1050,7 @@ F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8
 F test/icu.test 716a6b89fbabe5cc63e0cd4c260befb08fd7b9d761f04d43669233292f0753b1
 F test/ieee754.test b0945d12be7d255f3dfa18e2511b17ca37e0edd2b803231c52d05b86c04ab26e
 F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8
-F test/in.test ae4ba0fe3232fdd84ef1090a68c5cd6ccd93f1f8774d5c967dd0c1b301492eed
+F test/in.test 95a704448eaa348300a517df267b37488c854aa0713125f4fa7b6280cfd4ed0f
 F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75
 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0
 F test/in4.test 65460600d48933adba4283c6ebd089aae173d16136ab9d01f74c89089090c5a5
@@ -1103,11 +1103,11 @@ F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4
 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b
 F test/istrue.test 06f92ea38750fa74df7dbbe6920205251c2310861fbbe23a3adfa918a2e2ba74
 F test/join.test bca044589e94bb466e4c1e91fb6fecdc3f3326ca6b3f590f555f1958156eb321
-F test/join2.test 7d24d095ab88d3910228d53a3b548b7baf2e0e7d8aac6731a273e300e1b34b61
+F test/join2.test 21fc30e54ab35ed66bf51b89cec18729205497f5cc43c83bc042f96a73721593
 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0
 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916
 F test/join5.test 3a96dc62f0b45402d7207e22d1993fe0c2fce1c57644a11439891dd62b990eb7
-F test/join6.test cfe6503791ceb0cbb509966740286ec423cbf10b
+F test/join6.test f809c025fa253f9e150c0e9afd4cef8813257bceeb6f46e04041228c9403cc2c
 F test/journal1.test c7b768041b7f494471531e17abc2f4f5ebf9e5096984f43ed17c4eb80ba34497
 F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4
 F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ffa140e
@@ -1272,7 +1272,7 @@ F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8
 F test/regexp2.test 40e894223b3d6672655481493f1be12012f2b33c
 F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2118d
 F test/releasetest.tcl fb76d8fcc95ac29d6356cd9e52b726ab9e43a24082897618dfbcb7c2b0049153 x
-F test/releasetest_data.tcl 9919fc6ac5bc92f8878fecfd1840db15999f660a6c9f609240b41aa62b885c88
+F test/releasetest_data.tcl ce3bebf5a6755412eca6f37f4cb7f8768027a8d9af8aac129dc10e656f4c0ff5
 F test/resetdb.test 8062cf10a09d8c048f8de7711e94571c38b38168db0e5877ba7561789e5eeb2b
 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb
 F test/rollback.test 06680159bc6746d0f26276e339e3ae2f951c64812468308838e0a3362d911eaa
@@ -1437,7 +1437,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46
 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd
 F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b
 F test/threadtest2.c a70a8e94bef23339d34226eb9521015ef99f4df8
-F test/threadtest3.c 38a612ea62854349ed66372f330a40d73c5cf956
+F test/threadtest3.c e63013af10cf236c7610eb06d33bde08c861806dc64be811940ff4d9ddd34a4f
 F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925
 F test/time-wordcount.sh 8e0b0f8109367827ad5d58f5cc849705731e4b90
 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c
@@ -1613,6 +1613,7 @@ F test/trustschema1.test 4e970aef0bfe0cee139703cc7209d0e0f07725d999b180ba50770f4
 F test/tt3_checkpoint.c 9e75cf7c1c364f52e1c47fd0f14c4340a9db0fe1
 F test/tt3_index.c 39eec10a35f57672225be4d182862152896dee4a
 F test/tt3_lookaside1.c 0377e202c3c2a50d688cb65ba203afeda6fafeb9
+F test/tt3_shared.c b37d22defc944a2ac4c91c927fd06c1d48cd51e2ce9d004fe868625bd2399f93
 F test/tt3_stress.c f9a769ca8b026ecc76ee93ca8c9700a5619f8e51c581107c4053ba6ac97f616f
 F test/tt3_vacuum.c 1753f45917699c9c1f66b64c717a717c9379f776
 F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff
@@ -1879,10 +1880,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b050976079ba4a22d4dfeadb81f40cf71da6588c95bf2b634d88b416de5accd7
-R 799818c17104aa7af945cf9f5cad3762
-T +bgcolor * #d0c0ff
-T +sym-release *
-T +sym-version-3.33.0 *
-U drh
-Z 652aa26f564c1138ae3a9226a8f034b3
+P 1f0055d0a2b36f9bd27d9d47a45a01be2644fc3be53d7c598fa8e112dd13e12b
+Q +3d35fa0be866213274fc09250225b345f6b08a9b4ec373d53d95e627e24512be
+R cc4823fc34bda6bfce8c062812255ccd
+U dan
+Z 2b8a424999570b5b73d98e1ebc73c1de

manifest.uuid

@@ -1 +1 @@
-fca8dc8b578f215a969cd899336378966156154710873e68b3d9ac5881b0ff3f
+807643c596b2315feed9e9c492dcdba1dc35d6eb81253a72f0bca320fcaa4fca

src/btree.c

@@ -2605,7 +2605,7 @@ btree_open_out:
     ** do not change the pager-cache size.
     */
     if( sqlite3BtreeSchema(p, 0, 0)==0 ){
-      sqlite3PagerSetCachesize(p->pBt->pPager, SQLITE_DEFAULT_CACHE_SIZE);
+      sqlite3BtreeSetCacheSize(p, SQLITE_DEFAULT_CACHE_SIZE);
     }
 
     pFile = sqlite3PagerFile(pBt->pPager);

src/build.c

@@ -1883,12 +1883,15 @@ static int resizeIndexObject(sqlite3 *db, Index *pIdx, int N){
   int nByte;
   if( pIdx->nColumn>=N ) return SQLITE_OK;
   assert( pIdx->isResized==0 );
-  nByte = (sizeof(char*) + sizeof(i16) + 1)*N;
+  nByte = (sizeof(char*) + sizeof(LogEst) + sizeof(i16) + 1)*N;
   zExtra = sqlite3DbMallocZero(db, nByte);
   if( zExtra==0 ) return SQLITE_NOMEM_BKPT;
   memcpy(zExtra, pIdx->azColl, sizeof(char*)*pIdx->nColumn);
   pIdx->azColl = (const char**)zExtra;
   zExtra += sizeof(char*)*N;
+  memcpy(zExtra, pIdx->aiRowLogEst, sizeof(LogEst)*(pIdx->nKeyCol+1));
+  pIdx->aiRowLogEst = (LogEst*)zExtra;
+  zExtra += sizeof(LogEst)*N;
   memcpy(zExtra, pIdx->aiColumn, sizeof(i16)*pIdx->nColumn);
   pIdx->aiColumn = (i16*)zExtra;
   zExtra += sizeof(i16)*N;

src/expr.c

@@ -44,7 +44,7 @@ char sqlite3TableColumnAffinity(Table *pTab, int iCol){
 */
 char sqlite3ExprAffinity(const Expr *pExpr){
   int op;
-  while( ExprHasProperty(pExpr, EP_Skip) ){
+  while( ExprHasProperty(pExpr, EP_Skip|EP_IfNullRow) ){
     assert( pExpr->op==TK_COLLATE || pExpr->op==TK_IF_NULL_ROW );
     pExpr = pExpr->pLeft;
     assert( pExpr!=0 );
@@ -115,7 +115,7 @@ Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){
 */
 Expr *sqlite3ExprSkipCollate(Expr *pExpr){
   while( pExpr && ExprHasProperty(pExpr, EP_Skip) ){
-    assert( pExpr->op==TK_COLLATE || pExpr->op==TK_IF_NULL_ROW );
+    assert( pExpr->op==TK_COLLATE );
     pExpr = pExpr->pLeft;
   }   
   return pExpr;
@@ -134,7 +134,7 @@ Expr *sqlite3ExprSkipCollateAndLikely(Expr *pExpr){
       assert( pExpr->op==TK_FUNCTION );
       pExpr = pExpr->x.pList->a[0].pExpr;
     }else{
-      assert( pExpr->op==TK_COLLATE || pExpr->op==TK_IF_NULL_ROW );
+      assert( pExpr->op==TK_COLLATE );
       pExpr = pExpr->pLeft;
     }
   }   
@@ -768,6 +768,7 @@ int sqlite3SelectExprHeight(Select *p){
 ** Expr.flags. 
 */
 void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){
+  if( pParse->nErr ) return;
   if( p && p->x.pList && !ExprHasProperty(p, EP_xIsSelect) ){
     p->flags |= EP_Propagate & sqlite3ExprListFlags(p->x.pList);
   }

src/main.c

@@ -3822,7 +3822,9 @@ int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){
       }
       rc = SQLITE_OK;
     }else{
+      int nSave = db->busyHandler.nBusy;
       rc = sqlite3OsFileControl(fd, op, pArg);
+      db->busyHandler.nBusy = nSave;
     }
     sqlite3BtreeLeave(pBtree);
   }

src/os_unix.c

@@ -1544,6 +1544,9 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){
   return rc;
 }
 
+/* Forward declaration*/
+static int unixSleep(sqlite3_vfs*,int);
+
 /*
 ** Set a posix-advisory-lock.
 **
@@ -1573,7 +1576,7 @@ static int osSetPosixAdvisoryLock(
     ** generic posix, however, there is no such API.  So we simply try the
     ** lock once every millisecond until either the timeout expires, or until
     ** the lock is obtained. */
-    usleep(1000);
+    unixSleep(0,1000);
     rc = osFcntl(h,F_SETLK,pLock);
     tm--;
   }
@@ -2144,6 +2147,7 @@ static int unixClose(sqlite3_file *id){
   }
   sqlite3_mutex_leave(pInode->pLockMutex);
   releaseInodeInfo(pFile);
+  assert( pFile->pShm==0 );
   rc = closeUnixFile(id);
   unixLeaveMutex();
   return rc;
@@ -6543,7 +6547,8 @@ static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){
   UNUSED_PARAMETER(NotUsed);
   return microseconds;
 #elif defined(HAVE_USLEEP) && HAVE_USLEEP
-  usleep(microseconds);
+  if( microseconds>=1000000 ) sleep(microseconds/1000000);
+  if( microseconds%1000000 ) usleep(microseconds%1000000);
   UNUSED_PARAMETER(NotUsed);
   return microseconds;
 #else
@@ -7116,7 +7121,7 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){
       
       if( nTries==1 ){
         conchModTime = buf.st_mtimespec;
-        usleep(500000); /* wait 0.5 sec and try the lock again*/
+        unixSleep(0,500000); /* wait 0.5 sec and try the lock again*/
         continue;  
       }
 
@@ -7142,7 +7147,7 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){
           /* don't break the lock on short read or a version mismatch */
           return SQLITE_BUSY;
         }
-        usleep(10000000); /* wait 10 sec and try the lock again */
+        unixSleep(0,10000000); /* wait 10 sec and try the lock again */
         continue; 
       }
       

src/prepare.c

@@ -746,6 +746,7 @@ static int sqlite3LockAndPrepare(
   sqlite3BtreeLeaveAll(db);
   rc = sqlite3ApiExit(db, rc);
   assert( (rc&db->errMask)==rc );
+  db->busyHandler.nBusy = 0;
   sqlite3_mutex_leave(db->mutex);
   return rc;
 }

src/resolve.c

@@ -1075,7 +1075,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
       assert( !ExprHasProperty(pExpr, EP_Reduced) );
       /* Handle special cases of "x IS TRUE", "x IS FALSE", "x IS NOT TRUE",
       ** and "x IS NOT FALSE". */
-      if( pRight && pRight->op==TK_ID ){
+      if( pRight && (pRight->op==TK_ID || pRight->op==TK_TRUEFALSE) ){
         int rc = resolveExprStep(pWalker, pRight);
         if( rc==WRC_Abort ) return WRC_Abort;
         if( pRight->op==TK_TRUEFALSE ){

src/select.c

@@ -3506,7 +3506,7 @@ static Expr *substExpr(
           ifNullRow.op = TK_IF_NULL_ROW;
           ifNullRow.pLeft = pCopy;
           ifNullRow.iTable = pSubst->iNewTable;
-          ifNullRow.flags = EP_Skip;
+          ifNullRow.flags = EP_IfNullRow;
           pCopy = &ifNullRow;
         }
         testcase( ExprHasProperty(pCopy, EP_Subquery) );

src/shell.c.in

@@ -4426,6 +4426,19 @@ static void shellIdQuote(
   }
 }
 
+/*
+** Scalar function "usleep(X)" invokes sqlite3_sleep(X) and returns X.
+*/
+static void shellUSleepFunc(
+  sqlite3_context *context, 
+  int argc, 
+  sqlite3_value **argv
+){
+  int sleep = sqlite3_value_int(argv[0]);
+  sqlite3_sleep(sleep/1000);
+  sqlite3_result_int(context, sleep);
+}
+
 /*
 ** Scalar function "shell_escape_crnl" used by the .recover command.
 ** The argument passed to this function is the output of built-in
@@ -4609,6 +4622,8 @@ static void open_db(ShellState *p, int openFlags){
                             shellInt32, 0, 0);
     sqlite3_create_function(p->db, "shell_idquote", 1, SQLITE_UTF8, 0,
                             shellIdQuote, 0, 0);
+    sqlite3_create_function(p->db, "usleep",1,SQLITE_UTF8,0,
+                            shellUSleepFunc, 0, 0);
 #ifndef SQLITE_NOHAVE_SYSTEM
     sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0,
                             editFunc, 0, 0);

src/sqliteInt.h

@@ -2715,7 +2715,7 @@ struct Expr {
 #define EP_TokenOnly  0x004000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */
 #define EP_Win        0x008000 /* Contains window functions */
 #define EP_MemToken   0x010000 /* Need to sqlite3DbFree() Expr.zToken */
-                  /*  0x020000 // available for reuse */
+#define EP_IfNullRow  0x020000 /* The TK_IF_NULL_ROW opcode */
 #define EP_Unlikely   0x040000 /* unlikely() or likelihood() function */
 #define EP_ConstFunc  0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */
 #define EP_CanBeNull  0x100000 /* Can be null despite NOT NULL constraint */

src/sqliteLimit.h

@@ -60,11 +60,7 @@
 ** The maximum depth of an expression tree. This is limited to 
 ** some extent by SQLITE_MAX_SQL_LENGTH. But sometime you might 
 ** want to place more severe limits on the complexity of an 
-** expression.
-**
-** A value of 0 used to mean that the limit was not enforced.
-** But that is no longer true.  The limit is now strictly enforced
-** at all times.
+** expression. A value of 0 means that there is no limit.
 */
 #ifndef SQLITE_MAX_EXPR_DEPTH
 # define SQLITE_MAX_EXPR_DEPTH 1000

src/util.c

@@ -681,6 +681,7 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){
     incr = 1;
   }else{
     incr = 2;
+    length &= ~1;
     assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 );
     for(i=3-enc; i<length && zNum[i]==0; i+=2){}
     nonNum = i<length;

src/vdbesort.c

@@ -970,13 +970,16 @@ int sqlite3VdbeSorterInit(
   if( pSorter==0 ){
     rc = SQLITE_NOMEM_BKPT;
   }else{
+    Btree *pBt = db->aDb[0].pBt;
     pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz);
     memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo);
     pKeyInfo->db = 0;
     if( nField && nWorker==0 ){
       pKeyInfo->nKeyField = nField;
     }
-    pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
+    sqlite3BtreeEnter(pBt);
+    pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(pBt);
+    sqlite3BtreeLeave(pBt);
     pSorter->nTask = nWorker + 1;
     pSorter->iPrev = (u8)(nWorker - 1);
     pSorter->bUseThreads = (pSorter->nTask>1);

test/atof1.test

@@ -75,10 +75,10 @@ do_execsql_test atof1-2.30 {
   CREATE INDEX i1 ON t1(a);
   SELECT count(*) FROM t1 WHERE substr(a,',');
 } {1}
-
-
-
-
+# 2020-08-27 OSSFuzz find related to the above.
+do_execsql_test atof1-2.40 {
+  SELECT randomblob(0) - 1;
+} {-1}
 
 
 finish_test

test/busy2.test

@@ -131,5 +131,41 @@ do_multiclient_test tn {
   }
 }
 
+#-------------------------------------------------------------------------
+# Check that even if the busy-handler fails (returns zero) within a
+# call to sqlite3_prepare() (or _v2(), or _v3()), it is still invoked
+# the next time an SQLITE_BUSY is encountered.
+#
+
+do_multiclient_test tn {
+  code1 {
+    set ::busy_called 0
+    proc busy {args} {
+      if {$::busy_called} { return 1 }
+      set ::busy_called 1
+      return 0
+    }
+    db busy busy
+  }
+
+  do_test 3.$tn.1 {
+    sql2 {
+      CREATE TABLE t1(x);
+      BEGIN EXCLUSIVE;
+        INSERT INTO t1 VALUES('x');
+    }
+  } {}
+
+  do_test 3.$tn.2 {
+    set ::busy_called 0
+    list [catch { sql1 { SELECT * FROM t1 } } msg] $msg $::busy_called
+  } {1 {database is locked} 1}
+
+  do_test 3.$tn.3 {
+    set ::busy_called 0
+    list [catch { sql1 { SELECT * FROM t1 } } msg] $msg $::busy_called
+  } {1 {database is locked} 1}
+}
+
 finish_test
 

test/in.test

@@ -791,4 +791,10 @@ do_execsql_test in-19.40 {
   PRAGMA integrity_check;
 } {ok}
 
+# Ticket f3ff1472887
+#
+do_execsql_test in-20.1 {
+  SELECT (1 IN (2 IS TRUE));
+} {1}
+
 finish_test

test/join2.test

@@ -294,7 +294,10 @@ do_execsql_test 8.1 {
 }
 
 #-------------------------------------------------------------------------
-# Ticket [45f4bf4eb].
+# Ticket [45f4bf4eb] reported by Manuel Rigger (2020-04-25)
+#
+# Follow up error reported by Eric Speckman on the SQLite forum
+# https://sqlite.org/forum/info/c49496d24d35bd7c (2020-08-19)
 #
 reset_db
 do_execsql_test 9.0 {
@@ -324,5 +327,17 @@ do_execsql_test 9.5 {
   UNION SELECT 0,0 WHERE 0; 
 } {0 0}
 
+do_execsql_test 9.10 {
+  CREATE TABLE t1 (aaa);
+  INSERT INTO t1 VALUES(23456);
+  CREATE TABLE t2(bbb);
+  CREATE VIEW v2(ccc) AS SELECT bbb IS 1234 FROM t2;
+  SELECT ccc, ccc IS NULL AS ddd FROM t1 LEFT JOIN v2;
+} {{} 1}
+optimization_control db query-flattener 0
+do_execsql_test 9.11 {
+  SELECT ccc, ccc IS NULL AS ddd FROM t1 LEFT JOIN v2;
+} {{} 1}
+
 
 finish_test

test/join6.test

@@ -147,6 +147,22 @@ ifcapable compound {
   } {1 91 92 3 93 5}
 }
 
+do_execsql_test join6-5.1 {
+  CREATE TABLE tx(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o PRIMARY KEY) 
+  WITHOUT ROWID;
+  INSERT INTO tx VALUES(
+    1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+  );
+} {}
+do_execsql_test joint6-5.2 {
+  SELECT o FROM tx NATURAL JOIN tx;
+} {15}
+
+do_execsql_test join6-5.3 {
+  CREATE TABLE ty(a,Ñ,x6,x7,x8,Q,I,v,x1,L,E,x2,x3,x4,x5,s,g PRIMARY KEY,b,c)
+  WITHOUT ROWID;
+  SELECT a FROM ty NATURAL JOIN ty;
+}
 
 
 

test/releasetest_data.tcl

@@ -135,6 +135,11 @@ array set ::Configs [strip_comments {
     -DSQLITE_MUTATION_TEST
     --enable-fts5 --enable-json1
   }
+  "Debug-Two" {
+    -DSQLITE_DEFAULT_MEMSTATUS=0
+    -DSQLITE_MAX_EXPR_DEPTH=0
+    --enable-debug
+  }
   "Fast-One" {
     -O6
     -DSQLITE_ENABLE_FTS4=1
@@ -284,6 +289,7 @@ array set ::Platforms [strip_comments {
     "Check-Symbols*"          checksymbols
     "Fast-One"                "fuzztest test"
     "Debug-One"               "mptest test"
+    "Debug-Two"               "test"
     "Have-Not"                test
     "Secure-Delete"           test
     "Unlock-Notify"           "QUICKTEST_INCLUDE=notify2.test test"

test/threadtest3.c

@@ -1433,6 +1433,7 @@ static void dynamic_triggers(int nMs){
 #include "tt3_lookaside1.c"
 #include "tt3_vacuum.c"
 #include "tt3_stress.c"
+#include "tt3_shared.c"
 
 int main(int argc, char **argv){
   struct ThreadTest {
@@ -1457,6 +1458,7 @@ int main(int argc, char **argv){
     { vacuum1,             "vacuum1", 10000 },
     { stress1,             "stress1", 10000 },
     { stress2,             "stress2", 60000 },
+    { shared1,             "shared1", 10000 },
   };
   static char *substArgv[] = { 0, "*", 0 };
   int i, iArg;

test/tt3_shared.c

@@ -0,0 +1,55 @@
+/*
+** 2020 September 5
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+**
+*/
+
+
+/*
+*/
+static char *shared_thread1(int iTid, void *pArg){
+  Error err = {0};                /* Error code and message */
+
+  while( !timetostop(&err) ){
+    Sqlite db = {0};              /* SQLite database connection */
+    opendb(&err, &db, "test.db", 0);
+    sql_script(&err, &db, "SELECT * FROM t1");
+    closedb(&err, &db);
+  }
+  print_and_free_err(&err);
+  return sqlite3_mprintf("done!");
+}
+
+
+static void shared1(int nMs){
+  Error err = {0};
+  Sqlite db = {0};              /* SQLite database connection */
+  Threadset threads = {0};
+  int ii;
+
+  opendb(&err, &db, "test.db", 1);
+  sql_script(&err, &db, "CREATE TABLE t1(x)");
+  closedb(&err, &db);
+
+  setstoptime(&err, nMs);
+  sqlite3_enable_shared_cache(1);
+
+  for(ii=0; ii<5; ii++){
+    launch_thread(&err, &threads, shared_thread1, 0);
+  }
+
+  join_all_threads(&err, &threads);
+  sqlite3_enable_shared_cache(0);
+
+  print_and_free_err(&err);
+}
+