From f16cf653f44115821c4718a255f8be6d8d594baa Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 19 May 2020 12:27:29 +0000 Subject: [PATCH 1/8] In wal.c, improved comments on concurrency issues. More use of AtomicLoad() and AtomicStore(). FossilOrigin-Name: 4bf566feca3a8fbe5e386533aac30e0ac25836cfc820a3abd91e156bd6198b4a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/wal.c | 48 +++++++++++++++++++++--------------------------- 3 files changed, 29 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index 9f4f6e3da8..26d7750597 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\svdbevtab.lo\sto\sbuild\sof\slibsqlite3.lib\sin\sMakefile.msc. -D 2020-05-18T18:02:37.420 +C In\swal.c,\simproved\scomments\son\sconcurrency\sissues.\s\sMore\suse\sof\nAtomicLoad()\sand\sAtomicStore(). +D 2020-05-19T12:27:29.008 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -617,7 +617,7 @@ F src/vdbetrace.c fa3bf238002f0bbbdfb66cc8afb0cea284ff9f148d6439bc1f6f2b4c3b7143 F src/vdbevtab.c ee5b4c902fdda2230f9503ac7b84c6d614c91e8f6f4dc1633e2e8dfef8ffb144 F src/vtab.c 7b452592ed2ee95dedb1f323d557cebede5a6f3b4558b21a5dca527e6ae9b12c F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c d0365581aed11a75e176f67cdbdc1bdfe685c5381a052360a00bbf58d3252fa0 +F src/wal.c fc763af2c20d1770e872378b28031c100b52061ec47599f38499cf4bd4977284 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c 7c429c694abd12413a5c17aec9f47cfe9eba6807e6b0a32df883e8e3a14835ed F src/where.c 9546c82056e8cdb27291f98cf1adca5d271240b399bb97b32f77fc2bea6146c9 @@ -1866,7 +1866,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 69e149f76853d196c8855fedfc98848b60fb116ac36bc08824b1a122469f8ece -R 3630bc41d5bae3aeb507521e3969369e -U dan -Z ad12d06ec98773d9201d7cf643b16e9f +P f64d054b799c0af8d582fd9aff9bcbfa380ef08026ba5db8c5f89f6f56dfdad9 +R 7f686b5b79d2dddadbc47e9d743b06cb +U drh +Z 8798d48c38c827000797710a250a5723 diff --git a/manifest.uuid b/manifest.uuid index 7c14e40f17..09b77250a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f64d054b799c0af8d582fd9aff9bcbfa380ef08026ba5db8c5f89f6f56dfdad9 \ No newline at end of file +4bf566feca3a8fbe5e386533aac30e0ac25836cfc820a3abd91e156bd6198b4a \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index e8f7f24f85..3816883686 100644 --- a/src/wal.c +++ b/src/wal.c @@ -689,6 +689,10 @@ static void walChecksumBytes( aOut[1] = s2; } +/* +** If there is the possibility of concurrent access to the SHM file +** from multiple threads and/or processes, then do a memory barrier. +*/ static void walShmBarrier(Wal *pWal){ if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){ sqlite3OsShmBarrier(pWal->pDbFd); @@ -708,6 +712,7 @@ static void walIndexWriteHdr(Wal *pWal){ pWal->hdr.isInit = 1; pWal->hdr.iVersion = WALINDEX_MAX_VERSION; walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); + /* Possible TSAN false-positive. See tag-20200519-1 */ memcpy((void*)&aHdr[1], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); walShmBarrier(pWal); memcpy((void*)&aHdr[0], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); @@ -1897,32 +1902,13 @@ static int walCheckpoint( mxSafeFrame = pWal->hdr.mxFrame; mxPage = pWal->hdr.nPage; for(i=1; iaReadMark[i]; + u32 y = AtomicLoad(pInfo->aReadMark+i); if( mxSafeFrame>y ){ assert( y<=pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); if( rc==SQLITE_OK ){ - pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED); + u32 iMark = (i==1 ? mxSafeFrame : READMARK_NOT_USED); + AtomicStore(pInfo->aReadMark+i, iMark); walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); }else if( rc==SQLITE_BUSY ){ mxSafeFrame = y; @@ -1940,7 +1926,7 @@ static int walCheckpoint( } if( pIter - && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0),1))==SQLITE_OK + && (rc = walBusyLock(pWal,xBusy,pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK ){ u32 nBackfill = pInfo->nBackfill; @@ -2168,13 +2154,19 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ ** meaning it is possible that an inconsistent snapshot is read ** from the file. If this happens, return non-zero. ** + ** tag-20200519-1: ** There are two copies of the header at the beginning of the wal-index. ** When reading, read [0] first then [1]. Writes are in the reverse order. ** Memory barriers are used to prevent the compiler or the hardware from - ** reordering the reads and writes. + ** reordering the reads and writes. TSAN and similar tools can sometimes + ** give false-positive warnings about these accesses because the tools do not + ** account for the double-read and the memory barrier. The use of mutexes + ** here would be problematic as the memory being accessed is potentially + ** shared among multiple processes and not all mutex implementions work + ** reliably in that environment. */ aHdr = walIndexHdr(pWal); - memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); + memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); /* Possible TSAN false-positive */ walShmBarrier(pWal); memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); @@ -3015,14 +3007,15 @@ int sqlite3WalFindFrame( int iKey; /* Hash slot index */ int nCollide; /* Number of hash collisions remaining */ int rc; /* Error code */ + u32 iH; rc = walHashGet(pWal, iHash, &sLoc); if( rc!=SQLITE_OK ){ return rc; } nCollide = HASHTABLE_NSLOT; - for(iKey=walHash(pgno); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ - u32 iH = sLoc.aHash[iKey]; + iKey = walHash(pgno); + while( (iH = AtomicLoad(&sLoc.aHash[iKey]))!=0 ){ u32 iFrame = iH + sLoc.iZero; if( iFrame<=iLast && iFrame>=pWal->minFrame && sLoc.aPgno[iH]==pgno ){ assert( iFrame>iRead || CORRUPT_DB ); @@ -3031,6 +3024,7 @@ int sqlite3WalFindFrame( if( (nCollide--)==0 ){ return SQLITE_CORRUPT_BKPT; } + iKey = walNextHash(iKey); } if( iRead ) break; } From 1a7feefac5c1729394bc70f22e652e0097c55747 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 19 May 2020 12:29:56 +0000 Subject: [PATCH 2/8] User-suggested improvement to the README.txt for the ICU extension. FossilOrigin-Name: 1d1293c25abdd4b0fca64e5b1d0a118e2f339635dbc89422c0c2463fbf9ee27f --- ext/icu/README.txt | 3 ++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ext/icu/README.txt b/ext/icu/README.txt index af75d22e61..be443f54ef 100644 --- a/ext/icu/README.txt +++ b/ext/icu/README.txt @@ -116,7 +116,8 @@ SQLite. Documentation follows. and use it as a dynamically loadable SQLite extension. To do this using gcc on *nix: - gcc -shared icu.c `icu-config --ldflags` -o libSqliteIcu.so + gcc -fPIC -shared icu.c `pkg-config --libs --cflags icu-uc icu-io` \ + -o libSqliteIcu.so You may need to add "-I" flags so that gcc can find sqlite3ext.h and sqlite3.h. The resulting shared lib, libSqliteIcu.so, may be diff --git a/manifest b/manifest index 26d7750597..3afe29ae0d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\swal.c,\simproved\scomments\son\sconcurrency\sissues.\s\sMore\suse\sof\nAtomicLoad()\sand\sAtomicStore(). -D 2020-05-19T12:27:29.008 +C User-suggested\simprovement\sto\sthe\sREADME.txt\sfor\sthe\sICU\sextension. +D 2020-05-19T12:29:56.039 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -230,7 +230,7 @@ F ext/fts5/tool/fts5txt2db.tcl 526a9979c963f1c54fd50976a05a502e533a4c59 F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45 F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c -F ext/icu/README.txt a295e91db742b153e8dce8f7efd31d28ad1eea4df31ef4daa3eedc85be2f5138 +F ext/icu/README.txt 1c48ffaf7f255bd73d00a35f68f6de357c2a6594f16cb00506a151be23694706 F ext/icu/icu.c 91c021c7e3e8bbba286960810fa303295c622e323567b2e6def4ce58e4466e60 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/lsm1/Makefile a553b728bba6c11201b795188c5708915cc4290f02b7df6ba7e8c4c943fd5cd9 @@ -1866,7 +1866,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f64d054b799c0af8d582fd9aff9bcbfa380ef08026ba5db8c5f89f6f56dfdad9 -R 7f686b5b79d2dddadbc47e9d743b06cb +P 4bf566feca3a8fbe5e386533aac30e0ac25836cfc820a3abd91e156bd6198b4a +R 967365fc0f01aa59917c371276c2fe8f U drh -Z 8798d48c38c827000797710a250a5723 +Z 6f8bf30a9a141e896db917f9dbf77f7a diff --git a/manifest.uuid b/manifest.uuid index 09b77250a7..78e460a488 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4bf566feca3a8fbe5e386533aac30e0ac25836cfc820a3abd91e156bd6198b4a \ No newline at end of file +1d1293c25abdd4b0fca64e5b1d0a118e2f339635dbc89422c0c2463fbf9ee27f \ No newline at end of file From 10757ed0cae8a7b8b70e74bea2ba521008e0bc2a Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 19 May 2020 15:40:07 +0000 Subject: [PATCH 3/8] In os_win.c, avoid calling sqlite3_uri_boolean() on anything other than a main-db filename. FossilOrigin-Name: cb0a18e64f8d81c2ada9f698faaf5ca68a0666687bf6f3abc860324cb1997463 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 4 +++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3afe29ae0d..f97053e0ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C User-suggested\simprovement\sto\sthe\sREADME.txt\sfor\sthe\sICU\sextension. -D 2020-05-19T12:29:56.039 +C In\sos_win.c,\savoid\scalling\ssqlite3_uri_boolean()\son\sanything\sother\sthan\sa\smain-db\sfilename. +D 2020-05-19T15:40:07.909 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -518,7 +518,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 13f983da988b6460ef3c4c22099c67ab0938291e543644ac4d99eccc8ba604f1 -F src/os_win.c 317718e8f04c449e7d9e1eacac0d14e7508e4a77c9d4c3cb7382299bd24561b7 +F src/os_win.c e832e21e830c1f9409c9c54053939b6dcb14c1e92128b756204ce1e3e331d678 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 96436cb1920074d4ade120a1a8a9d0ae3f52df06651e21b7eccc5eae2f02b111 F src/pager.h 8d1dc9a2c3fc5eb6eeed75f48a076f425e77706f8935f05817fa05a308f587b5 @@ -1866,7 +1866,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4bf566feca3a8fbe5e386533aac30e0ac25836cfc820a3abd91e156bd6198b4a -R 967365fc0f01aa59917c371276c2fe8f -U drh -Z 6f8bf30a9a141e896db917f9dbf77f7a +P 1d1293c25abdd4b0fca64e5b1d0a118e2f339635dbc89422c0c2463fbf9ee27f +R 77702df5e982362c55239f265fbb9eea +U dan +Z 7af2360054e2377399137e1a09b7af22 diff --git a/manifest.uuid b/manifest.uuid index 78e460a488..82c4823f99 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1d1293c25abdd4b0fca64e5b1d0a118e2f339635dbc89422c0c2463fbf9ee27f \ No newline at end of file +cb0a18e64f8d81c2ada9f698faaf5ca68a0666687bf6f3abc860324cb1997463 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 86789ced87..0adfd7fbf2 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -5272,7 +5272,9 @@ static int winOpen( if( isReadonly ){ pFile->ctrlFlags |= WINFILE_RDONLY; } - if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ + if( (flags & SQLITE_OPEN_MAIN_DB) + && sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) + ){ pFile->ctrlFlags |= WINFILE_PSOW; } pFile->lastErrno = NO_ERROR; From 5a8cd2e40ce5287e638f77d4922068dbf7ba7e03 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 19 May 2020 15:51:10 +0000 Subject: [PATCH 4/8] Add compiler hints to disable TSAN for the routines that access the -shm file header in WAL mode using a double-read with memory barrier. FossilOrigin-Name: 3117c1b5a9e348fd8d16ba9d03fdafaad8514567fb3403f72b86d6162ad40bde --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/wal.c | 17 +++++++++++++++-- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f97053e0ff..a5b7b1fcbf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sos_win.c,\savoid\scalling\ssqlite3_uri_boolean()\son\sanything\sother\sthan\sa\smain-db\sfilename. -D 2020-05-19T15:40:07.909 +C Add\scompiler\shints\sto\sdisable\sTSAN\sfor\sthe\sroutines\sthat\saccess\sthe\s-shm\nfile\sheader\sin\sWAL\smode\susing\sa\sdouble-read\swith\smemory\sbarrier. +D 2020-05-19T15:51:10.216 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -617,7 +617,7 @@ F src/vdbetrace.c fa3bf238002f0bbbdfb66cc8afb0cea284ff9f148d6439bc1f6f2b4c3b7143 F src/vdbevtab.c ee5b4c902fdda2230f9503ac7b84c6d614c91e8f6f4dc1633e2e8dfef8ffb144 F src/vtab.c 7b452592ed2ee95dedb1f323d557cebede5a6f3b4558b21a5dca527e6ae9b12c F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c fc763af2c20d1770e872378b28031c100b52061ec47599f38499cf4bd4977284 +F src/wal.c 17ea0a319d3ead17ef3b16aa30f10f2626056893effea7e609a20a6661ffec1b F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c 7c429c694abd12413a5c17aec9f47cfe9eba6807e6b0a32df883e8e3a14835ed F src/where.c 9546c82056e8cdb27291f98cf1adca5d271240b399bb97b32f77fc2bea6146c9 @@ -1866,7 +1866,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1d1293c25abdd4b0fca64e5b1d0a118e2f339635dbc89422c0c2463fbf9ee27f -R 77702df5e982362c55239f265fbb9eea -U dan -Z 7af2360054e2377399137e1a09b7af22 +P cb0a18e64f8d81c2ada9f698faaf5ca68a0666687bf6f3abc860324cb1997463 +R 8462379efa542a8195db58fd9ef94a35 +U drh +Z 5f8f2ff4f8e169f83e77c91d9875e042 diff --git a/manifest.uuid b/manifest.uuid index 82c4823f99..c90a442804 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cb0a18e64f8d81c2ada9f698faaf5ca68a0666687bf6f3abc860324cb1997463 \ No newline at end of file +3117c1b5a9e348fd8d16ba9d03fdafaad8514567fb3403f72b86d6162ad40bde \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 3816883686..63dee82330 100644 --- a/src/wal.c +++ b/src/wal.c @@ -699,12 +699,25 @@ static void walShmBarrier(Wal *pWal){ } } +/* +** Add the SQLITE_NO_TSAN as part of the return-type of a function +** definition as a hint that the function contains constructs that +** might give false-positive TSAN warnings. +** +** See tag-20200519-1. +*/ +#if defined(__clang__) && !defined(SQLITE_NO_TSAN) +# define SQLITE_NO_TSAN __attribute__((no_sanitize_thread)) +#else +# define SQLITE_NO_TSAN +#endif + /* ** Write the header information in pWal->hdr into the wal-index. ** ** The checksum on pWal->hdr is updated before it is written. */ -static void walIndexWriteHdr(Wal *pWal){ +static SQLITE_NO_TSAN void walIndexWriteHdr(Wal *pWal){ volatile WalIndexHdr *aHdr = walIndexHdr(pWal); const int nCksum = offsetof(WalIndexHdr, aCksum); @@ -2141,7 +2154,7 @@ int sqlite3WalClose( ** If the checksum cannot be verified return non-zero. If the header ** is read successfully and the checksum verified, return zero. */ -static int walIndexTryHdr(Wal *pWal, int *pChanged){ +static SQLITE_NO_TSAN int walIndexTryHdr(Wal *pWal, int *pChanged){ u32 aCksum[2]; /* Checksum on the header content */ WalIndexHdr h1, h2; /* Two copies of the header content */ WalIndexHdr volatile *aHdr; /* Header in shared memory */ From 4f1315a43203026aa23f1c1df630249ccf2b32b1 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 May 2020 15:02:04 +0000 Subject: [PATCH 5/8] Back out the change from [7fab1393c2b22b1f] that tries to convert invalid surrogate characters in UTF16 into the replacement character 0xfffd, as we find that this breaks some software. FossilOrigin-Name: 4218c7b71fb6b227dbe4b852718584c150164af2d84e067cb810aa602554a609 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/utf.c | 16 ++++++++++++++++ test/tkt-3fe897352e.test | 8 ++++---- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index a5b7b1fcbf..81eb1b4a4a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\scompiler\shints\sto\sdisable\sTSAN\sfor\sthe\sroutines\sthat\saccess\sthe\s-shm\nfile\sheader\sin\sWAL\smode\susing\sa\sdouble-read\swith\smemory\sbarrier. -D 2020-05-19T15:51:10.216 +C Back\sout\sthe\schange\sfrom\s[7fab1393c2b22b1f]\sthat\stries\sto\sconvert\sinvalid\nsurrogate\scharacters\sin\sUTF16\sinto\sthe\sreplacement\scharacter\s0xfffd,\sas\swe\nfind\sthat\sthis\sbreaks\ssome\ssoftware. +D 2020-05-20T15:02:04.035 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -602,7 +602,7 @@ F src/treeview.c 82c6391a3ba76215d4185fd4719a56ec4caf186a40c8a7b6e6ba4ae4467c274 F src/trigger.c 4ada1037cc99777f647a882cdacbd1a4deb6567b69daf02946286401b88cdc04 F src/update.c 3199098455830fc2d8c8fc4ae3ec2ea513eef64339ae9a7048db62b21169bc7a F src/upsert.c 2920de71b20f04fe25eb00b655d086f0ba60ea133c59d7fa3325c49838818e78 -F src/utf.c 95fb6e03a5ca679045c5adccd05380f0addccabef5911abddcb06af069500ab7 +F src/utf.c d7a61c1dfdac3eb091d43341a674032dca5a34e122f78ef0b5bd2d5a31967dde F src/util.c 3b6cedf7a0c69bd6e1acce832873952d416212d6293b18d03064e07d7a9b5118 F src/vacuum.c de9780b89fa4ee74c3534f60b94820e3179aca759ffc1338ee53cb4ea7693dd3 F src/vdbe.c e3dba0dee25bc92e871c13cac655260912b3be4abcc85b439259c9934b208da3 @@ -1446,7 +1446,7 @@ F test/tkt-385a5b56b9.test 5204a7cba0e28c99df0acbf95af5e1af4d32965a7a14de6eccebf F test/tkt-38cb5df375.test f3cc8671f1eb604d4ae9cf886ed4366bec656678 F test/tkt-3998683a16.test 6d1d04d551ed1704eb3396ca87bb9ccc8c5c1eb7 F test/tkt-3a77c9714e.test 90e3e8455ee945a4076d4c44062b8845708af24a880355328fe7008f2047c9f0 -F test/tkt-3fe897352e.test 6849fde0a87165ff83f54f5047af7c743d72af26908fadb90174f3294450b3f4 +F test/tkt-3fe897352e.test 27e26eb0f1811aeba4d65aba43a4c52e99da5e70 F test/tkt-4a03edc4c8.test 91c0e135888cdc3d4eea82406a44b05c8c1648d0 F test/tkt-4c86b126f2.test cbcc611becd0396890169ab23102dd70048bbc9a F test/tkt-4dd95f6943.test 3d0ce415d2ee15d3d564121960016b9c7be79407 @@ -1866,7 +1866,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cb0a18e64f8d81c2ada9f698faaf5ca68a0666687bf6f3abc860324cb1997463 -R 8462379efa542a8195db58fd9ef94a35 +P 3117c1b5a9e348fd8d16ba9d03fdafaad8514567fb3403f72b86d6162ad40bde +R e941513d71fae9ce3b558d98a564d148 U drh -Z 5f8f2ff4f8e169f83e77c91d9875e042 +Z db42a456dee4143dc7c71c6203f4a854 diff --git a/manifest.uuid b/manifest.uuid index c90a442804..c96b83728e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3117c1b5a9e348fd8d16ba9d03fdafaad8514567fb3403f72b86d6162ad40bde \ No newline at end of file +4218c7b71fb6b227dbe4b852718584c150164af2d84e067cb810aa602554a609 \ No newline at end of file diff --git a/src/utf.c b/src/utf.c index f6c8e79e50..3a2becfbcc 100644 --- a/src/utf.c +++ b/src/utf.c @@ -284,6 +284,7 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ c = *(zIn++); c += (*(zIn++))<<8; if( c>=0xd800 && c<0xe000 ){ +#ifdef SQLITE_REPLACE_INVALID_UTF if( c>=0xdc00 || zIn>=zTerm ){ c = 0xfffd; }else{ @@ -296,6 +297,13 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ c = ((c&0x3ff)<<10) + (c2&0x3ff) + 0x10000; } } +#else + if( zIn=0xd800 && c<0xe000 ){ +#ifdef SQLITE_REPLACE_INVALID_UTF if( c>=0xdc00 || zIn>=zTerm ){ c = 0xfffd; }else{ @@ -317,6 +326,13 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ c = ((c&0x3ff)<<10) + (c2&0x3ff) + 0x10000; } } +#else + if( zIn Date: Thu, 21 May 2020 19:13:46 +0000 Subject: [PATCH 6/8] Avoid another potential OOB read in sqlite3expert.c. FossilOrigin-Name: 0ccea80092f16e7f17f4c4de4f8be3fdef217199fcc08ace37e179c1b22c1294 --- ext/expert/sqlite3expert.c | 6 ++++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c index 0685e54222..b5a6fd2ab4 100644 --- a/ext/expert/sqlite3expert.c +++ b/ext/expert/sqlite3expert.c @@ -1136,9 +1136,11 @@ int idxFindIndexes( for(i=0; i Date: Thu, 21 May 2020 20:38:39 +0000 Subject: [PATCH 7/8] Change a NEVER macro into a NO_TEST comment, as the conditional is reachable, but only when compiling for Windows 32-bit. FossilOrigin-Name: ce36b6d1331edba5a921fef32553e2470a79bdb1f62d2cfd81190691c83d5b06 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/memdb.c | 9 +++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index ca56096d0d..cb328cbc7e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sanother\spotential\sOOB\sread\sin\ssqlite3expert.c. -D 2020-05-21T19:13:46.854 +C Change\sa\sNEVER\smacro\sinto\sa\sNO_TEST\scomment,\sas\sthe\sconditional\sis\sreachable,\nbut\sonly\swhen\scompiling\sfor\sWindows\s32-bit. +D 2020-05-21T20:38:39.168 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -504,7 +504,7 @@ F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 -F src/memdb.c 1d05bcc3fb6bde5c4aab7a56279d232f204625d74e627121e0562f355f5c843e +F src/memdb.c 252137ca122acb8f54a99b48cf9f96a31d5130f19d174381a53294446d8b64a3 F src/memjournal.c 7561c01c90958f3ba9bc6cb2d857123d932bdfa5539ea34427a0957b2e35154d F src/msvc.h 3a15918220367a8876be3fa4f2abe423a861491e84b864fb2b7426bf022a28f8 F src/mutex.c 5e3409715552348732e97b9194abe92fdfcd934cfb681df4ba0ab87ac6c18d25 @@ -1866,7 +1866,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4218c7b71fb6b227dbe4b852718584c150164af2d84e067cb810aa602554a609 -R a34ce7e81c0a64aa29d603331470468c -U dan -Z d5344c700a7ec03fab9c05b6e6d6dc98 +P 0ccea80092f16e7f17f4c4de4f8be3fdef217199fcc08ace37e179c1b22c1294 +R b5a877f3e2a7ab538bc565ffd2aae787 +U drh +Z 2acfb479c943aedec5bae0c532948916 diff --git a/manifest.uuid b/manifest.uuid index ca563882f2..ffe54a24bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0ccea80092f16e7f17f4c4de4f8be3fdef217199fcc08ace37e179c1b22c1294 \ No newline at end of file +ce36b6d1331edba5a921fef32553e2470a79bdb1f62d2cfd81190691c83d5b06 \ No newline at end of file diff --git a/src/memdb.c b/src/memdb.c index aabac5226f..10da6d8018 100644 --- a/src/memdb.c +++ b/src/memdb.c @@ -613,10 +613,11 @@ int sqlite3MemdbInit(void){ sqlite3_vfs *pLower = sqlite3_vfs_find(0); int sz = pLower->szOsFile; memdb_vfs.pAppData = pLower; - /* In all known configurations of SQLite, the size of a default - ** sqlite3_file is greater than the size of a memdb sqlite3_file. - ** Should that ever change, remove the following NEVER() */ - if( NEVER(sz Date: Fri, 22 May 2020 17:46:16 +0000 Subject: [PATCH 8/8] Version 3.32.0 FossilOrigin-Name: 5998789c9c744bce92e4cff7636bba800a75574243d6977e1fc8281e360f8d5a --- manifest | 11 +++++++---- manifest.uuid | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/manifest b/manifest index cb328cbc7e..e020405f88 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sa\sNEVER\smacro\sinto\sa\sNO_TEST\scomment,\sas\sthe\sconditional\sis\sreachable,\nbut\sonly\swhen\scompiling\sfor\sWindows\s32-bit. -D 2020-05-21T20:38:39.168 +C Version\s3.32.0 +D 2020-05-22T17:46:16.912 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1866,7 +1866,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0ccea80092f16e7f17f4c4de4f8be3fdef217199fcc08ace37e179c1b22c1294 +P ce36b6d1331edba5a921fef32553e2470a79bdb1f62d2cfd81190691c83d5b06 R b5a877f3e2a7ab538bc565ffd2aae787 +T +bgcolor * #d0c0ff +T +sym-release * +T +sym-version-3.32.0 * U drh -Z 2acfb479c943aedec5bae0c532948916 +Z 4af13806b6e302c99aeddae94a54c013 diff --git a/manifest.uuid b/manifest.uuid index ffe54a24bf..d9fa627cdb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ce36b6d1331edba5a921fef32553e2470a79bdb1f62d2cfd81190691c83d5b06 \ No newline at end of file +5998789c9c744bce92e4cff7636bba800a75574243d6977e1fc8281e360f8d5a \ No newline at end of file