diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c index 0685e54222..b5a6fd2ab4 100644 --- a/ext/expert/sqlite3expert.c +++ b/ext/expert/sqlite3expert.c @@ -1136,9 +1136,11 @@ int idxFindIndexes( for(i=0; iszOsFile; memdb_vfs.pAppData = pLower; - /* In all known configurations of SQLite, the size of a default - ** sqlite3_file is greater than the size of a memdb sqlite3_file. - ** Should that ever change, remove the following NEVER() */ - if( NEVER(szctrlFlags |= WINFILE_RDONLY; } - if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ + if( (flags & SQLITE_OPEN_MAIN_DB) + && sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) + ){ pFile->ctrlFlags |= WINFILE_PSOW; } pFile->lastErrno = NO_ERROR; diff --git a/src/utf.c b/src/utf.c index f6c8e79e50..3a2becfbcc 100644 --- a/src/utf.c +++ b/src/utf.c @@ -284,6 +284,7 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ c = *(zIn++); c += (*(zIn++))<<8; if( c>=0xd800 && c<0xe000 ){ +#ifdef SQLITE_REPLACE_INVALID_UTF if( c>=0xdc00 || zIn>=zTerm ){ c = 0xfffd; }else{ @@ -296,6 +297,13 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ c = ((c&0x3ff)<<10) + (c2&0x3ff) + 0x10000; } } +#else + if( zIn=0xd800 && c<0xe000 ){ +#ifdef SQLITE_REPLACE_INVALID_UTF if( c>=0xdc00 || zIn>=zTerm ){ c = 0xfffd; }else{ @@ -317,6 +326,13 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ c = ((c&0x3ff)<<10) + (c2&0x3ff) + 0x10000; } } +#else + if( zInexclusiveMode!=WAL_HEAPMEMORY_MODE ){ sqlite3OsShmBarrier(pWal->pDbFd); } } +/* +** Add the SQLITE_NO_TSAN as part of the return-type of a function +** definition as a hint that the function contains constructs that +** might give false-positive TSAN warnings. +** +** See tag-20200519-1. +*/ +#if defined(__clang__) && !defined(SQLITE_NO_TSAN) +# define SQLITE_NO_TSAN __attribute__((no_sanitize_thread)) +#else +# define SQLITE_NO_TSAN +#endif + /* ** Write the header information in pWal->hdr into the wal-index. ** ** The checksum on pWal->hdr is updated before it is written. */ -static void walIndexWriteHdr(Wal *pWal){ +static SQLITE_NO_TSAN void walIndexWriteHdr(Wal *pWal){ volatile WalIndexHdr *aHdr = walIndexHdr(pWal); const int nCksum = offsetof(WalIndexHdr, aCksum); @@ -708,6 +725,7 @@ static void walIndexWriteHdr(Wal *pWal){ pWal->hdr.isInit = 1; pWal->hdr.iVersion = WALINDEX_MAX_VERSION; walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); + /* Possible TSAN false-positive. See tag-20200519-1 */ memcpy((void*)&aHdr[1], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); walShmBarrier(pWal); memcpy((void*)&aHdr[0], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); @@ -1897,32 +1915,13 @@ static int walCheckpoint( mxSafeFrame = pWal->hdr.mxFrame; mxPage = pWal->hdr.nPage; for(i=1; iaReadMark[i]; + u32 y = AtomicLoad(pInfo->aReadMark+i); if( mxSafeFrame>y ){ assert( y<=pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); if( rc==SQLITE_OK ){ - pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED); + u32 iMark = (i==1 ? mxSafeFrame : READMARK_NOT_USED); + AtomicStore(pInfo->aReadMark+i, iMark); walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); }else if( rc==SQLITE_BUSY ){ mxSafeFrame = y; @@ -1940,7 +1939,7 @@ static int walCheckpoint( } if( pIter - && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0),1))==SQLITE_OK + && (rc = walBusyLock(pWal,xBusy,pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK ){ u32 nBackfill = pInfo->nBackfill; @@ -2155,7 +2154,7 @@ int sqlite3WalClose( ** If the checksum cannot be verified return non-zero. If the header ** is read successfully and the checksum verified, return zero. */ -static int walIndexTryHdr(Wal *pWal, int *pChanged){ +static SQLITE_NO_TSAN int walIndexTryHdr(Wal *pWal, int *pChanged){ u32 aCksum[2]; /* Checksum on the header content */ WalIndexHdr h1, h2; /* Two copies of the header content */ WalIndexHdr volatile *aHdr; /* Header in shared memory */ @@ -2168,13 +2167,19 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ ** meaning it is possible that an inconsistent snapshot is read ** from the file. If this happens, return non-zero. ** + ** tag-20200519-1: ** There are two copies of the header at the beginning of the wal-index. ** When reading, read [0] first then [1]. Writes are in the reverse order. ** Memory barriers are used to prevent the compiler or the hardware from - ** reordering the reads and writes. + ** reordering the reads and writes. TSAN and similar tools can sometimes + ** give false-positive warnings about these accesses because the tools do not + ** account for the double-read and the memory barrier. The use of mutexes + ** here would be problematic as the memory being accessed is potentially + ** shared among multiple processes and not all mutex implementions work + ** reliably in that environment. */ aHdr = walIndexHdr(pWal); - memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); + memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); /* Possible TSAN false-positive */ walShmBarrier(pWal); memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); @@ -3015,14 +3020,15 @@ int sqlite3WalFindFrame( int iKey; /* Hash slot index */ int nCollide; /* Number of hash collisions remaining */ int rc; /* Error code */ + u32 iH; rc = walHashGet(pWal, iHash, &sLoc); if( rc!=SQLITE_OK ){ return rc; } nCollide = HASHTABLE_NSLOT; - for(iKey=walHash(pgno); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ - u32 iH = sLoc.aHash[iKey]; + iKey = walHash(pgno); + while( (iH = AtomicLoad(&sLoc.aHash[iKey]))!=0 ){ u32 iFrame = iH + sLoc.iZero; if( iFrame<=iLast && iFrame>=pWal->minFrame && sLoc.aPgno[iH]==pgno ){ assert( iFrame>iRead || CORRUPT_DB ); @@ -3031,6 +3037,7 @@ int sqlite3WalFindFrame( if( (nCollide--)==0 ){ return SQLITE_CORRUPT_BKPT; } + iKey = walNextHash(iKey); } if( iRead ) break; } diff --git a/test/tkt-3fe897352e.test b/test/tkt-3fe897352e.test index fa70ac7658..bc2b2033e7 100644 --- a/test/tkt-3fe897352e.test +++ b/test/tkt-3fe897352e.test @@ -33,28 +33,28 @@ do_test tkt-3fe89-1.1 { INSERT INTO t1 VALUES(hex_to_utf16be('D800')); SELECT hex(x) FROM t1; } -} {EFBFBD} +} {EDA080} do_test tkt-3fe89-1.2 { db eval { DELETE FROM t1; INSERT INTO t1 VALUES(hex_to_utf16le('00D8')); SELECT hex(x) FROM t1; } -} {EFBFBD} +} {EDA080} do_test tkt-3fe89-1.3 { db eval { DELETE FROM t1; INSERT INTO t1 VALUES(hex_to_utf16be('DFFF')); SELECT hex(x) FROM t1; } -} {EFBFBD} +} {EDBFBF} do_test tkt-3fe89-1.4 { db eval { DELETE FROM t1; INSERT INTO t1 VALUES(hex_to_utf16le('FFDF')); SELECT hex(x) FROM t1; } -} {EFBFBD} +} {EDBFBF} finish_test