Merge latest trunk with this branch.

FossilOrigin-Name: 606b1ead24eb5293d19542105bf2db64c9b1a76840a90a8f2c982146d76c007e

manifest

@@ -1,5 +1,5 @@
-C Change\sthe\sway\sa\scomparison\sused\sto\sdetect\scorrupt\sdatabases\sin\sfts3\sis\sdone\nto\savoid\spotential\spointer\soverflow\sin\s32-bit\sbuilds.
-D 2018-12-24T13:34:36.984
+C Merge\slatest\strunk\swith\sthis\sbranch.
+D 2018-12-24T15:15:21.032
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in d8b254f8bb81bab43c340d70d17dc3babab40fcc8a348c8255881f780a45fee6
@@ -494,7 +494,7 @@ F src/os.c 8aeb0b0f40f8f5b0da03fe49706695adaf42d2f516ab95abc72e86c245e119de
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c f6e91b8fd82af7afbfd073c4974ad6cdb8e62d9f65ceddb45167835a0567fdc0
+F src/os_unix.c 3de3133f268e98596a0b4ae81734539f0b01dd5e1a399834eb52bc4c9dcc66ae
 F src/os_win.c 85d9e532d0444ab6c16d7431490c2e279e282aa0917b0e988996b1ae0de5c5a0
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 75e0f3cfa3962c714f519f8a3d1e67ecca1c91de0e010a036b988e40ce9e4c73
@@ -561,12 +561,12 @@ F src/test_rtree.c 671f3fae50ff116ef2e32a3bf1fe21b5615b4b7b
 F src/test_schema.c f575932cb6274d12147a77e13ea4b49d52408513
 F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe
 F src/test_sqllog.c 11e6ce7575f489155c604ac4b439f2ac1d3d5aef
-F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e
+F src/test_superlock.c f4d4cc7319a608a54b7608158e8c7135fac19b88d6179e5bf17080e89d1f0278
 F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939
 F src/test_tclsh.c 06317648b0d85a85fd823f7973b55535c59a3156c1ef59394fe511f932cfa78d
 F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc
 F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858
-F src/test_vfs.c 4e31a7b98a401431c55590a0ac05f001f520f716d0707ec916b4022b8937fd06
+F src/test_vfs.c 9a53b40290db75833db58295485b955276daed99a2e57d7215719e175c0cfae0
 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698
 F src/test_windirent.c a895e2c068a06644eef91a7f0a32182445a893b9a0f33d0cdb4283dca2486ac1
 F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a90484215
@@ -592,7 +592,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7
 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392
 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 3f4f653daf234fe713edbcbca3fec2350417d159d28801feabc702a22c4e213f
+F src/wal.c d1c4f4edd172d4b9320d785304bad3efc433baf672621653017434ab87f3f336
 F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a
 F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66
 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e
@@ -1084,7 +1084,7 @@ F test/lock4.test 27143363eda1622f03c133efc8db808fc331afd973486cb571ea71cd717d37
 F test/lock5.test c6c5e0ebcb21c61a572870cc86c0cb9f14cede38
 F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5
 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431
-F test/lock_common.tcl 7ffb45accf6ee91c736df9bafe0806a44358f035
+F test/lock_common.tcl 2f3f7f2e9637f93ccf609df48ef5b27a50278b6b1cd752b445d52262e5841413
 F test/lookaside.test 5a828e7256f1ee4da8e1bdaa03373a3ccdb0f1ff98dfa82e9b76cb41a45b1083
 F test/main.test 6bbb3999fd461eb8fb335cbab97409a3d7f91bbb8da60635e8be3e4a04a77772
 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9
@@ -1187,7 +1187,7 @@ F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035c
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test 1f244543fbf05f3ef9a4596d7e7fe5d832b6d973c53a87189515e3c15db562a5
+F test/permutations.test 49b876f77f09f41cbd3b356935e946c26cbdd259b7dc9f4411529e75f966f277
 F test/pg_common.tcl 301ac19c1a52fd55166d26db929b3b89165c634d52b5f8ad76ea8cb06960db30
 F test/pragma.test c267bf02742c823a191960895b3d52933cebd7beee26757d1ed694f213fcd867
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
@@ -1291,6 +1291,7 @@ F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35
 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
 F test/shell8.test 96be02ea0c21f05b24c1883d7b711a1fa8525a68ab7b636aacf6057876941013
+F test/shmlock.test 3d1868f0386923c0592a235f2dd87ae52286b217bc695fbfd9d39a828e7be374
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1599,7 +1600,7 @@ F test/vtabJ.test d7b73675708cf63cfcb9d443bb451fc01a028347275b7311e51f9fdf3ca675
 F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c3784c6783
 F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bda2b65
 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad
-F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477
+F test/wal.test cdf0ca6cc0447520d19ef1c83287824ebeb3e82d75af856511ba96841a79fc9b
 F test/wal2.test 155b9efa999bdb38ce1cd729b9a4fcdbffd6b88be27f039bad1d2929d287d918
 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
@@ -1792,7 +1793,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2b6494b1509f0d0189f98aa34c990eee99c775ff57826e79b2c5b0a12b4c97ad
-R a02ab00644ca16cf9661dd1e6ee9ef5e
+P b9a741519055a8560ec1d0ab7d0c15ff8db53e740011825506cd2ede11c956f0 95a9a39ff784b960b62dd6298e722a620ba3f9c8b76839a49bbef35d7bc84a8e
+R 5ea7d8dcd45bcaea5cd65eff0166a924
 U dan
-Z a6822b20c230c488499f3dd180c02a24
+Z 2093b49ffe85aadf6485d43886cf919f

manifest.uuid

@@ -1 +1 @@
-95a9a39ff784b960b62dd6298e722a620ba3f9c8b76839a49bbef35d7bc84a8e
+606b1ead24eb5293d19542105bf2db64c9b1a76840a90a8f2c982146d76c007e

src/os_unix.c

@@ -46,6 +46,16 @@
 #include "sqliteInt.h"
 #if SQLITE_OS_UNIX              /* This file is used on unix only */
 
+/* Turn this feature on in all builds for now */
+#define SQLITE_MUTEXFREE_SHMLOCK 1
+#define SQLITE_MFS_EXCLUSIVE     255
+#ifndef SQLITE_MFS_NSHARD
+# define SQLITE_MFS_NSHARD       8
+#endif
+#if SQLITE_MFS_NSHARD<1
+# error "SQLITE_MFS_NSHARD must be greater than 0"
+#endif
+
 /*
 ** There are various methods for file locking used for concurrency
 ** control:
@@ -4239,8 +4249,41 @@ struct unixShmNode {
   u8 sharedMask;             /* Mask of shared locks held */
   u8 nextShmId;              /* Next available unixShm.id value */
 #endif
+
+#ifdef SQLITE_MUTEXFREE_SHMLOCK
+  /* In unix-excl mode, if SQLITE_MUTEXFREE_SHMLOCK is defined, all locks
+  ** are stored in the following 64-bit value. There are in total 8 
+  ** shm-locking slots, each of which are assigned 8-bits from the 64-bit
+  ** value. The least-significant 8 bits correspond to shm-locking slot
+  ** 0, and so on.
+  **
+  ** If the 8-bits corresponding to a shm-locking locking slot are set to
+  ** 0xFF, then a write-lock is held on the slot. Or, if they are set to
+  ** a non-zero value smaller than 0xFF, then they represent the total 
+  ** number of read-locks held on the slot. There is no way to distinguish
+  ** between a write-lock and 255 read-locks.  */
+  struct LockingSlot {
+    u32 nLock;
+    u64 aPadding[7];
+  } aMFSlot[3 + SQLITE_MFS_NSHARD*5];
+#endif
 };
 
+/*
+** Atomic CAS primitive used in multi-process mode. Equivalent to:
+** 
+**   int unixCompareAndSwap(u32 *ptr, u32 oldval, u32 newval){
+**     if( *ptr==oldval ){
+**       *ptr = newval;
+**       return 1;
+**     }
+**     return 0;
+**   }
+*/
+#define unixCompareAndSwap(ptr,oldval,newval) \
+    __sync_bool_compare_and_swap(ptr,oldval,newval)
+
+
 /*
 ** Structure used internally by this VFS to record the state of an
 ** open shared memory connection.
@@ -4261,6 +4304,9 @@ struct unixShm {
   u8 id;                     /* Id of this connection within its unixShmNode */
   u16 sharedMask;            /* Mask of shared locks held */
   u16 exclMask;              /* Mask of exclusive locks held */
+#ifdef SQLITE_MUTEXFREE_SHMLOCK
+  u8 aMFCurrent[8];          /* Current slot used for each shared lock */
+#endif
 };
 
 /*
@@ -4765,6 +4811,79 @@ shmpage_out:
   return rc;
 }
 
+#ifdef SQLITE_MUTEXFREE_SHMLOCK
+static int unixMutexFreeShmlock(
+  unixFile *pFd,             /* Database file holding the shared memory */
+  int ofst,                  /* First lock to acquire or release */
+  int n,                     /* Number of locks to acquire or release */
+  int flags                  /* What to do with the lock */
+){
+  struct LockMapEntry {
+    int iFirst;
+    int nSlot;
+  } aMap[9] = {
+    { 0, 1 },
+    { 1, 1 },
+    { 2, 1 },
+    { 3+0*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+1*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+2*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+3*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+4*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+5*SQLITE_MFS_NSHARD, 0 },
+  };
+
+  unixShm *p = pFd->pShm;               /* The shared memory being locked */
+  unixShmNode *pShmNode = p->pShmNode;  /* The underlying file iNode */
+
+  if( flags & SQLITE_SHM_SHARED ){
+    /* SHARED locks */
+    u32 iOld, iNew, *ptr;
+    int iIncr = -1;
+    if( (flags & SQLITE_SHM_UNLOCK)==0 ){
+      p->aMFCurrent[ofst] = (p->aMFCurrent[ofst] + 1) % aMap[ofst].nSlot;
+      iIncr = 1;
+    }
+    ptr = &pShmNode->aMFSlot[aMap[ofst].iFirst + p->aMFCurrent[ofst]].nLock;
+    do {
+      iOld = *ptr;
+      iNew = iOld + iIncr;
+      if( iNew>SQLITE_MFS_EXCLUSIVE ){
+        return SQLITE_BUSY;
+      }
+    }while( 0==unixCompareAndSwap(ptr, iOld, iNew) );
+  }else{
+    /* EXCLUSIVE locks */
+    int iFirst = aMap[ofst].iFirst;
+    int iLast = aMap[ofst+n].iFirst;
+    int i;
+    for(i=iFirst; i<iLast; i++){
+      u32 *ptr = &pShmNode->aMFSlot[i].nLock;
+      if( flags & SQLITE_SHM_UNLOCK ){
+        assert( (*ptr)==SQLITE_MFS_EXCLUSIVE );
+        *ptr = 0;
+      }else{
+        u32 iOld;
+        do {
+          iOld = *ptr;
+          if( iOld>0 ){
+            while( i>iFirst ){
+              i--;
+              pShmNode->aMFSlot[i].nLock = 0;
+            }
+            return SQLITE_BUSY;
+          }
+        }while( 0==unixCompareAndSwap(ptr, iOld, SQLITE_MFS_EXCLUSIVE) );
+      }
+    }
+  }
+
+  return SQLITE_OK;
+}
+#else
+# define unixMutexFreeShmlock(a,b,c,d) SQLITE_OK
+#endif
+
 /*
 ** Change the lock state for a shared-memory segment.
 **
@@ -4798,8 +4917,20 @@ static int unixShmLock(
   assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 );
   assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 );
 
+  if( pDbFd->pInode->bProcessLock ){
+    return unixMutexFreeShmlock(pDbFd, ofst, n, flags);
+  }
+
   mask = (1<<(ofst+n)) - (1<<ofst);
   assert( n>1 || mask==(1<<ofst) );
+  if( flags & SQLITE_SHM_LOCK ){
+    assert( !(flags&SQLITE_SHM_SHARED) || (p->sharedMask&mask)==0 );
+    assert( !(flags&SQLITE_SHM_EXCLUSIVE) || !(p->exclMask&mask) );
+  }else{
+    assert( !(flags&SQLITE_SHM_SHARED) || (p->sharedMask&mask)==mask );
+    assert( !(flags&SQLITE_SHM_EXCLUSIVE) || (p->exclMask&mask)==mask );
+  }
+
   sqlite3_mutex_enter(pShmNode->pShmMutex);
   if( flags & SQLITE_SHM_UNLOCK ){
     u16 allMask = 0; /* Mask of locks held by siblings */
@@ -4889,12 +5020,16 @@ static void unixShmBarrier(
   sqlite3_file *fd                /* Database file holding the shared memory */
 ){
   UNUSED_PARAMETER(fd);
+#ifdef SQLITE_MUTEXFREE_SHMLOCK
+  __sync_synchronize();
+#else
   sqlite3MemoryBarrier();         /* compiler-defined memory barrier */
   assert( fd->pMethods->xLock==nolockLock 
        || unixFileMutexNotheld((unixFile*)fd) 
   );
   unixEnterMutex();               /* Also mutex, for redundancy */
   unixLeaveMutex();
+#endif
 }
 
 /*

src/test_superlock.c

@@ -41,6 +41,8 @@ typedef struct SuperlockBusy SuperlockBusy;
 struct Superlock {
   sqlite3 *db;                    /* Database handle used to lock db */
   int bWal;                       /* True if db is a WAL database */
+  int bRecoveryLocked;            /* True if WAL RECOVERY lock is held */
+  int bReaderLocked;              /* True if WAL READER locks are held */
 };
 typedef struct Superlock Superlock;
 
@@ -107,12 +109,13 @@ static int superlockShmLock(
 ** Invoke the supplied busy-handler as required.
 */
 static int superlockWalLock(
-  sqlite3 *db,                    /* Database handle open on WAL database */
+  Superlock *pLock,               /* Superlock handle */
   SuperlockBusy *pBusy            /* Busy handler wrapper object */
 ){
   int rc;                         /* Return code */
   sqlite3_file *fd = 0;           /* Main database file handle */
   void volatile *p = 0;           /* Pointer to first page of shared memory */
+  sqlite3 *db = pLock->db;
 
   /* Obtain a pointer to the sqlite3_file object open on the main db file. */
   rc = sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, (void *)&fd);
@@ -121,8 +124,10 @@ static int superlockWalLock(
   /* Obtain the "recovery" lock. Normally, this lock is only obtained by
   ** clients running database recovery.  
   */
+  assert( pLock->bRecoveryLocked==0 );
   rc = superlockShmLock(fd, 2, 1, pBusy);
   if( rc!=SQLITE_OK ) return rc;
+  pLock->bRecoveryLocked = 1;
 
   /* Zero the start of the first shared-memory page. This means that any
   ** clients that open read or write transactions from this point on will
@@ -139,7 +144,9 @@ static int superlockWalLock(
   ** are held, it is guaranteed that there are no active reader, writer or 
   ** checkpointer clients.
   */
+  assert( pLock->bReaderLocked==0 );
   rc = superlockShmLock(fd, 3, SQLITE_SHM_NLOCK-3, pBusy);
+  if( rc==SQLITE_OK ) pLock->bReaderLocked = 1;
   return rc;
 }
 
@@ -156,8 +163,14 @@ void sqlite3demo_superunlock(void *pLock){
     sqlite3_file *fd = 0;
     rc = sqlite3_file_control(p->db, "main", SQLITE_FCNTL_FILE_POINTER, (void *)&fd);
     if( rc==SQLITE_OK ){
-      fd->pMethods->xShmLock(fd, 2, 1, flags);
-      fd->pMethods->xShmLock(fd, 3, SQLITE_SHM_NLOCK-3, flags);
+      if( p->bRecoveryLocked ){
+        fd->pMethods->xShmLock(fd, 2, 1, flags);
+        p->bRecoveryLocked = 0;
+      }
+      if( p->bReaderLocked ){
+        fd->pMethods->xShmLock(fd, 3, SQLITE_SHM_NLOCK-3, flags);
+        p->bReaderLocked = 0;
+      }
     }
   }
   sqlite3_close(p->db);
@@ -232,7 +245,7 @@ int sqlite3demo_superlock(
     if( SQLITE_OK==(rc = superlockIsWal(pLock)) && pLock->bWal ){
       rc = sqlite3_exec(pLock->db, "COMMIT", 0, 0, 0);
       if( rc==SQLITE_OK ){
-        rc = superlockWalLock(pLock->db, &busy);
+        rc = superlockWalLock(pLock, &busy);
       }
     }
   }

src/test_vfs.c

@@ -1564,8 +1564,62 @@ static int SQLITE_TCLAPI testvfs_cmd(
   return TCL_ERROR;
 }
 
+extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb);
+extern const char *sqlite3ErrName(int);
+
+/*
+** tclcmd: vfs_shmlock DB DBNAME (shared|exclusive) (lock|unlock) OFFSET N
+*/
+static int SQLITE_TCLAPI test_vfs_shmlock(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  const char *azArg1[] = {"shared", "exclusive", 0};
+  const char *azArg2[] = {"lock", "unlock", 0};
+  sqlite3 *db = 0;
+  int rc = SQLITE_OK;
+  const char *zDbname = 0;
+  int iArg1 = 0;
+  int iArg2 = 0;
+  int iOffset = 0;
+  int n = 0;
+  sqlite3_file *pFd;
+
+  if( objc!=7 ){
+    Tcl_WrongNumArgs(interp, 1, objv, 
+        "DB DBNAME (shared|exclusive) (lock|unlock) OFFSET N"
+    );
+    return TCL_ERROR;
+  }
+
+  zDbname = Tcl_GetString(objv[2]);
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) 
+   || Tcl_GetIndexFromObj(interp, objv[3], azArg1, "ARG", 0, &iArg1) 
+   || Tcl_GetIndexFromObj(interp, objv[4], azArg2, "ARG", 0, &iArg2) 
+   || Tcl_GetIntFromObj(interp, objv[5], &iOffset)
+   || Tcl_GetIntFromObj(interp, objv[6], &n)
+  ){
+    return TCL_ERROR;
+  }
+
+  sqlite3_file_control(db, zDbname, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd);
+  if( pFd==0 ){
+    return TCL_ERROR;
+  }
+  rc = pFd->pMethods->xShmLock(pFd, iOffset, n, 
+      (iArg1==0 ? SQLITE_SHM_SHARED : SQLITE_SHM_EXCLUSIVE)
+    | (iArg2==0 ? SQLITE_SHM_LOCK : SQLITE_SHM_UNLOCK)
+  );
+  Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+  return TCL_OK;
+}
+
+
 int Sqlitetestvfs_Init(Tcl_Interp *interp){
   Tcl_CreateObjCommand(interp, "testvfs", testvfs_cmd, 0, 0);
+  Tcl_CreateObjCommand(interp, "vfs_shmlock", test_vfs_shmlock, 0, 0);
   return TCL_OK;
 }
 

src/wal.c

@@ -2777,19 +2777,14 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){
       assert( pWal->readLock>0 || pWal->hdr.mxFrame==0 );
       assert( pInfo->aReadMark[pWal->readLock]<=pSnapshot->mxFrame );
 
-      /* It is possible that there is a checkpointer thread running 
-      ** concurrent with this code. If this is the case, it may be that the
-      ** checkpointer has already determined that it will checkpoint 
+      /* If it were possible for a checkpointer thread to run concurrent 
+      ** with this code, it would be a problem. In this case, it could be
+      ** that the checkpointer has already determined that it will checkpoint 
       ** snapshot X, where X is later in the wal file than pSnapshot, but 
       ** has not yet set the pInfo->nBackfillAttempted variable to indicate 
-      ** its intent. To avoid the race condition this leads to, ensure that
-      ** there is no checkpointer process by taking a shared CKPT lock 
-      ** before checking pInfo->nBackfillAttempted.  
-      **
-      ** TODO: Does the aReadMark[] lock prevent a checkpointer from doing
-      **       this already?
-      */
-      rc = walLockShared(pWal, WAL_CKPT_LOCK);
+      ** its intent. Fortunately this is not possible, as the call to
+      ** sqlite3WalSnapshotOpen() that sets pWal->pSnapshot also takes a
+      ** SHARED lock on the checkpointer slot.  */
 
       if( rc==SQLITE_OK ){
         /* Check that the wal file has not been wrapped. Assuming that it has
@@ -2808,8 +2803,6 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){
           rc = SQLITE_ERROR_SNAPSHOT;
         }
 
-        /* Release the shared CKPT lock obtained above. */
-        walUnlockShared(pWal, WAL_CKPT_LOCK);
         pWal->minFrame = 1;
       }
 

test/lock_common.tcl

@@ -15,18 +15,20 @@
 
 proc do_multiclient_test {varname script} {
 
-  foreach code [list {
+  foreach {tn code} [list 1 {
     if {[info exists ::G(valgrind)]} { db close ; continue }
     set ::code2_chan [launch_testfixture]
     set ::code3_chan [launch_testfixture]
     proc code2 {tcl} { testfixture $::code2_chan $tcl }
     proc code3 {tcl} { testfixture $::code3_chan $tcl }
-    set tn 1
-  } {
+  } 2 {
     proc code2 {tcl} { uplevel #0 $tcl }
     proc code3 {tcl} { uplevel #0 $tcl }
-    set tn 2
   }] {
+    # Do not run multi-process tests with the unix-excl VFS.
+    #
+    if {$tn==1 && [permutation]=="unix-excl"} continue
+
     faultsim_delete_and_reopen
 
     proc code1 {tcl} { uplevel #0 $tcl }

test/permutations.test

@@ -612,6 +612,16 @@ test_suite "onefile" -description {
   rollback.test  select1.test  select2.test  select3.test
 }
 
+# Run some tests using the "unix-excl" VFS.
+#
+test_suite "unix-excl" -description {
+  Run some tests using the "unix-excl" VFS
+} -initialize {
+  set ::G(perm:sqlite3_args) [list -vfs unix-excl]
+} -files {
+  shmlock.test
+}
+
 # Run some tests using UTF-16 databases.
 #
 test_suite "utf16" -description {

test/shmlock.test

@@ -0,0 +1,173 @@
+# 2018 December 6
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+set testprefix shmlock
+
+ifcapable !wal {finish_test ; return }
+
+sqlite3 db2 test.db
+sqlite3 db3 test.db
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 2);
+} {wal}
+do_test 1.1 { execsql { SELECT * FROM t1 } db2 } {1 2}
+do_test 1.2 { execsql { SELECT * FROM t1 } db3 } {1 2}
+
+foreach {tn dbhandle cmd res} {
+  1    db  {shared    lock   7 1}    OK
+  2    db2 {exclusive lock   7 1}    BUSY
+  3    db  {shared    unlock 7 1}    OK
+  4    db2 {exclusive lock   7 1}    OK
+  5    db  {shared    lock   7 1}    BUSY
+  6    db  {exclusive lock   7 1}    BUSY
+  7    db2 {exclusive unlock 7 1}    OK
+
+  8    db  {exclusive lock   0 8}    OK
+  9    db  {exclusive unlock 0 8}    OK
+  10   db2 {exclusive lock   0 8}    OK
+  11   db2 {exclusive unlock 0 8}    OK
+
+  12   db  {shared    lock   0 1}    OK
+  13   db2 {shared    lock   0 1}    OK
+  14   db3 {shared    lock   0 1}    OK
+  15   db3 {shared    unlock 0 1}    OK
+  16   db3 {exclusive lock   0 1}    BUSY
+  17   db2 {shared    unlock 0 1}    OK
+  18   db3 {exclusive lock   0 1}    BUSY
+  19   db  {shared    unlock 0 1}    OK
+  20   db3 {exclusive lock   0 1}    OK
+  21   db3 {exclusive unlock 0 1}    OK
+
+  22   db  {shared    lock   3 1}    OK
+  23   db2 {exclusive lock   2 2}    BUSY
+  24   db  {shared    lock   2 1}    OK
+  25   db2 {exclusive lock   0 5}    BUSY
+  26   db2 {exclusive lock   0 4}    BUSY
+  27   db2 {exclusive lock   0 3}    BUSY
+  28   db  {shared    unlock 3 1}    OK
+  29   db2 {exclusive lock   2 2}    BUSY
+  28   db  {shared    unlock 2 1}    OK
+  29   db2 {exclusive lock   2 2}    OK
+  29   db2 {exclusive unlock 2 2}    OK
+} {
+  do_test 1.3.$tn [list vfs_shmlock $dbhandle main {*}$cmd] "SQLITE_$res"
+}
+
+db  close
+db2 close
+db3 close
+
+if {[permutation]=="unix-excl"} {
+  do_test 2.0 {
+    for {set i 0} {$i < 256} {incr i} { 
+      sqlite3 db$i test.db 
+      execsql { SELECT * FROM t1 } db$i
+    }
+    for {set i 0} {$i < 255} {incr i} { 
+      set rc [vfs_shmlock db$i main shared lock 4 1]
+      if {$rc != "SQLITE_OK"} { error $rc }
+    }
+
+    vfs_shmlock db255 main shared lock 4 1
+  } {SQLITE_BUSY}
+
+  do_test 2.1 { vfs_shmlock db255 main exclusive lock   4 1 } SQLITE_BUSY
+  do_test 2.2 { vfs_shmlock db0   main shared    unlock 4 1 } SQLITE_OK
+  do_test 2.3 { vfs_shmlock db255 main shared    lock   4 1 } SQLITE_OK
+  do_test 2.4 { vfs_shmlock db255 main shared    unlock 4 1 } SQLITE_OK
+  do_test 2.5 { vfs_shmlock db255 main exclusive lock   4 1 } SQLITE_BUSY
+
+  do_test 2.6 {
+    for {set i 1} {$i < 255} {incr i} { 
+      set rc [vfs_shmlock db255 main exclusive lock 4 1]
+      if {$rc != "SQLITE_BUSY"} { error $rc }
+      set rc [vfs_shmlock db$i main shared unlock 4 1]
+      if {$rc != "SQLITE_OK"} { error $rc }
+    }
+
+    vfs_shmlock db255 main exclusive lock 4 1
+  } {SQLITE_OK}
+
+  vfs_shmlock db255 main exclusive unlock 4 1
+
+  for {set i 0} {$i < 256} {incr i} {
+    db$i close
+  }
+}
+
+sqlite3 db0 test.db
+sqlite3 db1 test.db
+do_test 3.1 { execsql { SELECT * FROM t1 } db0 } {1 2}
+do_test 3.2 { execsql { SELECT * FROM t1 } db1 } {1 2}
+
+set L(0) {n n n n n n n n}
+set L(1) {n n n n n n n n}
+proc random_lock_test {idx} {
+  global L
+  set iSlot [expr int(rand()*8)]
+  if {[expr int(rand()*2)]} {
+    # Unlock operation
+    if {[lindex $L($idx) $iSlot]!="n"} {
+      vfs_shmlock db$idx main [lindex $L($idx) $iSlot] unlock $iSlot 1
+      lset L($idx) $iSlot n
+    }
+  } else {
+    # Lock operation
+    if {[lindex $L($idx) $iSlot]=="n"} {
+      set locktype [lindex {e s} [expr int(rand()*2)]]
+      set n 1
+      if {$locktype=="e"} {
+        for {set l $iSlot} {$l<8 && [lindex $L($idx) $l]=="n"} {incr l} {}
+        set n [expr int(rand()*($l-$iSlot))+1]
+        # puts "iSlot=$iSlot l=$l L=$L($idx)"
+        # puts "$iSlot $n"
+      }
+      set res [vfs_shmlock db$idx main $locktype lock $iSlot $n]
+
+      set bBusy 0
+      for {set i $iSlot} {$i<($iSlot+$n)} {incr i} {
+        set other [lindex $L([expr ($idx+1)%2]) $i]
+        if {($other!="n" && $locktype=="e")||($other=="e" && $locktype=="s")} {
+          if {$res != "SQLITE_BUSY"} { error "BUSY not detected" }
+          set bBusy 1
+          break
+        } 
+      }
+
+      if {$bBusy==0} {
+        if {$res != "SQLITE_OK"} { error "BUSY false-positive" }
+        for {set i $iSlot} {$i<($iSlot+$n)} {incr i} {
+          lset L($idx) $i $locktype
+        }
+      }
+    }
+  }
+}
+
+set nStep 100000
+for {set i 0} {$i < $nStep} {incr i} {
+  random_lock_test 0
+  random_lock_test 1
+}
+
+db0 close
+db1 close
+
+finish_test
+
+

test/wal.test

@@ -1297,51 +1297,53 @@ do_test wal-19.4 {
 # At one point, SQLite was failing to grow the mapping of the wal-index
 # file in step 3 and the checkpoint was corrupting the database file.
 #
-do_test wal-20.1 {
-  catch {db close}
-  forcedelete test.db test.db-wal test.db-journal
-  sqlite3 db test.db
-  execsql {
-    PRAGMA journal_mode = WAL;
-    CREATE TABLE t1(x);
-    INSERT INTO t1 VALUES(randomblob(900));
-    SELECT count(*) FROM t1;
-  }
-} {wal 1}
-do_test wal-20.2 {
-  set ::buddy [launch_testfixture]
-  testfixture $::buddy {
+if {[permutation]!="unix-excl"} {
+  do_test wal-20.1 {
+    catch {db close}
+    forcedelete test.db test.db-wal test.db-journal
     sqlite3 db test.db
-    db transaction { db eval {
-      PRAGMA wal_autocheckpoint = 0;
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 2 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 4 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 8 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 16 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 32 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 64 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 128 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 256 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 512 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 1024 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 2048 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 4096 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 8192 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 16384 */
-    } }
-  }
-} {0}
-do_test wal-20.3 {
-  close $::buddy
-  execsql { PRAGMA wal_checkpoint }
-  execsql { SELECT count(*) FROM t1 }
-} {16384}
-do_test wal-20.4 {
-  db close
-  sqlite3 db test.db
-  execsql { SELECT count(*) FROM t1 }
-} {16384}
-integrity_check wal-20.5
+    execsql {
+      PRAGMA journal_mode = WAL;
+      CREATE TABLE t1(x);
+      INSERT INTO t1 VALUES(randomblob(900));
+      SELECT count(*) FROM t1;
+    }
+  } {wal 1}
+  do_test wal-20.2 {
+    set ::buddy [launch_testfixture]
+    testfixture $::buddy {
+      sqlite3 db test.db
+      db transaction { db eval {
+        PRAGMA wal_autocheckpoint = 0;
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 2 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 4 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 8 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 16 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 32 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 64 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 128 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 256 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 512 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 1024 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 2048 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 4096 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 8192 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 16384 */
+      } }
+    }
+  } {0}
+  do_test wal-20.3 {
+    close $::buddy
+    execsql { PRAGMA wal_checkpoint }
+    execsql { SELECT count(*) FROM t1 }
+  } {16384}
+  do_test wal-20.4 {
+    db close
+    sqlite3 db test.db
+    execsql { SELECT count(*) FROM t1 }
+  } {16384}
+  integrity_check wal-20.5
+}
 
 catch { db2 close }
 catch { db close }