Fix some comments in new code on this branch. No changes to code.

FossilOrigin-Name: cbec1bfe4b9f0e9fc9849db623694902308ecc05f3898368201a9902c39a138f

manifest

@@ -1,5 +1,5 @@
-C The\sextensions\sfunctions\sin\sspellfix\sare\sall\sdeterministic.
-D 2017-11-03T08:46:48.968
+C Fix\ssome\scomments\sin\snew\scode\son\sthis\sbranch.\sNo\schanges\sto\scode.
+D 2017-11-03T19:34:17.982
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -454,7 +454,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 7edc872747feaa3016bd04e5e4389743bacafc0fee3444b0ecdec5d8f45049df
+F src/os_unix.c 7cc36e2b1c3bc45cab3f4af4b9ef498a5ba14c60edc71bdce8c712ce78abdc24
 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
@@ -472,7 +472,7 @@ F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee
 F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a
-F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
+F src/sqlite.h.in fac062a139e95b9ea2074518e4de1edceee4b5ff8d3befb2dc51634dc0475b2a
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
 F src/sqliteInt.h f5377febf86654c975e1d4e4353a5ad2fbaa5bc86b584ba3761ed33e24ce2c0e
@@ -550,7 +550,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c cc9b1120f1955b66af425630c9893acd537a39d967fd39d404417f0a1b4c1579
+F src/wal.c 298acb2281a73fa15971f40b2283e49c317ef1ab08d1e53c6b5956ee9140152d
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
@@ -1507,7 +1507,7 @@ F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c37
 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad
 F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477
-F test/wal2.test 6ac39b94a284ebac6efb6be93b0cdfe73ee6083f129555e3144d8a615e9900ef
+F test/wal2.test 4c44bbe447959638e5163631a1fe95c9dbc01a06eff6eb34449be06b6e0ed64c
 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
 F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9
@@ -1532,7 +1532,8 @@ F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496
 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20
-F test/walro.test 4ab7ac01b77c2f894235c699d59e3e3c7f15a160
+F test/walro.test 04f52c23ecd18acb493c9c12798d7ef1618c5f547558fe6767cca8f613591e62
+F test/walro2.test 216e952db319eec60b33579e4436cbe97319c3c7c4769ff1a907d1ccde241f66
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
@@ -1673,7 +1674,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d088c5a3f1a803118cb67560fc35c11178e1b7e54c0e1511677559906b9da980
-R b7838526febd368042ba32a4576c580c
-U drh
-Z 18ebbc1917110c35a4b2606f5ce6f533
+P 585e0a09c5e1cd3d32e7b3fe228bd3eb674631f4074f517aad4c2600818a337a
+R 07390855efc0141c7a26cb08f8119f74
+U dan
+Z 9e604d6a7024cca2a24e6e925cfd2e77

manifest.uuid

@@ -1 +1 @@
-29ec855e13e0dcd675dcf12948b42f9e669d0a31c5d9efb95857888aba0beeee
+cbec1bfe4b9f0e9fc9849db623694902308ecc05f3898368201a9902c39a138f

src/os_unix.c

@@ -3795,6 +3795,7 @@ static void unixModeBit(unixFile *pFile, unsigned char mask, int *pArg){
 
 /* Forward declaration */
 static int unixGetTempname(int nBuf, char *zBuf);
+static int fcntlReadShm(unixFile*,void*);
 
 /*
 ** Information and control of an open file handle.
@@ -3817,6 +3818,10 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){
     }
 #endif /* __linux__ && SQLITE_ENABLE_BATCH_ATOMIC_WRITE */
 
+    case SQLITE_FCNTL_READ_SHM: {
+      return fcntlReadShm(pFile, pArg);
+    }
+
     case SQLITE_FCNTL_LOCKSTATE: {
       *(int*)pArg = pFile->eFileLock;
       return SQLITE_OK;
@@ -4108,6 +4113,7 @@ struct unixShmNode {
   int szRegion;              /* Size of shared-memory regions */
   u16 nRegion;               /* Size of array apRegion */
   u8 isReadonly;             /* True if read-only */
+  u8 isUnlocked;             /* True if no DMS lock held */
   char **apRegion;           /* Array of mapped shared-memory regions */
   int nRef;                  /* Number of unixShm objects pointing to this */
   unixShm *pFirst;           /* All unixShm objects pointing to this */
@@ -4146,6 +4152,51 @@ struct unixShm {
 #define UNIX_SHM_BASE   ((22+SQLITE_SHM_NLOCK)*4)         /* first lock byte */
 #define UNIX_SHM_DMS    (UNIX_SHM_BASE+SQLITE_SHM_NLOCK)  /* deadman switch */
 
+/*
+** Implementation of file-control SQLITE_FCNTL_READ_SHM.
+*/
+static int fcntlReadShm(unixFile *pFile, void *pArg){
+  struct ReadShmParam {
+    void *pBuf;
+    int nBuf;
+  } *pParam = (struct ReadShmParam*)pArg;
+  unixShmNode *pShmNode = pFile->pShm->pShmNode;
+  int rc = SQLITE_OK;
+  sqlite3_mutex_enter( pShmNode->mutex );
+  if( pShmNode->isUnlocked==1 ){
+    struct stat buf;          /* Used to hold return values of fstat() */
+    if( osFstat(pShmNode->h, &buf) ){
+      rc = SQLITE_IOERR_FSTAT;
+    }else if( 0!=lseek(pShmNode->h, 0, SEEK_SET) ){
+      rc = SQLITE_IOERR_READ;
+    }else{
+      int nRead = buf.st_size;
+      if( nRead==0 ){
+        pParam->nBuf = 0;
+        pParam->pBuf = 0;
+      }else{    
+        pParam->pBuf = sqlite3_malloc(nRead);
+        if( pParam->pBuf ){
+          int res = osRead(pShmNode->h, pParam->pBuf, nRead);
+          if( res!=nRead ){
+            sqlite3_free(pParam->pBuf);
+            pParam->pBuf = 0;
+            rc = SQLITE_IOERR_READ;
+          }else{
+            pParam->nBuf = nRead;
+          }
+        }else{
+          rc = SQLITE_NOMEM_BKPT;
+        }
+      }
+    }
+  }else{
+    rc = SQLITE_BUSY;
+  }
+  sqlite3_mutex_leave( pShmNode->mutex );
+  return rc;
+}
+
 /*
 ** Apply posix advisory locks for all bytes from ofst through ofst+n-1.
 **
@@ -4270,6 +4321,64 @@ static void unixShmPurge(unixFile *pFd){
   }
 }
 
+/*
+** The DMS lock has not yet been taken on shm file pShmNode. Attempt to
+** take it now. Return SQLITE_OK if successful, or an SQLite error
+** code otherwise.
+**
+** If the DMS cannot be locked because this is a readonly_shm=1 
+** connection and no other process already holds a lock, return
+** SQLITE_READONLY_CANTLOCK and set pShmNode->isUnlocked=1.
+*/
+static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){
+  struct flock lock;
+  int rc = SQLITE_OK;
+
+  /* Use F_GETLK to determine the locks other processes are holding
+  ** on the DMS byte. If it indicates that another process is holding
+  ** a SHARED lock, then this process may also take a SHARED lock
+  ** and proceed with opening the *-shm file. 
+  **
+  ** Or, if no other process is holding any lock, then this process
+  ** is the first to open it. In this case take an EXCLUSIVE lock on the
+  ** DMS byte and truncate the *-shm file to zero bytes in size. Then
+  ** downgrade to a SHARED lock on the DMS byte.
+  **
+  ** If another process is holding an EXCLUSIVE lock on the DMS byte,
+  ** return SQLITE_BUSY to the caller (it will try again). An earlier
+  ** version of this code attempted the SHARED lock at this point. But
+  ** this introduced a subtle race condition: if the process holding
+  ** EXCLUSIVE failed just before truncating the *-shm file, then this
+  ** process might open and use the *-shm file without truncating it.
+  ** And if the *-shm file has been corrupted by a power failure or
+  ** system crash, the database itself may also become corrupt.  */
+  lock.l_whence = SEEK_SET;
+  lock.l_start = UNIX_SHM_DMS;
+  lock.l_len = 1;
+  lock.l_type = F_WRLCK;
+  if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) {
+    rc = SQLITE_IOERR_LOCK;
+  }else if( lock.l_type==F_UNLCK ){
+    if( pShmNode->isReadonly ){
+      pShmNode->isUnlocked = 1;
+      rc = SQLITE_READONLY_CANTLOCK;
+    }else{
+      rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1);
+      if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){
+        rc = unixLogError(SQLITE_IOERR_SHMOPEN,"ftruncate",pShmNode->zFilename);
+      }
+    }
+  }else if( lock.l_type==F_WRLCK ){
+    rc = SQLITE_BUSY;
+  }
+
+  if( rc==SQLITE_OK ){
+    assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK );
+    rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1);
+  }
+  return rc;
+}
+
 /*
 ** Open a shared-memory area associated with open database file pDbFd.  
 ** This particular implementation uses mmapped files.
@@ -4308,7 +4417,7 @@ static void unixShmPurge(unixFile *pFd){
 static int unixOpenSharedMemory(unixFile *pDbFd){
   struct unixShm *p = 0;          /* The connection to be opened */
   struct unixShmNode *pShmNode;   /* The underlying mmapped file */
-  int rc;                         /* Result code */
+  int rc = SQLITE_OK;             /* Result code */
   unixInodeInfo *pInode;          /* The inode of fd */
   char *zShmFilename;             /* Name of the file used for SHM */
   int nShmFilename;               /* Size of the SHM filename in bytes */
@@ -4388,20 +4497,9 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
       ** the original owner will not be able to connect.
       */
       robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid);
-  
-      /* Check to see if another process is holding the dead-man switch.
-      ** If not, truncate the file to zero length. 
-      */
-      rc = SQLITE_OK;
-      if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){
-        if( robust_ftruncate(pShmNode->h, 0) ){
-          rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename);
-        }
-      }
-      if( rc==SQLITE_OK ){
-        rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1);
-      }
-      if( rc ) goto shm_open_err;
+
+      rc = unixLockSharedMemory(pDbFd, pShmNode);
+      if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTLOCK ) goto shm_open_err;
     }
   }
 
@@ -4425,7 +4523,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
   p->pNext = pShmNode->pFirst;
   pShmNode->pFirst = p;
   sqlite3_mutex_leave(pShmNode->mutex);
-  return SQLITE_OK;
+  return rc;
 
   /* Jump here on any error */
 shm_open_err:
@@ -4477,6 +4575,11 @@ static int unixShmMap(
   p = pDbFd->pShm;
   pShmNode = p->pShmNode;
   sqlite3_mutex_enter(pShmNode->mutex);
+  if( pShmNode->isUnlocked ){
+    rc = unixLockSharedMemory(pDbFd, pShmNode);
+    if( rc!=SQLITE_OK ) goto shmpage_out;
+    pShmNode->isUnlocked = 0;
+  }
   assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 );
   assert( pShmNode->pInode==pDbFd->pInode );
   assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 );

src/sqlite.h.in

@@ -1060,6 +1060,20 @@ struct sqlite3_io_methods {
 ** so that all subsequent write operations are independent.
 ** ^SQLite will never invoke SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE without
 ** a prior successful call to [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE].
+**
+** <li>[[SQLITE_FCNTL_READ_SHM]]
+** The [SQLITE_FCNTL_READ_SHM] opcode may be used to read data from 
+** the shared-memory region used by wal mode if the previous call
+** to the xShmMap method returned SQLITE_READONLY_CANTLOCK. The argument
+** must point to an instance of a structure declared as:
+** <blockquote><pre>
+**   struct ReadShmArg { void *pBuf; int nBuf };
+** </pre></blockquote>)^
+** Before returning, this operation allocates a buffer large enough for
+** the entire shared-memory using sqlite3_malloc() and populates it with
+** a copy thereof. ReadShmArg.pBuf is set to point to the buffer, and
+** ReadShmArg to its size in bytes. It is the responsibility of the caller
+** to eventually free the buffer using sqlite3_free().
 ** </ul>
 */
 #define SQLITE_FCNTL_LOCKSTATE               1
@@ -1094,6 +1108,7 @@ struct sqlite3_io_methods {
 #define SQLITE_FCNTL_BEGIN_ATOMIC_WRITE     31
 #define SQLITE_FCNTL_COMMIT_ATOMIC_WRITE    32
 #define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE  33
+#define SQLITE_FCNTL_READ_SHM               34
 
 /* deprecated names */
 #define SQLITE_GET_LOCKPROXYFILE      SQLITE_FCNTL_GET_LOCKPROXYFILE

src/wal.c

@@ -455,6 +455,7 @@ struct Wal {
   u8 truncateOnCommit;       /* True to truncate WAL file on commit */
   u8 syncHeader;             /* Fsync the WAL header if true */
   u8 padToSectorBoundary;    /* Pad transactions out to the next sector */
+  u8 bUnlocked;              /* A readonly_shm connection without DMS lock */
   WalIndexHdr hdr;           /* Wal-index header for current transaction */
   u32 minFrame;              /* Ignore wal frames before this one */
   u32 iReCksum;              /* On commit, recalculate checksums from here */
@@ -539,6 +540,27 @@ struct WalIterator {
     sizeof(ht_slot)*HASHTABLE_NSLOT + HASHTABLE_NPAGE*sizeof(u32) \
 )
 
+/*
+** Grow the pWal->apWiData[] array so that it is at least nPage entries
+** in size. Return SQLITE_NOMEM if an OOM error occurs, or SQLITE_OK 
+** otherwise.
+*/
+static int walIndexGrowArray(Wal *pWal, int nPage){
+  if( nPage>pWal->nWiData ){
+    int nByte = sizeof(u32*)*nPage;
+    volatile u32 **apNew;
+    apNew = (volatile u32 **)sqlite3_realloc64((void *)pWal->apWiData, nByte);
+    if( !apNew ){
+      return SQLITE_NOMEM_BKPT;
+    }
+    memset((void*)&apNew[pWal->nWiData], 0,
+           sizeof(u32*)*(nPage-pWal->nWiData));
+    pWal->apWiData = apNew;
+    pWal->nWiData = nPage;
+  }
+  return SQLITE_OK;
+}
+
 /*
 ** Obtain a pointer to the iPage'th page of the wal-index. The wal-index
 ** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are
@@ -552,18 +574,9 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){
   int rc = SQLITE_OK;
 
   /* Enlarge the pWal->apWiData[] array if required */
-  if( pWal->nWiData<=iPage ){
-    int nByte = sizeof(u32*)*(iPage+1);
-    volatile u32 **apNew;
-    apNew = (volatile u32 **)sqlite3_realloc64((void *)pWal->apWiData, nByte);
-    if( !apNew ){
-      *ppPage = 0;
-      return SQLITE_NOMEM_BKPT;
-    }
-    memset((void*)&apNew[pWal->nWiData], 0,
-           sizeof(u32*)*(iPage+1-pWal->nWiData));
-    pWal->apWiData = apNew;
-    pWal->nWiData = iPage+1;
+  if( walIndexGrowArray(pWal, iPage+1) ){
+    *ppPage = 0;
+    return SQLITE_NOMEM;
   }
 
   /* Request a pointer to the required page from the VFS */
@@ -575,9 +588,11 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){
       rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, 
           pWal->writeLock, (void volatile **)&pWal->apWiData[iPage]
       );
-      if( rc==SQLITE_READONLY ){
+      if( (rc&0xff)==SQLITE_READONLY ){
         pWal->readOnly |= WAL_SHM_RDONLY;
-        rc = SQLITE_OK;
+        if( rc==SQLITE_READONLY ){
+          rc = SQLITE_OK;
+        }
       }
     }
   }
@@ -1099,24 +1114,28 @@ static int walIndexRecover(Wal *pWal){
   i64 nSize;                      /* Size of log file */
   u32 aFrameCksum[2] = {0, 0};
   int iLock;                      /* Lock offset to lock for checkpoint */
-  int nLock;                      /* Number of locks to hold */
 
-  /* Obtain an exclusive lock on all byte in the locking range not already
-  ** locked by the caller. The caller is guaranteed to have locked the
-  ** WAL_WRITE_LOCK byte, and may have also locked the WAL_CKPT_LOCK byte.
-  ** If successful, the same bytes that are locked here are unlocked before
-  ** this function returns.
-  */
+  /* Obtain an exclusive lock on all bytes in the locking range except
+  ** WAL_READ_LOCK(0) not already locked by the caller. The caller is
+  ** guaranteed to have locked the WAL_WRITE_LOCK byte, and may have also
+  ** locked the WAL_CKPT_LOCK byte. If successful, the same bytes that are
+  ** locked here are unlocked before this function returns.  */
   assert( pWal->ckptLock==1 || pWal->ckptLock==0 );
   assert( WAL_ALL_BUT_WRITE==WAL_WRITE_LOCK+1 );
   assert( WAL_CKPT_LOCK==WAL_ALL_BUT_WRITE );
   assert( pWal->writeLock );
   iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock;
-  nLock = SQLITE_SHM_NLOCK - iLock;
-  rc = walLockExclusive(pWal, iLock, nLock);
+  rc = walLockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock);
+  if( rc==SQLITE_OK ){
+    rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
+    if( rc!=SQLITE_OK ){
+      walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock);
+    }
+  }
   if( rc ){
     return rc;
   }
+
   WALTRACE(("WAL%p: recovery begin...\n", pWal));
 
   memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
@@ -1254,7 +1273,8 @@ finished:
 
 recovery_error:
   WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok"));
-  walUnlockExclusive(pWal, iLock, nLock);
+  walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock);
+  walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
   return rc;
 }
 
@@ -2082,6 +2102,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   ** wal-index header) is mapped. Return early if an error occurs here.
   */
   assert( pChanged );
+  assert( pWal->bUnlocked==0 );
   rc = walIndexPage(pWal, 0, &page0);
   if( rc!=SQLITE_OK ){
     return rc;
@@ -2101,7 +2122,10 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   assert( badHdr==0 || pWal->writeLock==0 );
   if( badHdr ){
     if( pWal->readOnly & WAL_SHM_RDONLY ){
-      if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){
+      if( pWal->bUnlocked ){
+        rc = SQLITE_READONLY_RECOVERY;
+      }
+      else if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){
         walUnlockShared(pWal, WAL_WRITE_LOCK);
         rc = SQLITE_READONLY_RECOVERY;
       }
@@ -2140,6 +2164,95 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
 */
 #define WAL_RETRY  (-1)
 
+/*
+** When this function is called, a call to xShmMap has just returned
+** SQLITE_READONLY_CANTLOCK, indicating that the *-shm file was opened
+** read-only and that there were no other connections to the database 
+** at that point.
+*/
+static int walBeginUnlocked(Wal *pWal, int *pChanged){
+  struct ReadShmParam {
+    void *pBuf;
+    int nBuf;
+  } buf = {0, 0};
+  i64 nSize;
+  int rc;
+  volatile void *pDummy;
+
+  assert( pWal->nWiData>0 && pWal->apWiData[0]==0 );
+  assert( pWal->readOnly & WAL_SHM_RDONLY );
+  
+  /* Take WAL_READ_LOCK(0). This has the effect of preventing any
+  ** live clients from running a checkpoint, but does not stop them
+  ** from running recovery.  */
+  rc = walLockShared(pWal, WAL_READ_LOCK(0));
+  if( rc!=SQLITE_OK ){
+    return (rc==SQLITE_BUSY ? WAL_RETRY : rc);
+  }
+
+  /* Try to map the *-shm file again. If it succeeds this time, then 
+  ** a non-readonly_shm connection has already connected to the database.
+  ** In this case, start over with opening the transaction.
+  **
+  ** The WAL_READ_LOCK(0) lock held by this client prevents a checkpoint
+  ** from taking place. But it does not prevent the wal from being wrapped
+  ** if a checkpoint has already taken place. This means that if another
+  ** client is connected at this point, it may have already checkpointed 
+  ** the entire wal. In that case it would not be safe to continue with
+  ** the unlocked transaction, as the other client may overwrite wal 
+  ** frames that this client is still using.  */
+  rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy);
+  if( rc!=SQLITE_READONLY_CANTLOCK ){
+    rc = (rc==SQLITE_OK ? WAL_RETRY : rc);
+    goto begin_unlocked_out;
+  }
+  pWal->readLock = 0;
+
+  /* Figure out how large the wal file is. If it is zero bytes in size,
+  ** do not bother loading the *-shm file. Just read from the db file
+  ** exclusively like any other Wal.readLock==0 client. Otherwise,
+  ** use SQLITE_FCNTL_READ_SHM to load the contents of the *-shm file 
+  ** into heap memory. */
+  rc = sqlite3OsFileSize(pWal->pWalFd, &nSize);
+  if( rc!=SQLITE_OK ) return rc;
+
+  memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
+  if( nSize>0 ){
+    rc = sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_READ_SHM, (void*)&buf);
+    if( rc!=SQLITE_OK ){
+      return rc==SQLITE_BUSY ? WAL_RETRY : rc;
+    }
+    if( buf.nBuf<WALINDEX_PGSZ ){
+      rc = SQLITE_READONLY_RECOVERY;
+    }else{
+      int i;
+      int nChunk = buf.nBuf/WALINDEX_PGSZ;
+      if( walIndexGrowArray(pWal, nChunk) ){
+        rc = SQLITE_NOMEM;
+      }else{
+        u8 *aBuf = (u8*)buf.pBuf;
+        for(i=0; i<nChunk; i++){
+          pWal->apWiData[i] = (volatile u32*)&aBuf[i*WALINDEX_PGSZ];
+        }
+        if( walIndexTryHdr(pWal, pChanged) ){
+          rc = SQLITE_READONLY_RECOVERY;
+        }
+      }
+    }
+  }
+
+ begin_unlocked_out:
+  if( rc!=SQLITE_OK ){
+    sqlite3_free(buf.pBuf);
+    memset(pWal->apWiData, 0, pWal->nWiData*sizeof(u32*));
+    walUnlockShared(pWal, WAL_READ_LOCK(0));
+  }else{
+    pWal->bUnlocked = 1;
+    *pChanged = 1;
+  }
+  return rc;
+}
+
 /*
 ** Attempt to start a read transaction.  This might fail due to a race or
 ** other transient condition.  When that happens, it returns WAL_RETRY to
@@ -2229,6 +2342,15 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
 
   if( !useWal ){
     rc = walIndexReadHdr(pWal, pChanged);
+    if( rc==SQLITE_READONLY_CANTLOCK ){
+      /* This is a readonly_shm connection and there are no other connections
+      ** to the database. So the *-shm file may not be accessed using mmap.
+      ** Try to open an "unlocked" transaction - one that loads the *-shm
+      ** file into memory using read() - instead.  */
+      assert( pWal->nWiData>0 && pWal->apWiData[0]==0 );
+      assert( pWal->readOnly & WAL_SHM_RDONLY );
+      return walBeginUnlocked(pWal, pChanged);
+    }
     if( rc==SQLITE_BUSY ){
       /* If there is not a recovery running in another thread or process
       ** then convert BUSY errors to WAL_RETRY.  If recovery is known to
@@ -2259,8 +2381,10 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
     }
   }
 
-  pInfo = walCkptInfo(pWal);
-  if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame 
+  assert( pWal->nWiData>0 );
+  assert( pWal->apWiData[0] || (pWal->readOnly & WAL_SHM_RDONLY) );
+  pInfo = pWal->apWiData[0] ? walCkptInfo(pWal) : 0;
+  if( !useWal && (pInfo==0 || pInfo->nBackfill==pWal->hdr.mxFrame)
 #ifdef SQLITE_ENABLE_SNAPSHOT
    && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0
      || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr)))
@@ -2272,7 +2396,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
     rc = walLockShared(pWal, WAL_READ_LOCK(0));
     walShmBarrier(pWal);
     if( rc==SQLITE_OK ){
-      if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){
+      if( pInfo
+       && memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) 
+      ){
         /* It is not safe to allow the reader to continue here if frames
         ** may have been appended to the log before READ_LOCK(0) was obtained.
         ** When holding READ_LOCK(0), the reader ignores the entire log file,
@@ -2576,6 +2702,11 @@ void sqlite3WalEndReadTransaction(Wal *pWal){
   if( pWal->readLock>=0 ){
     walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
     pWal->readLock = -1;
+    if( pWal->bUnlocked ){
+      sqlite3_free((void*)pWal->apWiData[0]);
+      memset(pWal->apWiData, 0, sizeof(u32*)*pWal->nWiData);
+      pWal->bUnlocked = 0;
+    }
   }
 }
 
@@ -2606,7 +2737,7 @@ int sqlite3WalFindFrame(
   ** then the WAL is ignored by the reader so return early, as if the 
   ** WAL were empty.
   */
-  if( iLast==0 || pWal->readLock==0 ){
+  if( iLast==0 || (pWal->readLock==0 && !pWal->bUnlocked) ){
     *piRead = 0;
     return SQLITE_OK;
   }

test/wal2.test

@@ -122,8 +122,8 @@ do_test wal2-1.1 {
 } {4 10}
 
 set RECOVER [list                                      \
-  {0 1 lock exclusive}   {1 7 lock exclusive}          \
-  {1 7 unlock exclusive} {0 1 unlock exclusive}        \
+  {0 1 lock exclusive}   {1 2 lock exclusive} {4 4 lock exclusive} \
+  {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive}  \
 ]
 set READ [list                                         \
   {4 1 lock shared}    {4 1 unlock shared}             \
@@ -393,8 +393,10 @@ tvfs delete
 set expected_locks [list]
 lappend expected_locks {1 1 lock exclusive}   ;# Lock checkpoint
 lappend expected_locks {0 1 lock exclusive}   ;# Lock writer
-lappend expected_locks {2 6 lock exclusive}   ;# Lock recovery & all aReadMark[]
-lappend expected_locks {2 6 unlock exclusive} ;# Unlock recovery & aReadMark[]
+lappend expected_locks {2 1 lock exclusive}   ;# Lock recovery
+lappend expected_locks {4 4 lock exclusive}   ;# Lock all aReadMark[]
+lappend expected_locks {2 1 unlock exclusive} ;# Unlock recovery 
+lappend expected_locks {4 4 unlock exclusive} ;# Unlock all aReadMark[] 
 lappend expected_locks {0 1 unlock exclusive} ;# Unlock writer
 lappend expected_locks {3 1 lock exclusive}   ;# Lock aReadMark[0]
 lappend expected_locks {3 1 unlock exclusive} ;# Unlock aReadMark[0]
@@ -615,8 +617,8 @@ do_test wal2-6.4.1 {
 } {}
 
 set RECOVERY {
-  {0 1 lock exclusive} {1 7 lock exclusive} 
-  {1 7 unlock exclusive} {0 1 unlock exclusive}
+  {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive}
+  {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive}
 }
 set READMARK0_READ {
   {3 1 lock shared} {3 1 unlock shared}

test/walro.test

@@ -155,16 +155,21 @@ do_multiclient_test tn {
     code1 { sqlite3 db file:test.db?readonly_shm=1 }
     csql1 { SELECT * FROM sqlite_master }
   } {1 {unable to open database file}}
+
+  # Update: This test used to fail (because a 0 byte *-shm file does not
+  # contain a valid shm-header) but now succeeds (because a readonly_shm
+  # connection ignores the *-shm file completely if the wal file is also
+  # zero bytes in size).
   do_test 1.3.2.3 {
     code1 { db close }
     close [open test.db-shm w]
     file attributes test.db-shm -permissions r--r--r--
     code1 { sqlite3 db file:test.db?readonly_shm=1 }
     csql1 { SELECT * FROM t1 }
-  } {1 {attempt to write a readonly database}}
+  } {0 {a b c d e f g h i j k l}}
   do_test 1.3.2.4 {
     code1 { sqlite3_extended_errcode db } 
-  } {SQLITE_READONLY_RECOVERY}
+  } {SQLITE_OK}
 
   #-----------------------------------------------------------------------
   # Test cases 1.4.* check that checkpoints and log wraps don't prevent

test/walro2.test

@@ -0,0 +1,119 @@
+# 2011 May 09
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for using WAL databases in read-only mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set ::testprefix walro
+
+# These tests are only going to work on unix.
+#
+if {$::tcl_platform(platform) != "unix"} {
+  finish_test
+  return
+}
+
+# And only if the build is WAL-capable.
+#
+ifcapable !wal {
+  finish_test
+  return
+}
+
+do_multiclient_test tn {
+  
+  # Close all connections and delete the database.
+  #
+  code1 { db close  }
+  code2 { db2 close }
+  code3 { db3 close }
+  forcedelete test.db
+  
+  # Do not run tests with the connections in the same process.
+  #
+  if {$tn==2} continue
+
+  foreach c {code1 code2 code3} {
+    $c {
+      sqlite3_shutdown
+      sqlite3_config_uri 1
+    }
+  }
+
+  do_test 1.1 {
+    code2 { sqlite3 db2 test.db }
+    sql2 { 
+      CREATE TABLE t1(x, y);
+      PRAGMA journal_mode = WAL;
+      INSERT INTO t1 VALUES('a', 'b');
+      INSERT INTO t1 VALUES('c', 'd');
+    }
+    file exists test.db-shm
+  } {1}
+
+  do_test 1.2.1 {
+    forcecopy test.db test.db2
+    forcecopy test.db-wal test.db2-wal
+    forcecopy test.db-shm test.db2-shm
+    code1 {
+      sqlite3 db file:test.db2?readonly_shm=1
+    }
+  } {}
+  do_test 1.2.2 {
+    sql1 { SELECT * FROM t1 }
+  } {a b c d}
+
+  do_test 1.3.1 {
+    code3 { sqlite3 db3 test.db2 }
+    sql3 { SELECT * FROM t1 }
+  } {a b c d}
+
+  do_test 1.3.2 {
+    sql1 { SELECT * FROM t1 }
+  } {a b c d}
+
+  code1 { db close  }
+  code2 { db2 close }
+  code3 { db3 close }
+
+  do_test 2.1 {
+    code2 { sqlite3 db2 test.db }
+    sql2 { 
+      INSERT INTO t1 VALUES('e', 'f');
+      INSERT INTO t1 VALUES('g', 'h');
+    }
+    file exists test.db-shm
+  } {1}
+
+  do_test 2.2 {
+    forcecopy test.db test.db2
+    forcecopy test.db-wal test.db2-wal
+    forcecopy test.db-shm test.db2-shm
+    code1 {
+      sqlite3 db file:test.db2?readonly_shm=1
+    }
+    sql1 { 
+      BEGIN;
+      SELECT * FROM t1;
+    }
+  } {a b c d e f g h}
+
+  do_test 2.3.1 {
+    code3 { sqlite3 db3 test.db2 }
+    sql3 { SELECT * FROM t1 }
+  } {a b c d e f g h}
+
+}
+
+finish_test