Merge latest mutexfree-shmlock changes into this branch.

FossilOrigin-Name: 186b376ec3713f9c8a1499a9675b3c1b42ef3a123848506a48d3a196ba6461f3

ext/session/changebatch1.test

@@ -0,0 +1,222 @@
+# 2016 August 23
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source $testdir/tester.tcl
+ifcapable !session {finish_test; return}
+
+set testprefix changebatch1
+
+
+proc sql_to_changeset {method sql} {
+  sqlite3session S db main
+  S attach *
+  execsql $sql
+  set ret [S $method]
+  S delete
+  return $ret
+}
+
+proc do_changebatch_test {tn method args} {
+  set C [list]
+  foreach a $args {
+    lappend C [sql_to_changeset $method $a]
+  }
+
+  sqlite3changebatch cb db
+  set i 1
+  foreach ::cs [lrange $C 0 end-1] {
+    set rc [cb add $::cs]
+    if {$rc!="SQLITE_OK"} { error "expected SQLITE_OK, got $rc (i=$i)" }
+    incr i
+  }
+
+  set ::cs [lindex $C end]
+  do_test $tn { cb add [set ::cs] } SQLITE_CONSTRAINT
+  cb delete
+}
+
+proc do_changebatch_test1 {tn args} {
+  uplevel do_changebatch_test $tn changeset $args
+}
+proc do_changebatch_test2 {tn args} {
+  uplevel do_changebatch_test $tn fullchangeset $args
+}
+
+#-------------------------------------------------------------------------
+# The body of the following loop contains tests for database schemas
+# that do not feature multi-column UNIQUE constraints. In this case
+# it doesn't matter if the changesets are generated using
+# sqlite3session_changeset() or sqlite3session_fullchangeset().
+#
+foreach {tn testfunction} {
+  1 do_changebatch_test1
+  2 do_changebatch_test2
+} {
+  reset_db
+
+  #-------------------------------------------------------------------------
+  #
+  do_execsql_test $tn.1.0 {
+    CREATE TABLE t1(a PRIMARY KEY, b);
+  }
+  
+  $testfunction $tn.1.1 {
+    INSERT INTO t1 VALUES(1, 1);
+  } {
+    DELETE FROM t1 WHERE a=1;
+  }
+  
+  do_execsql_test $tn.1.2.0 {
+    INSERT INTO t1 VALUES(1, 1);
+    INSERT INTO t1 VALUES(2, 2);
+    INSERT INTO t1 VALUES(3, 3);
+  }
+  $testfunction $tn.1.2.1 {
+    DELETE FROM t1 WHERE a=2;
+  } {
+    INSERT INTO t1 VALUES(2, 2);
+  }
+  
+  #-------------------------------------------------------------------------
+  #
+  do_execsql_test $tn.2.0 {
+    CREATE TABLE x1(a, b PRIMARY KEY, c UNIQUE);
+    CREATE TABLE x2(a PRIMARY KEY, b UNIQUE, c UNIQUE);
+    CREATE INDEX x1a ON x1(a);
+  
+    INSERT INTO x1 VALUES(1, 1, 'a');
+    INSERT INTO x1 VALUES(1, 2, 'b');
+    INSERT INTO x1 VALUES(1, 3, 'c');
+  }
+  
+  $testfunction $tn.2.1 {
+    DELETE FROM x1 WHERE b=2;
+  } {
+    UPDATE x1 SET c='b' WHERE b=3;
+  }
+  
+  $testfunction $tn.2.2 {
+    DELETE FROM x1 WHERE b=1;
+  } {
+    INSERT INTO x1 VALUES(1, 5, 'a');
+  }
+
+  set L [list]
+  for {set i 1000} {$i < 10000} {incr i} {
+    lappend L "INSERT INTO x2 VALUES($i, $i, 'x' || $i)"
+  }
+  lappend L "DELETE FROM x2 WHERE b=1005"
+  $testfunction $tn.2.3 {*}$L
+
+  execsql { INSERT INTO x1 VALUES('f', 'f', 'f') }
+  $testfunction $tn.2.4 {
+    INSERT INTO x2 VALUES('f', 'f', 'f');
+  } {
+    INSERT INTO x1 VALUES('g', 'g', 'g');
+  } {
+    DELETE FROM x1 WHERE b='f';
+  } {
+    INSERT INTO x2 VALUES('g', 'g', 'g');
+  } {
+    INSERT INTO x1 VALUES('f', 'f', 'f');
+  }
+
+  execsql {
+    DELETE FROM x1;
+    INSERT INTO x1 VALUES(1.5, 1.5, 1.5);
+  }
+  $testfunction $tn.2.5 {
+    DELETE FROM x1 WHERE b BETWEEN 1 AND 2;
+  } {
+    INSERT INTO x1 VALUES(2.5, 2.5, 2.5);
+  } {
+    INSERT INTO x1 VALUES(1.5, 1.5, 1.5);
+  }
+
+  execsql {
+    DELETE FROM x2;
+    INSERT INTO x2 VALUES(X'abcd', X'1234', X'7890');
+    INSERT INTO x2 VALUES(X'0000', X'0000', X'0000');
+  }
+  breakpoint
+  $testfunction $tn.2.6 {
+    UPDATE x2 SET c = X'1234' WHERE a=X'abcd';
+    INSERT INTO x2 VALUES(X'1234', X'abcd', X'7890');
+  } {
+    DELETE FROM x2 WHERE b=X'0000';
+  } {
+    INSERT INTO x2 VALUES(1, X'0000', NULL);
+  }
+}
+
+#-------------------------------------------------------------------------
+# Test some multi-column UNIQUE constraints. First Using _changeset() to
+# demonstrate the problem, then using _fullchangeset() to show that it has
+# been fixed.
+#
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE y1(a PRIMARY KEY, b, c, UNIQUE(b, c));
+  INSERT INTO y1 VALUES(1, 1, 1);
+  INSERT INTO y1 VALUES(2, 2, 2);
+  INSERT INTO y1 VALUES(3, 3, 3);
+  INSERT INTO y1 VALUES(4, 3, 4);
+  BEGIN;
+}
+
+do_test 3.1.1 {
+  set c1 [sql_to_changeset changeset { DELETE FROM y1 WHERE a=4    }]
+  set c2 [sql_to_changeset changeset { UPDATE y1 SET c=4 WHERE a=3 }]
+  sqlite3changebatch cb db
+  cb add $c1
+  cb add $c2
+} {SQLITE_OK}
+do_test 3.1.2 {
+  cb delete
+  execsql ROLLBACK
+} {}
+
+do_test 3.1.1 {
+  set c1 [sql_to_changeset fullchangeset { DELETE FROM y1 WHERE a=4    }]
+  set c2 [sql_to_changeset fullchangeset { UPDATE y1 SET c=4 WHERE a=3 }]
+  sqlite3changebatch cb db
+  cb add $c1
+  cb add $c2
+} {SQLITE_CONSTRAINT}
+do_test 3.1.2 {
+  cb delete
+} {}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 4.0 {
+  CREATE TABLE t1(x, y, z, PRIMARY KEY(x, y), UNIQUE(z));
+}
+
+do_test 4.1 {
+  set c1 [sql_to_changeset fullchangeset { INSERT INTO t1 VALUES(1, 2, 3) }]
+  execsql {
+    DROP TABLE t1;
+    CREATE TABLE t1(w, x, y, z, PRIMARY KEY(x, y), UNIQUE(z));
+  }
+  sqlite3changebatch cb db
+  list [catch { cb add $c1 } msg] $msg
+} {1 SQLITE_RANGE}
+
+
+
+finish_test

ext/session/changebatchfault.test

@@ -0,0 +1,42 @@
+# 2011 Mar 21
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the session module.
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source [file join [file dirname [info script]] session_common.tcl]
+source $testdir/tester.tcl
+ifcapable !session {finish_test; return}
+set testprefix changebatchfault
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c PRIMARY KEY, UNIQUE(a, b));
+  INSERT INTO t1 VALUES('a', 'a', 'a');
+  INSERT INTO t1 VALUES('b', 'b', 'b');
+}
+
+set ::c1 [changeset_from_sql { delete from t1 where c='a' }]
+set ::c2 [changeset_from_sql { insert into t1 values('c', 'c', 'c') }]
+
+do_faultsim_test 1 -faults oom-* -body {
+  sqlite3changebatch cb db
+  cb add $::c1
+  cb add $::c2
+} -test {
+  faultsim_test_result {0 SQLITE_OK} {1 SQLITE_NOMEM}
+  catch { cb delete }
+}
+
+
+finish_test

ext/session/sqlite3changebatch.c

@@ -0,0 +1,485 @@
+
+#if !defined(SQLITE_TEST) || (defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK))
+
+#include "sqlite3session.h"
+#include "sqlite3changebatch.h"
+
+#include <assert.h>
+#include <string.h>
+
+typedef struct BatchTable BatchTable;
+typedef struct BatchIndex BatchIndex;
+typedef struct BatchIndexEntry BatchIndexEntry;
+typedef struct BatchHash BatchHash;
+
+struct sqlite3_changebatch {
+  sqlite3 *db;                    /* Database handle used to read schema */
+  BatchTable *pTab;               /* First in linked list of tables */
+  int iChangesetId;               /* Current changeset id */
+  int iNextIdxId;                 /* Next available index id */
+  int nEntry;                     /* Number of entries in hash table */
+  int nHash;                      /* Number of hash buckets */
+  BatchIndexEntry **apHash;       /* Array of hash buckets */
+};
+
+struct BatchTable {
+  BatchIndex *pIdx;               /* First in linked list of UNIQUE indexes */
+  BatchTable *pNext;              /* Next table */
+  char zTab[1];                   /* Table name */
+};
+
+struct BatchIndex {
+  BatchIndex *pNext;              /* Next index on same table */
+  int iId;                        /* Index id (assigned internally) */
+  int bPk;                        /* True for PK index */
+  int nCol;                       /* Size of aiCol[] array */
+  int *aiCol;                     /* Array of columns that make up index */
+};
+
+struct BatchIndexEntry {
+  BatchIndexEntry *pNext;         /* Next colliding hash table entry */
+  int iChangesetId;               /* Id of associated changeset */
+  int iIdxId;                     /* Id of index this key is from */
+  int szRecord;
+  char aRecord[1];
+};
+
+/*
+** Allocate and zero a block of nByte bytes. Must be freed using cbFree().
+*/
+static void *cbMalloc(int *pRc, int nByte){
+  void *pRet;
+
+  if( *pRc ){
+    pRet = 0;
+  }else{
+    pRet = sqlite3_malloc(nByte);
+    if( pRet ){
+      memset(pRet, 0, nByte);
+    }else{
+      *pRc = SQLITE_NOMEM;
+    }
+  }
+
+  return pRet;
+}
+
+/*
+** Free an allocation made by cbMalloc().
+*/
+static void cbFree(void *p){
+  sqlite3_free(p);
+}
+
+/*
+** Return the hash bucket that pEntry belongs in.
+*/
+static int cbHash(sqlite3_changebatch *p, BatchIndexEntry *pEntry){
+  unsigned int iHash = (unsigned int)pEntry->iIdxId;
+  unsigned char *pEnd = (unsigned char*)&pEntry->aRecord[pEntry->szRecord];
+  unsigned char *pIter;
+
+  for(pIter=(unsigned char*)pEntry->aRecord; pIter<pEnd; pIter++){
+    iHash += (iHash << 7) + *pIter;
+  }
+
+  return (int)(iHash % p->nHash);
+}
+
+/*
+** Resize the hash table.
+*/
+static int cbHashResize(sqlite3_changebatch *p){
+  int rc = SQLITE_OK;
+  BatchIndexEntry **apNew;
+  int nNew = (p->nHash ? p->nHash*2 : 512);
+  int i;
+
+  apNew = cbMalloc(&rc, sizeof(BatchIndexEntry*) * nNew);
+  if( rc==SQLITE_OK ){
+    int nHash = p->nHash;
+    p->nHash = nNew;
+    for(i=0; i<nHash; i++){
+      BatchIndexEntry *pEntry;
+      while( (pEntry=p->apHash[i])!=0 ){
+        int iHash = cbHash(p, pEntry);
+        p->apHash[i] = pEntry->pNext;
+        pEntry->pNext = apNew[iHash];
+        apNew[iHash] = pEntry;
+      }
+    }
+
+    cbFree(p->apHash);
+    p->apHash = apNew;
+  }
+
+  return rc;
+}
+
+
+/*
+** Allocate a new sqlite3_changebatch object.
+*/
+int sqlite3changebatch_new(sqlite3 *db, sqlite3_changebatch **pp){
+  sqlite3_changebatch *pRet;
+  int rc = SQLITE_OK;
+  *pp = pRet = (sqlite3_changebatch*)cbMalloc(&rc, sizeof(sqlite3_changebatch));
+  if( pRet ){
+    pRet->db = db;
+  }
+  return rc;
+}
+
+/*
+** Add a BatchIndex entry for index zIdx to table pTab.
+*/
+static int cbAddIndex(
+  sqlite3_changebatch *p, 
+  BatchTable *pTab, 
+  const char *zIdx, 
+  int bPk
+){
+  int nCol = 0;
+  sqlite3_stmt *pIndexInfo = 0;
+  BatchIndex *pNew = 0;
+  int rc;
+  char *zIndexInfo;
+
+  zIndexInfo = (char*)sqlite3_mprintf("PRAGMA main.index_info = %Q", zIdx);
+  if( zIndexInfo ){
+    rc = sqlite3_prepare_v2(p->db, zIndexInfo, -1, &pIndexInfo, 0);
+    sqlite3_free(zIndexInfo);
+  }else{
+    rc = SQLITE_NOMEM;
+  }
+
+  if( rc==SQLITE_OK ){
+    while( SQLITE_ROW==sqlite3_step(pIndexInfo) ){ nCol++; }
+    rc = sqlite3_reset(pIndexInfo);
+  }
+
+  pNew = (BatchIndex*)cbMalloc(&rc, sizeof(BatchIndex) + sizeof(int) * nCol);
+  if( rc==SQLITE_OK ){
+    pNew->nCol = nCol;
+    pNew->bPk = bPk;
+    pNew->aiCol = (int*)&pNew[1];
+    pNew->iId = p->iNextIdxId++;
+    while( SQLITE_ROW==sqlite3_step(pIndexInfo) ){ 
+      int i = sqlite3_column_int(pIndexInfo, 0);
+      int j = sqlite3_column_int(pIndexInfo, 1);
+      pNew->aiCol[i] = j;
+    }
+    rc = sqlite3_reset(pIndexInfo);
+  }
+
+  if( rc==SQLITE_OK ){
+    pNew->pNext = pTab->pIdx;
+    pTab->pIdx = pNew;
+  }else{
+    cbFree(pNew);
+  }
+  sqlite3_finalize(pIndexInfo);
+
+  return rc;
+}
+
+/*
+** Free the object passed as the first argument.
+*/
+static void cbFreeTable(BatchTable *pTab){
+  BatchIndex *pIdx;
+  BatchIndex *pIdxNext;
+  for(pIdx=pTab->pIdx; pIdx; pIdx=pIdxNext){
+    pIdxNext = pIdx->pNext;
+    cbFree(pIdx);
+  }
+  cbFree(pTab);
+}
+
+/*
+** Find or create the BatchTable object named zTab.
+*/
+static int cbFindTable(
+  sqlite3_changebatch *p, 
+  const char *zTab, 
+  BatchTable **ppTab
+){
+  BatchTable *pRet = 0;
+  int rc = SQLITE_OK;
+
+  for(pRet=p->pTab; pRet; pRet=pRet->pNext){
+    if( 0==sqlite3_stricmp(zTab, pRet->zTab) ) break;
+  }
+
+  if( pRet==0 ){
+    int nTab = strlen(zTab);
+    pRet = (BatchTable*)cbMalloc(&rc, nTab + sizeof(BatchTable));
+    if( pRet ){
+      sqlite3_stmt *pIndexList = 0;
+      char *zIndexList = 0;
+      int rc2;
+      memcpy(pRet->zTab, zTab, nTab);
+
+      zIndexList = sqlite3_mprintf("PRAGMA main.index_list = %Q", zTab);
+      if( zIndexList==0 ){
+        rc = SQLITE_NOMEM;
+      }else{
+        rc = sqlite3_prepare_v2(p->db, zIndexList, -1, &pIndexList, 0);
+        sqlite3_free(zIndexList);
+      }
+
+      while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pIndexList) ){
+        if( sqlite3_column_int(pIndexList, 2) ){
+          const char *zIdx = (const char*)sqlite3_column_text(pIndexList, 1);
+          const char *zTyp = (const char*)sqlite3_column_text(pIndexList, 3);
+          rc = cbAddIndex(p, pRet, zIdx, (zTyp[0]=='p'));
+        }
+      }
+      rc2 = sqlite3_finalize(pIndexList);
+      if( rc==SQLITE_OK ) rc = rc2;
+
+      if( rc==SQLITE_OK ){
+        pRet->pNext = p->pTab;
+        p->pTab = pRet;
+      }else{
+        cbFreeTable(pRet);
+        pRet = 0;
+      }
+    }
+  }
+
+  *ppTab = pRet;
+  return rc;
+}
+
+/*
+** Extract value iVal from the changeset iterator passed as the first
+** argument. Set *ppVal to point to the value before returning.
+**
+** This function attempts to extract the value using function xVal
+** (which is always either sqlite3changeset_new or sqlite3changeset_old).
+** If the call returns SQLITE_OK but does not supply an sqlite3_value*
+** pointer, an attempt to extract the value is made using the xFallback 
+** function.
+*/
+static int cbGetChangesetValue(
+  sqlite3_changeset_iter *pIter, 
+  int (*xVal)(sqlite3_changeset_iter*,int,sqlite3_value**),
+  int (*xFallback)(sqlite3_changeset_iter*,int,sqlite3_value**),
+  int iVal,
+  sqlite3_value **ppVal
+){
+  int rc = xVal(pIter, iVal, ppVal);
+  if( rc==SQLITE_OK && *ppVal==0 && xFallback ){
+    rc = xFallback(pIter, iVal, ppVal);
+  }
+  return rc;
+}
+
+static int cbAddToHash(
+  sqlite3_changebatch *p, 
+  sqlite3_changeset_iter *pIter, 
+  BatchIndex *pIdx, 
+  int (*xVal)(sqlite3_changeset_iter*,int,sqlite3_value**),
+  int (*xFallback)(sqlite3_changeset_iter*,int,sqlite3_value**),
+  int *pbConf
+){
+  BatchIndexEntry *pNew;
+  int sz = pIdx->nCol;
+  int i;
+  int iOut = 0;
+  int rc = SQLITE_OK;
+
+  for(i=0; rc==SQLITE_OK && i<pIdx->nCol; i++){
+    sqlite3_value *pVal;
+    rc = cbGetChangesetValue(pIter, xVal, xFallback, pIdx->aiCol[i], &pVal);
+    if( rc==SQLITE_OK ){
+      int eType = 0;
+      if( pVal ) eType = sqlite3_value_type(pVal);
+      switch( eType ){
+        case 0:
+        case SQLITE_NULL:
+          return SQLITE_OK;
+
+        case SQLITE_INTEGER:
+          sz += 8;
+          break;
+        case SQLITE_FLOAT:
+          sz += 8;
+          break;
+
+        default:
+          assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB );
+          sz += sqlite3_value_bytes(pVal);
+          break;
+      }
+    }
+  }
+
+  pNew = cbMalloc(&rc, sizeof(BatchIndexEntry) + sz);
+  if( pNew ){
+    pNew->iChangesetId = p->iChangesetId;
+    pNew->iIdxId = pIdx->iId;
+    pNew->szRecord = sz;
+
+    for(i=0; i<pIdx->nCol; i++){
+      int eType;
+      sqlite3_value *pVal;
+      rc = cbGetChangesetValue(pIter, xVal, xFallback, pIdx->aiCol[i], &pVal);
+      if( rc!=SQLITE_OK ) break;  /* coverage: condition is never true */
+      eType = sqlite3_value_type(pVal);
+      pNew->aRecord[iOut++] = eType;
+      switch( eType ){
+        case SQLITE_INTEGER: {
+          sqlite3_int64 i64 = sqlite3_value_int64(pVal);
+          memcpy(&pNew->aRecord[iOut], &i64, 8);
+          iOut += 8;
+          break;
+        }
+        case SQLITE_FLOAT: {
+          double d64 = sqlite3_value_double(pVal);
+          memcpy(&pNew->aRecord[iOut], &d64, sizeof(double));
+          iOut += sizeof(double);
+          break;
+        }
+
+        default: {
+          int nByte = sqlite3_value_bytes(pVal);
+          const char *z = (const char*)sqlite3_value_blob(pVal);
+          memcpy(&pNew->aRecord[iOut], z, nByte);
+          iOut += nByte;
+          break;
+        }
+      }
+    }
+  }
+
+  if( rc==SQLITE_OK && p->nEntry>=(p->nHash/2) ){
+    rc = cbHashResize(p);
+  }
+
+  if( rc==SQLITE_OK ){
+    BatchIndexEntry *pIter;
+    int iHash = cbHash(p, pNew);
+
+    assert( iHash>=0 && iHash<p->nHash );
+    for(pIter=p->apHash[iHash]; pIter; pIter=pIter->pNext){
+      if( pNew->szRecord==pIter->szRecord 
+       && 0==memcmp(pNew->aRecord, pIter->aRecord, pNew->szRecord)
+      ){
+        if( pNew->iChangesetId!=pIter->iChangesetId ){
+          *pbConf = 1;
+        }
+        cbFree(pNew);
+        pNew = 0;
+        break;
+      }
+    }
+
+    if( pNew ){
+      pNew->pNext = p->apHash[iHash];
+      p->apHash[iHash] = pNew;
+      p->nEntry++;
+    }
+  }else{
+    cbFree(pNew);
+  }
+
+  return rc;
+}
+
+
+/*
+** Add a changeset to the current batch.
+*/
+int sqlite3changebatch_add(sqlite3_changebatch *p, void *pBuf, int nBuf){
+  sqlite3_changeset_iter *pIter;  /* Iterator opened on pBuf/nBuf */
+  int rc;                         /* Return code */
+  int bConf = 0;                  /* Conflict was detected */
+
+  rc = sqlite3changeset_start(&pIter, nBuf, pBuf);
+  if( rc==SQLITE_OK ){
+    int rc2;
+    for(rc2 = sqlite3changeset_next(pIter);
+        rc2==SQLITE_ROW;
+        rc2 = sqlite3changeset_next(pIter)
+    ){
+      BatchTable *pTab;
+      BatchIndex *pIdx;
+      const char *zTab;           /* Table this change applies to */
+      int nCol;                   /* Number of columns in table */
+      int op;                     /* UPDATE, INSERT or DELETE */
+
+      sqlite3changeset_op(pIter, &zTab, &nCol, &op, 0);
+      assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE );
+
+      rc = cbFindTable(p, zTab, &pTab);
+      assert( pTab || rc!=SQLITE_OK );
+      if( pTab ){
+        for(pIdx=pTab->pIdx; pIdx && rc==SQLITE_OK; pIdx=pIdx->pNext){
+          if( op==SQLITE_UPDATE && pIdx->bPk ) continue;
+          if( op==SQLITE_UPDATE || op==SQLITE_DELETE ){
+            rc = cbAddToHash(p, pIter, pIdx, sqlite3changeset_old, 0, &bConf);
+          }
+          if( op==SQLITE_UPDATE || op==SQLITE_INSERT ){
+            rc = cbAddToHash(p, pIter, pIdx, 
+                sqlite3changeset_new, sqlite3changeset_old, &bConf
+            );
+          }
+        }
+      }
+      if( rc!=SQLITE_OK ) break;
+    }
+
+    rc2 = sqlite3changeset_finalize(pIter);
+    if( rc==SQLITE_OK ) rc = rc2;
+  }
+
+  if( rc==SQLITE_OK && bConf ){
+    rc = SQLITE_CONSTRAINT;
+  }
+  p->iChangesetId++;
+  return rc;
+}
+
+/*
+** Zero an existing changebatch object.
+*/
+void sqlite3changebatch_zero(sqlite3_changebatch *p){
+  int i;
+  for(i=0; i<p->nHash; i++){
+    BatchIndexEntry *pEntry;
+    BatchIndexEntry *pNext;
+    for(pEntry=p->apHash[i]; pEntry; pEntry=pNext){
+      pNext = pEntry->pNext;
+      cbFree(pEntry);
+    }
+  }
+  cbFree(p->apHash);
+  p->nHash = 0;
+  p->apHash = 0;
+}
+
+/*
+** Delete a changebatch object.
+*/
+void sqlite3changebatch_delete(sqlite3_changebatch *p){
+  BatchTable *pTab;
+  BatchTable *pTabNext;
+
+  sqlite3changebatch_zero(p);
+  for(pTab=p->pTab; pTab; pTab=pTabNext){
+    pTabNext = pTab->pNext;
+    cbFreeTable(pTab);
+  }
+  cbFree(p);
+}
+
+/*
+** Return the db handle.
+*/
+sqlite3 *sqlite3changebatch_db(sqlite3_changebatch *p){
+  return p->db;
+}
+
+#endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */

ext/session/sqlite3changebatch.h

@@ -0,0 +1,82 @@
+
+#if !defined(SQLITECHANGEBATCH_H_) 
+#define SQLITECHANGEBATCH_H_ 1
+
+typedef struct sqlite3_changebatch sqlite3_changebatch;
+
+/*
+** Create a new changebatch object for detecting conflicts between
+** changesets associated with a schema equivalent to that of the "main"
+** database of the open database handle db passed as the first
+** parameter. It is the responsibility of the caller to ensure that
+** the database handle is not closed until after the changebatch
+** object has been deleted.
+**
+** A changebatch object is used to detect batches of non-conflicting
+** changesets. Changesets that do not conflict may be applied to the 
+** target database in any order without affecting the final state of 
+** the database.
+**
+** The changebatch object only works reliably if PRIMARY KEY and UNIQUE
+** constraints on tables affected by the changesets use collation
+** sequences that are equivalent to built-in collation sequence 
+** BINARY for the == operation.
+**
+** If successful, SQLITE_OK is returned and (*pp) set to point to
+** the new changebatch object. If an error occurs, an SQLite error
+** code is returned and the final value of (*pp) is undefined.
+*/
+int sqlite3changebatch_new(sqlite3 *db, sqlite3_changebatch **pp);
+
+/*
+** Argument p points to a buffer containing a changeset n bytes in
+** size. Assuming no error occurs, this function returns SQLITE_OK
+** if the changeset does not conflict with any changeset passed 
+** to an sqlite3changebatch_add() call made on the same 
+** sqlite3_changebatch* handle since the most recent call to
+** sqlite3changebatch_zero(). If the changeset does conflict with 
+** an earlier such changeset, SQLITE_CONSTRAINT is returned. Or, 
+** if an error occurs, some other SQLite error code may be returned.
+**
+** One changeset is said to conflict with another if
+** either:
+**
+**   * the two changesets contain operations (INSERT, UPDATE or 
+**     DELETE) on the same row, identified by primary key, or
+**
+**   * the two changesets contain operations (INSERT, UPDATE or 
+**     DELETE) on rows with identical values in any combination 
+**     of fields constrained by a UNIQUE constraint.
+**
+** Even if this function returns SQLITE_CONFLICT, the current
+** changeset is added to the internal data structures - so future
+** calls to this function may conflict with it. If this function
+** returns any result code other than SQLITE_OK or SQLITE_CONFLICT,
+** the result of any future call to sqlite3changebatch_add() is
+** undefined.
+**
+** Only changesets may be passed to this function. Passing a 
+** patchset to this function results in an SQLITE_MISUSE error.
+*/
+int sqlite3changebatch_add(sqlite3_changebatch*, void *p, int n);
+
+/*
+** Zero a changebatch object. This causes the records of all earlier 
+** calls to sqlite3changebatch_add() to be discarded.
+*/
+void sqlite3changebatch_zero(sqlite3_changebatch*);
+
+/*
+** Return a copy of the first argument passed to the sqlite3changebatch_new()
+** call used to create the changebatch object passed as the only argument
+** to this function.
+*/
+sqlite3 *sqlite3changebatch_db(sqlite3_changebatch*);
+
+/*
+** Delete a changebatch object.
+*/
+void sqlite3changebatch_delete(sqlite3_changebatch*);
+
+#endif /* !defined(SQLITECHANGEBATCH_H_) */
+

ext/session/sqlite3session.c

@@ -25,6 +25,13 @@ typedef struct SessionInput SessionInput;
 # endif
 #endif
 
+/*
+** The three different types of changesets generated.
+*/
+#define SESSIONS_PATCHSET      0
+#define SESSIONS_CHANGESET     1
+#define SESSIONS_FULLCHANGESET 2
+
 static int sessions_strm_chunk_size = SESSIONS_STRM_CHUNK_SIZE;
 
 typedef struct SessionHook SessionHook;
@@ -2052,7 +2059,7 @@ static void sessionAppendCol(
 */ 
 static int sessionAppendUpdate(
   SessionBuffer *pBuf,            /* Buffer to append to */
-  int bPatchset,                  /* True for "patchset", 0 for "changeset" */
+  int ePatchset,                  /* True for "patchset", 0 for "changeset" */
   sqlite3_stmt *pStmt,            /* Statement handle pointing at new row */
   SessionChange *p,               /* Object containing old values */
   u8 *abPK                        /* Boolean array - true for PK columns */
@@ -2115,8 +2122,8 @@ static int sessionAppendUpdate(
 
     /* Add a field to the old.* record. This is omitted if this modules is
     ** currently generating a patchset. */
-    if( bPatchset==0 ){
-      if( bChanged || abPK[i] ){
+    if( ePatchset!=SESSIONS_PATCHSET ){
+      if( ePatchset==SESSIONS_FULLCHANGESET || bChanged || abPK[i] ){
         sessionAppendBlob(pBuf, pCsr, nAdvance, &rc);
       }else{
         sessionAppendByte(pBuf, 0, &rc);
@@ -2125,7 +2132,7 @@ static int sessionAppendUpdate(
 
     /* Add a field to the new.* record. Or the only record if currently
     ** generating a patchset.  */
-    if( bChanged || (bPatchset && abPK[i]) ){
+    if( bChanged || (ePatchset==SESSIONS_PATCHSET && abPK[i]) ){
       sessionAppendCol(&buf2, pStmt, i, &rc);
     }else{
       sessionAppendByte(&buf2, 0, &rc);
@@ -2151,7 +2158,7 @@ static int sessionAppendUpdate(
 */
 static int sessionAppendDelete(
   SessionBuffer *pBuf,            /* Buffer to append to */
-  int bPatchset,                  /* True for "patchset", 0 for "changeset" */
+  int eChangeset,                 /* One of SESSIONS_CHANGESET etc. */
   SessionChange *p,               /* Object containing old values */
   int nCol,                       /* Number of columns in table */
   u8 *abPK                        /* Boolean array - true for PK columns */
@@ -2161,7 +2168,7 @@ static int sessionAppendDelete(
   sessionAppendByte(pBuf, SQLITE_DELETE, &rc);
   sessionAppendByte(pBuf, p->bIndirect, &rc);
 
-  if( bPatchset==0 ){
+  if( eChangeset!=SESSIONS_PATCHSET ){
     sessionAppendBlob(pBuf, p->aRecord, p->nRecord, &rc);
   }else{
     int i;
@@ -2334,12 +2341,12 @@ static int sessionSelectBind(
 */
 static void sessionAppendTableHdr(
   SessionBuffer *pBuf,            /* Append header to this buffer */
-  int bPatchset,                  /* Use the patchset format if true */
+  int ePatchset,                  /* Use the patchset format if true */
   SessionTable *pTab,             /* Table object to append header for */
   int *pRc                        /* IN/OUT: Error code */
 ){
   /* Write a table header */
-  sessionAppendByte(pBuf, (bPatchset ? 'P' : 'T'), pRc);
+  sessionAppendByte(pBuf, (ePatchset==SESSIONS_PATCHSET) ? 'P' : 'T', pRc);
   sessionAppendVarint(pBuf, pTab->nCol, pRc);
   sessionAppendBlob(pBuf, pTab->abPK, pTab->nCol, pRc);
   sessionAppendBlob(pBuf, (u8 *)pTab->zName, (int)strlen(pTab->zName)+1, pRc);
@@ -2357,7 +2364,7 @@ static void sessionAppendTableHdr(
 */
 static int sessionGenerateChangeset(
   sqlite3_session *pSession,      /* Session object */
-  int bPatchset,                  /* True for patchset, false for changeset */
+  int ePatchset,                  /* One of SESSIONS_CHANGESET etc. */
   int (*xOutput)(void *pOut, const void *pData, int nData),
   void *pOut,                     /* First argument for xOutput */
   int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
@@ -2402,7 +2409,7 @@ static int sessionGenerateChangeset(
       }
 
       /* Write a table header */
-      sessionAppendTableHdr(&buf, bPatchset, pTab, &rc);
+      sessionAppendTableHdr(&buf, ePatchset, pTab, &rc);
 
       /* Build and compile a statement to execute: */
       if( rc==SQLITE_OK ){
@@ -2426,10 +2433,10 @@ static int sessionGenerateChangeset(
                 sessionAppendCol(&buf, pSel, iCol, &rc);
               }
             }else{
-              rc = sessionAppendUpdate(&buf, bPatchset, pSel, p, abPK);
+              rc = sessionAppendUpdate(&buf, ePatchset, pSel, p, abPK);
             }
           }else if( p->op!=SQLITE_INSERT ){
-            rc = sessionAppendDelete(&buf, bPatchset, p, nCol, abPK);
+            rc = sessionAppendDelete(&buf, ePatchset, p, nCol, abPK);
           }
           if( rc==SQLITE_OK ){
             rc = sqlite3_reset(pSel);
@@ -2486,7 +2493,8 @@ int sqlite3session_changeset(
   int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
   void **ppChangeset              /* OUT: Buffer containing changeset */
 ){
-  return sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset, ppChangeset);
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_CHANGESET, 0, 0, pnChangeset, ppChangeset);
 }
 
 /*
@@ -2497,7 +2505,8 @@ int sqlite3session_changeset_strm(
   int (*xOutput)(void *pOut, const void *pData, int nData),
   void *pOut
 ){
-  return sessionGenerateChangeset(pSession, 0, xOutput, pOut, 0, 0);
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_CHANGESET, xOutput, pOut, 0, 0);
 }
 
 /*
@@ -2508,7 +2517,8 @@ int sqlite3session_patchset_strm(
   int (*xOutput)(void *pOut, const void *pData, int nData),
   void *pOut
 ){
-  return sessionGenerateChangeset(pSession, 1, xOutput, pOut, 0, 0);
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_PATCHSET, xOutput, pOut, 0, 0);
 }
 
 /*
@@ -2523,9 +2533,20 @@ int sqlite3session_patchset(
   int *pnPatchset,                /* OUT: Size of buffer at *ppChangeset */
   void **ppPatchset               /* OUT: Buffer containing changeset */
 ){
-  return sessionGenerateChangeset(pSession, 1, 0, 0, pnPatchset, ppPatchset);
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_PATCHSET, 0, 0, pnPatchset, ppPatchset);
 }
 
+int sqlite3session_fullchangeset(
+  sqlite3_session *pSession,      /* Session object */
+  int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
+  void **ppChangeset              /* OUT: Buffer containing changeset */
+){
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_FULLCHANGESET, 0, 0, pnChangeset, ppChangeset);
+}
+
+
 /*
 ** Enable or disable the session object passed as the first argument.
 */
@@ -4927,10 +4948,11 @@ static int sessionChangegroupOutput(
   ** hash tables attached to the SessionTable objects in list p->pList. 
   */
   for(pTab=pGrp->pList; rc==SQLITE_OK && pTab; pTab=pTab->pNext){
+    int eChangeset = pGrp->bPatch ? SESSIONS_PATCHSET : SESSIONS_CHANGESET;
     int i;
     if( pTab->nEntry==0 ) continue;
 
-    sessionAppendTableHdr(&buf, pGrp->bPatch, pTab, &rc);
+    sessionAppendTableHdr(&buf, eChangeset, pTab, &rc);
     for(i=0; i<pTab->nChange; i++){
       SessionChange *p;
       for(p=pTab->apChange[i]; p; p=p->pNext){

ext/session/sqlite3session.h

@@ -323,6 +323,19 @@ int sqlite3session_changeset(
   void **ppChangeset              /* OUT: Buffer containing changeset */
 );
 
+/*
+** CAPI3REF: Generate A Full Changeset From A Session Object
+**
+** This function is similar to sqlite3session_changeset(), except that for
+** each row affected by an UPDATE statement, all old.* values are recorded
+** as part of the changeset, not just those modified.
+*/
+int sqlite3session_fullchangeset(
+  sqlite3_session *pSession,      /* Session object */
+  int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
+  void **ppChangeset              /* OUT: Buffer containing changeset */
+);
+
 /*
 ** CAPI3REF: Load The Difference Between Tables Into A Session
 ** METHOD: sqlite3_session

ext/session/test_session.c

@@ -218,6 +218,7 @@ static int testStreamOutput(
 **          $session indirect INTEGER
 **          $session patchset
 **          $session table_filter SCRIPT
+**          $session fullchangeset
 */
 static int SQLITE_TCLAPI test_session_cmd(
   void *clientData,
@@ -231,17 +232,17 @@ static int SQLITE_TCLAPI test_session_cmd(
     const char *zSub;
     int nArg;
     const char *zMsg;
-    int iSub;
   } aSub[] = {
-    { "attach",       1, "TABLE",      }, /* 0 */
-    { "changeset",    0, "",           }, /* 1 */
-    { "delete",       0, "",           }, /* 2 */
-    { "enable",       1, "BOOL",       }, /* 3 */
-    { "indirect",     1, "BOOL",       }, /* 4 */
-    { "isempty",      0, "",           }, /* 5 */
-    { "table_filter", 1, "SCRIPT",     }, /* 6 */
+    { "attach",       1, "TABLE"       }, /* 0 */
+    { "changeset",    0, ""            }, /* 1 */
+    { "delete",       0, ""            }, /* 2 */
+    { "enable",       1, "BOOL"        }, /* 3 */
+    { "indirect",     1, "BOOL"        }, /* 4 */
+    { "isempty",      0, ""            }, /* 5 */
+    { "table_filter", 1, "SCRIPT"      }, /* 6 */
     { "patchset",     0, "",           }, /* 7 */
-    { "diff",         2, "FROMDB TBL", }, /* 8 */
+    { "diff",         2, "FROMDB TBL"  }, /* 8 */
+    { "fullchangeset",0, ""            }, /* 9 */
     { 0 }
   };
   int iSub;
@@ -271,10 +272,11 @@ static int SQLITE_TCLAPI test_session_cmd(
       break;
     }
 
+    case 9:        /* fullchangeset */
     case 7:        /* patchset */
     case 1: {      /* changeset */
       TestSessionsBlob o = {0, 0};
-      if( test_tcl_integer(interp, SESSION_STREAM_TCL_VAR) ){
+      if( iSub!=9 && test_tcl_integer(interp, SESSION_STREAM_TCL_VAR) ){
         void *pCtx = (void*)&o;
         if( iSub==7 ){
           rc = sqlite3session_patchset_strm(pSession, testStreamOutput, pCtx);
@@ -284,6 +286,8 @@ static int SQLITE_TCLAPI test_session_cmd(
       }else{
         if( iSub==7 ){
           rc = sqlite3session_patchset(pSession, &o.n, &o.p);
+        }else if( iSub==9 ){
+          rc = sqlite3session_fullchangeset(pSession, &o.n, &o.p);
         }else{
           rc = sqlite3session_changeset(pSession, &o.n, &o.p);
         }
@@ -298,6 +302,7 @@ static int SQLITE_TCLAPI test_session_cmd(
       break;
     }
 
+
     case 2:        /* delete */
       Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
       break;
@@ -478,7 +483,7 @@ static int test_filter_handler(
 
   Tcl_DecrRefCount(pEval);
   return res;
-}  
+}
 
 static int test_conflict_handler(
   void *pCtx,                     /* Pointer to TestConflictHandler structure */
@@ -1120,6 +1125,127 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach(
   return TCL_OK;
 }
 
+#include "sqlite3changebatch.h"
+
+typedef struct TestChangebatch TestChangebatch;
+struct TestChangebatch {
+  sqlite3_changebatch *pChangebatch;
+};
+
+/*
+** Tclcmd:  $changebatch add BLOB
+**          $changebatch zero
+**          $changebatch delete
+*/
+static int SQLITE_TCLAPI test_changebatch_cmd(
+  void *clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  TestChangebatch *p = (TestChangebatch*)clientData;
+  sqlite3_changebatch *pChangebatch = p->pChangebatch;
+  struct SessionSubcmd {
+    const char *zSub;
+    int nArg;
+    const char *zMsg;
+    int iSub;
+  } aSub[] = {
+    { "add",          1, "CHANGESET",  }, /* 0 */
+    { "zero",         0, "",           }, /* 1 */
+    { "delete",       0, "",           }, /* 2 */
+    { 0 }
+  };
+  int iSub;
+  int rc;
+
+  if( objc<2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ...");
+    return TCL_ERROR;
+  }
+  rc = Tcl_GetIndexFromObjStruct(interp, 
+      objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub
+  );
+  if( rc!=TCL_OK ) return rc;
+  if( objc!=2+aSub[iSub].nArg ){
+    Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg);
+    return TCL_ERROR;
+  }
+
+  switch( iSub ){
+    case 0: {      /* add */
+      int nArg;
+      unsigned char *pArg = Tcl_GetByteArrayFromObj(objv[2], &nArg);
+      rc = sqlite3changebatch_add(pChangebatch, pArg, nArg);
+      if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){
+        return test_session_error(interp, rc, 0);
+      }else{
+        extern const char *sqlite3ErrName(int);
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+      }
+      break;
+    }
+
+    case 1: {      /* zero */
+      sqlite3changebatch_zero(pChangebatch);
+      break;
+    }
+
+    case 2:        /* delete */
+      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
+      break;
+  }
+
+  return TCL_OK;
+}
+
+static void SQLITE_TCLAPI test_changebatch_del(void *clientData){
+  TestChangebatch *p = (TestChangebatch*)clientData;
+  sqlite3changebatch_delete(p->pChangebatch);
+  ckfree((char*)p);
+}
+
+/*
+** Tclcmd:  sqlite3changebatch CMD DB-HANDLE
+*/
+static int SQLITE_TCLAPI test_sqlite3changebatch(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3 *db;
+  Tcl_CmdInfo info;
+  int rc;                         /* sqlite3session_create() return code */
+  TestChangebatch *p;             /* New wrapper object */
+
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "CMD DB-HANDLE");
+    return TCL_ERROR;
+  }
+
+  if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[2]), &info) ){
+    Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[2]), 0);
+    return TCL_ERROR;
+  }
+  db = *(sqlite3 **)info.objClientData;
+
+  p = (TestChangebatch*)ckalloc(sizeof(TestChangebatch));
+  memset(p, 0, sizeof(TestChangebatch));
+  rc = sqlite3changebatch_new(db, &p->pChangebatch);
+  if( rc!=SQLITE_OK ){
+    ckfree((char*)p);
+    return test_session_error(interp, rc, 0);
+  }
+
+  Tcl_CreateObjCommand(
+      interp, Tcl_GetString(objv[1]), test_changebatch_cmd, (ClientData)p,
+      test_changebatch_del
+  );
+  Tcl_SetObjResult(interp, objv[1]);
+  return TCL_OK;
+}
+
 /*
 ** tclcmd: CMD configure REBASE-BLOB
 ** tclcmd: CMD rebase CHANGESET
@@ -1302,6 +1428,10 @@ int TestSession_Init(Tcl_Interp *interp){
     Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, 0, 0);
   }
 
+
+  Tcl_CreateObjCommand(
+      interp, "sqlite3changebatch", test_sqlite3changebatch, 0, 0
+  );
   return TCL_OK;
 }
 

main.mk

@@ -432,7 +432,8 @@ TESTSRC2 = \
   $(TOP)/ext/async/sqlite3async.c \
   $(TOP)/ext/misc/stmt.c \
   $(TOP)/ext/session/sqlite3session.c \
-  $(TOP)/ext/session/test_session.c
+  $(TOP)/ext/session/sqlite3changebatch.c \
+  $(TOP)/ext/session/test_session.c 
 
 # Header files used by all library source files.
 #
@@ -966,6 +967,9 @@ THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
 threadtest3$(EXE): sqlite3.o $(THREADTEST3_SRC) $(TOP)/src/test_multiplex.c
 	$(TCCX) $(TOP)/test/threadtest3.c $(TOP)/src/test_multiplex.c sqlite3.o -o $@ $(THREADLIB)
 
+bc_test1$(EXE): sqlite3.o $(TOP)/test/bc_test1.c $(TOP)/test/tt3_core.c
+	$(TCCX) $(TOP)/test/bc_test1.c sqlite3.o -o $@ $(THREADLIB)
+
 threadtest: threadtest3$(EXE)
 	./threadtest3$(EXE)
 

manifest

@@ -1,5 +1,5 @@
-C Invoking\sthe\ssqlite3_trace()\sor\ssqlite3_trace_v2()\sinterfaces\scancels\nany\ssqlite3_profile()\sthat\sis\srunning.
-D 2018-12-06T03:59:25.918
+C Merge\slatest\smutexfree-shmlock\schanges\sinto\sthis\sbranch.
+D 2018-12-10T16:53:32.838
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c
@@ -391,6 +391,8 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de
 F ext/rtree/util/randomshape.tcl 54ee03d0d4a1c621806f7f44d5b78d2db8fac26e0e8687c36c4bd0203b27dbff
 F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
 F ext/rtree/visual01.txt e9c2564083bcd30ec51b07f881bffbf0e12b50a3f6fced0c222c5c1d2f94ac66
+F ext/session/changebatch1.test 9ceaa8f7b2a505933e250fbe6cbc550e4ce1e59d
+F ext/session/changebatchfault.test be49c793219bf387ad692a60856b921f0854ad6d
 F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a
 F ext/session/changesetfuzz.c 255826ee367781ec4c29a36c7c63e9ee6387dc03a8346cf240feb8d597958b0e
 F ext/session/changesetfuzz1.test 2e1b90d888fbf0eea5e1bd2f1e527a48cc85f8e0ff75df1ec4e320b21f580b3a
@@ -420,16 +422,18 @@ F ext/session/sessioninvert.test ae1a003a9ab1f8d64227dbb5c3a4c97e65b561b01e7b295
 F ext/session/sessionrebase.test ccfa716b23bd1d3b03217ee58cfd90c78d4b99f53e6a9a2f05e82363b9142810
 F ext/session/sessionstat1.test 218d351cf9fcd6648f125a26b607b140310160184723c2666091b54450a68fb5
 F ext/session/sessionwor.test 07f0b304dc4df5454906069140bf6ec67edcaa3c548f3683354003cf2c22b64a
-F ext/session/sqlite3session.c dbd6f7a89c1236d8e7b58f4e217391670ce91b9e6f85c16ba02832674ef0cf3c
-F ext/session/sqlite3session.h 05351d2f50a1203fdffbeb590fdbbc796c9a6bfcd0c9b26cf6db3854e3eb4294
-F ext/session/test_session.c 98797aba475a799376c9a42214f2d1debf2d0c3cb657d9c8bbf4f70bf3fb4aec
+F ext/session/sqlite3changebatch.c d5553b79e012ee2cb06c0a96bdf9dfe19e66354390ea0036cc46c4953142d517
+F ext/session/sqlite3changebatch.h e72016998c9a22d439ddfd547b69e1ebac810c24
+F ext/session/sqlite3session.c 133b7aec935c77c3ff669e65a85eb763503d1f7107ceb7fcc26df1a6cd8124dc
+F ext/session/sqlite3session.h 2a449bb4ba954dd374bd8524af6187454f98fa1d61d16a9f6709ce0a191cf4f1
+F ext/session/test_session.c 60e15d5db8ae7a0f521e70a7504ba1f74fc50548a25a5397808f487bc6a92b5d
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c f81aa5a3ecacf406f170c62a144405858f6f6de51dbdc0920134e629edbe2648
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 55f94164ecc194b067d9c55e106f37fd3c9b39f9668e8b568c98f008b6f9ec90
+F main.mk 7c4484f59dc2ccf831c9786b9eb4e8131d63b80328e1d92ddefaee6284f26386
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -446,12 +450,12 @@ F src/analyze.c 3dc6b98cf007b005af89df165c966baaa48e8124f38c87b4d2b276fe7f0b9eb9
 F src/attach.c 92b51739a885da8bd84bc9a05485f1e48148bce5c15432f059b45af98fff75cd
 F src/auth.c 0fac71038875693a937e506bceb492c5f136dd7b1249fbd4ae70b4e8da14f9df
 F src/backup.c 78d3cecfbe28230a3a9a1793e2ead609f469be43e8f486ca996006be551857ab
-F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
+F src/bitvec.c 8433d9e98dd6f2ea3286e0d2fe5d65de1bfc18a706486eb2026b01be066b5806
 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6
-F src/btree.c ba7c7eef4461790f37c309936bfc5d0d6ba9b194b02d3c8ff1fd53b420ea6d3b
-F src/btree.h febb2e817be499570b7a2e32a9bbb4b607a9234f6b84bb9ae84916d4806e96f2
-F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96
-F src/build.c ef9d7dc73e40dd9d10c28848343e21e8bc1baaab92cfb75eda893fff4fbf6b55
+F src/btree.c 4a2184be69d491d4b0228d4e397d67cb0802bbec06e7615b485ea1af69a131f6
+F src/btree.h 1ed41c71481a1196a520064f2282bc13d768bbd8ae2850e319a3048f8ee7cb3d
+F src/btreeInt.h 6c65e6c96f561596f6870c79a64d4706af81613881d7947e3f063e923f14115f
+F src/build.c a5402bf34cb8e4cde86f6c6fc0356a5d26a6d561b647464c13ef3366f545b56e
 F src/callback.c 25dda5e1c2334a367b94a64077b1d06b2553369f616261ca6783c48bcb6bda73
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 109e58d00f62e8e71ee1eb5944ac18b90171c928ab2e082e058056e1137cc20b
@@ -462,7 +466,7 @@ F src/delete.c f7938125847e8ef485448db5fbad29acb2991381a02887dd854c1617315ab9fb
 F src/expr.c a3e90131d7f92b77a6a6ae2e71f0d2882237041ed11dbe3c914e5e278ddc2d24
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 972a4ba14296bef2303a0abbad1e3d82bc3c61f9e6ce4e8e9528bdee68748812
-F src/func.c 7c288b4ce309b5a8b8473514b88e1f8e69a80134509a8c0db8e39c858e367e7f
+F src/func.c 8efa2c813b3f6a831a070311b5bcbc97993b79cbcd274bdb49bde56ccd3d37bc
 F src/global.c 8291eee0782b83124de14ec0389ec9fd6ae1873358a6b0d9469fe17a46ad803b
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
@@ -471,7 +475,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c f12f27eb606d601825be9a229a7390a8d64d40226697883f96de8e088d620055
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 9050dd153b5583804184be9c9dee9ebb554178d6db1f8ac280899e8aad9060e6
-F src/main.c 156076e2fc4eea09523323bae1d5f177a95a777a3b22e49e173437556fe3571c
+F src/main.c 11e50602244f0594af3d86f4997a2cc0a36a29b278500ef7bbff89fed084def3
 F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -491,33 +495,33 @@ F src/os.c 8aeb0b0f40f8f5b0da03fe49706695adaf42d2f516ab95abc72e86c245e119de
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c f6e91b8fd82af7afbfd073c4974ad6cdb8e62d9f65ceddb45167835a0567fdc0
+F src/os_unix.c 177e5490d3a3c702fac87ff6dd183b56a24fb5fabe76841c3bf8a03ad85ff43b
 F src/os_win.c 85d9e532d0444ab6c16d7431490c2e279e282aa0917b0e988996b1ae0de5c5a0
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c 75e0f3cfa3962c714f519f8a3d1e67ecca1c91de0e010a036b988e40ce9e4c73
-F src/pager.h 217921e81eb5fe455caa5cda96061959706bcdd29ddb57166198645ef7822ac3
-F src/parse.y 5cf85c2b9dfac38ac4e2bf2776484705186ce2eda8631e65cc0b04bf566c1173
+F src/pager.c 79b8ced46a3c32be19ab4b00d8836292e460411d283a333cb008923c69fbca6a
+F src/pager.h 389ba8f526d13026aa7081dc581aa742eb7207e3277e7106c522c5b65ad92590
+F src/parse.y 9281e9941d5df0f7978c4bfe1521090ba7363ac4535c79f57531ba0194904d43
 F src/pcache.c 696a01f1a6370c1b50a09c15972bc3bee3333f8fcd1f2da8e9a76b1b062c59ee
 F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586
 F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc
 F src/pragma.c 4e056f042683b99c4ea0db395f68d051b1a95833ab40951c40d3ef7e1fee1354
-F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13
+F src/pragma.h 7003ea8e45e5da0a7cd6d35846214f9ae9ecf5be66b268415ceea5855324af11
 F src/prepare.c 66b5f9791a3ef57131cbba58c33104ebea814a70a5cfcafabf5aed5a3e3858fe
 F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381
-F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
+F src/random.c f27af4099afaea7284ade5c206224dcfdb2334cfd119d018b470d46356b3f27d
 F src/resolve.c 976e7879286a1eecdc71ceff64f6d1b3f58c8f8096537ba668b3dc0887f410c1
 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93
-F src/select.c 61e867a906f140b73baf4ce7a201ad6dcba30820969f5618ee40e9a0d32c6f5f
+F src/select.c e1cad752987c476a9d1d053682558caec048ea799ef7ba1b6e38e2266451010d
 F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883d
-F src/sqlite.h.in 908ec406feefc4c7e1486a2e3dc30a8bfb51c5a345a8e8130ac201962db171c4
+F src/sqlite.h.in 334afcaa55742ea7605bd2962c7de658be0c1f2173317a78124e07239d20f322
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683
-F src/sqliteInt.h 97adda953e7f118d47b8135f76f88c6420ff6707285782616f393a9ea255d577
+F src/sqliteInt.h c319213f1263293c6d569f73fc00fe4e7b1f95e20f5b626a624d0119046748de
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c e72862a271348d779672b45a730c33fd0c535e630ff927e8ce4a0c908d1d28c6
-F src/test1.c 5390e5afb31fed61f72d0be0cb1b322d198a6e03fc13ff245ae76d17b4dcf2e9
+F src/test1.c 7928f1eb530dac8da21dc7f1e18935988f1f0e3e2a2d99993b522c4fb201d71b
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644
 F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6
@@ -532,7 +536,7 @@ F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0
 F src/test_bestindex.c 78809f11026f18a93fcfd798d9479cba37e1201c830260bf1edc674b2fa9b857
 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce
 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274
-F src/test_config.c 5ebafbcd5c75ac1c16bb0c8fe926dc325cc03e780943a88ca50e0d9a4fc4d2f5
+F src/test_config.c b0bac20e47824b0959f3c1e77b6b2c72b32b2dc62cd8b78cba57b460682b0284
 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f
 F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e
 F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2
@@ -558,12 +562,12 @@ F src/test_rtree.c 671f3fae50ff116ef2e32a3bf1fe21b5615b4b7b
 F src/test_schema.c f575932cb6274d12147a77e13ea4b49d52408513
 F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe
 F src/test_sqllog.c 11e6ce7575f489155c604ac4b439f2ac1d3d5aef
-F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e
+F src/test_superlock.c f4d4cc7319a608a54b7608158e8c7135fac19b88d6179e5bf17080e89d1f0278
 F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939
 F src/test_tclsh.c 06317648b0d85a85fd823f7973b55535c59a3156c1ef59394fe511f932cfa78d
 F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc
 F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858
-F src/test_vfs.c 112f1f9271c33c211812e0e681830a84262dac065da58579ff49f9cefec97d4f
+F src/test_vfs.c 194b8e78e9b4279bc69693cb5cfc343e0a08482c430a571ec96d80440e337a44
 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698
 F src/test_windirent.c a895e2c068a06644eef91a7f0a32182445a893b9a0f33d0cdb4283dca2486ac1
 F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a90484215
@@ -573,24 +577,24 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 9e781e1ca80eefe7b5d6a9e2cd5c678c847da55fd6f093781fad7950934d4c83
 F src/treeview.c 7b12ac059de54c939b6eb0dbffc9410c29c80d2470cee5cbe07d5ff9ea2d9253
 F src/trigger.c d3d78568f37fb2e6cdcc2d1e7b60156f15b0b600adec55b83c5d42f6cad250bd
-F src/update.c 1816d56c1bca1ba4e0ef98cac2f49be62858e9df1dc08844c7067eb41cc44274
+F src/update.c 31d5208f7c8b129a355ce797bdd2fd70511d18ca6883c77aa816eb4fe486b687
 F src/upsert.c 0dd81b40206841814d46942a7337786932475f085716042d0cb2fc7791bf8ca4
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157
-F src/vacuum.c 836cadc922de866c849e23a75f93d344cdc143d388339305d09a3fed27e8798d
-F src/vdbe.c 005e691ea4c7d51e6c1a69d9389aeb34700884c85f51681817ddea3fdc2fc39b
+F src/vacuum.c 8747a99e0687ae5cb3515b70a9d82bbd20370de9f097c7bd93a392ece3dea03c
+F src/vdbe.c d2672a54cc283e1425cc8c7c45271530b3686b8b5f06bf4426d9a5812e951abe
 F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907
 F src/vdbeInt.h 437e6c6af679fdf157867eb83a8adc6cf5145d6774453c2214cfd0bd01d92980
 F src/vdbeapi.c 9709452bee82963e1f7f1f5d0c71db823d553f8dbb2c47a911c4983d537a1947
-F src/vdbeaux.c 8e2fe020824b743090025ff6f9ffeec3ca4624523ddb2d1af1b1f61abaab3db4
+F src/vdbeaux.c b3c364b1cfb9cda13bf86d571c572ff89ffefa89bbfc9a506731214a004789a8
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9
 F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7f
 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392
 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 3f4f653daf234fe713edbcbca3fec2350417d159d28801feabc702a22c4e213f
-F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a
+F src/wal.c 47e3230db60654d4d332ace2716fe16b99a8574189d80d6367c14f1dd3eb6ab5
+F src/wal.h f325a5856b669f5ba449157485915816103857c8574efc746ac55eba3335c5e0
 F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66
 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e
 F src/whereInt.h f125f29fca80890768e0b2caa14f95db74b2dacd3a122a168f97aa7b64d6968f
@@ -669,6 +673,7 @@ F test/backup_malloc.test 0c9abdf74c51e7bedb66d504cd684f28d4bd4027
 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f
 F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f
 F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c
+F test/bc_test1.c e0a092579552e066ed4ce7bcdaecfa69c4aacc8d
 F test/bestindex1.test 852170bddbb21daa121fabcc274640ff83d7d8705912e8b5fe7ed2c5a9a9224a
 F test/bestindex2.test 9a0ccd320b6525eec3a706aae6cdab7e1b7b5abca75027e39f39f755e76e5928
 F test/bestindex3.test 001788a114ad96d81d5154fe77c7f1e26e84b3a2b5635ca29e4f96f6decc534e
@@ -725,6 +730,14 @@ F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6
 F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95
 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1
 F test/colname.test fb28b3687e03625425bc216edf8b186ce974aa71008e2aa1f426a7dcb75a601d
+F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43
+F test/concurrent.test 86661967a680670127a62a819e60dc93c2d3d49043ac95b26dfa70d3e60dbde5
+F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8
+F test/concurrent3.test 530671ac706f6a1d0f4992dbdd33a86408330d03cd90fb9e82ecb1b27f5fd081
+F test/concurrent4.test e0b12cd467137e50259df3b4f837507e82aaa07c35941c88664dc8ed1d089c44
+F test/concurrent5.test 0c16cbf7446af162a14e6def30445e94016064eb994e5aa4ebb2bebc59554176
+F test/concurrent6.test a7860e9ca13bb5fb76bcf41c5524fbfa9c37e6e258ecf84ffb5748a272488c67
+F test/concurrent7.test b96fa5c4cfdf8d5c0bc66b6934214500bad0260884a736f054ccc76e81aae85d
 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db
 F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c
 F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9
@@ -1078,7 +1091,7 @@ F test/lock4.test 27143363eda1622f03c133efc8db808fc331afd973486cb571ea71cd717d37
 F test/lock5.test c6c5e0ebcb21c61a572870cc86c0cb9f14cede38
 F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5
 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431
-F test/lock_common.tcl 7ffb45accf6ee91c736df9bafe0806a44358f035
+F test/lock_common.tcl 2f3f7f2e9637f93ccf609df48ef5b27a50278b6b1cd752b445d52262e5841413
 F test/lookaside.test 5a828e7256f1ee4da8e1bdaa03373a3ccdb0f1ff98dfa82e9b76cb41a45b1083
 F test/main.test 6bbb3999fd461eb8fb335cbab97409a3d7f91bbb8da60635e8be3e4a04a77772
 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9
@@ -1177,11 +1190,10 @@ F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f
 F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8
 F test/pageropt.test 84e4cc5cbca285357f7906e99b21be4f2bf5abc0
 F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305
-F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035ce4b3
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test cf0b4e498db1d0143c19641d4420df7cc27fab2c95ed0abd2c7c5753beab25b8
+F test/permutations.test 455862c587efbe2b2eb3a1770c602b1c97243fe78e09b9c222c9e03a5828444c
 F test/pg_common.tcl 301ac19c1a52fd55166d26db929b3b89165c634d52b5f8ad76ea8cb06960db30
 F test/pragma.test c267bf02742c823a191960895b3d52933cebd7beee26757d1ed694f213fcd867
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
@@ -1285,6 +1297,7 @@ F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35
 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
 F test/shell8.test 96be02ea0c21f05b24c1883d7b711a1fa8525a68ab7b636aacf6057876941013
+F test/shmlock.test 3d1868f0386923c0592a235f2dd87ae52286b217bc695fbfd9d39a828e7be374
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1368,7 +1381,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46
 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd
 F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b
 F test/threadtest2.c a70a8e94bef23339d34226eb9521015ef99f4df8
-F test/threadtest3.c 38a612ea62854349ed66372f330a40d73c5cf956
+F test/threadtest3.c 4b413718ab3ef5e4f9f1ab0502b89707c5a9ccdc
 F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925
 F test/time-wordcount.sh 8e0b0f8109367827ad5d58f5cc849705731e4b90
 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c
@@ -1538,6 +1551,7 @@ F test/triggerE.test ede2e4bce4ba802337bd69d39447fa04a938e06d84a8bfc53c76850fc36
 F test/triggerF.test 6a8c22bd058cf467f0c7d112afe87f7a8c579c0c4681b914b8f19020f48528a4
 F test/triggerG.test d5caeef6144ede2426dd13211fd72248241ff2ebc68e12a4c0bf30f5faa21499
 F test/tt3_checkpoint.c 9e75cf7c1c364f52e1c47fd0f14c4340a9db0fe1
+F test/tt3_core.c 8cd89ead95410f70e7fb02c79f1e040f9c5ad5cf
 F test/tt3_index.c 39eec10a35f57672225be4d182862152896dee4a
 F test/tt3_lookaside1.c 0377e202c3c2a50d688cb65ba203afeda6fafeb9
 F test/tt3_stress.c c57d804716165811d979d4a719e05baccd79277f
@@ -1592,8 +1606,8 @@ F test/vtabJ.test d7b73675708cf63cfcb9d443bb451fc01a028347275b7311e51f9fdf3ca675
 F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c3784c6783
 F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bda2b65
 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad
-F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477
-F test/wal2.test 155b9efa999bdb38ce1cd729b9a4fcdbffd6b88be27f039bad1d2929d287d918
+F test/wal.test cdf0ca6cc0447520d19ef1c83287824ebeb3e82d75af856511ba96841a79fc9b
+F test/wal2.test a225bafac35a47765b890bacdeb57e5e81039f21cc18a1e8ce88eb76e56b843c
 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
 F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9
@@ -1602,7 +1616,7 @@ F test/wal64k.test 2a525c0f45d709bae3765c71045ccec5df7d100ccbd3a7860fdba46c9addb
 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd
 F test/wal8.test d9df3fba4caad5854ed69ed673c68482514203c8
 F test/wal9.test 378e76a9ad09cd9bee06c172ad3547b0129a6750
-F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe
+F test/wal_common.tcl 37902864b63794e9bf59393b8887e21a9cbd4db8
 F test/walbak.test 018d4e5a3d45c6298d11b99f09a8ef6876527946
 F test/walbig.test f437473a16cfb314867c6b5d1dbcd519e73e3434
 F test/walblock.test be48f3a75eff0b4456209f26b3ce186c2015497d
@@ -1686,7 +1700,7 @@ F test/zipfilefault.test 44d4d7a7f7cca7521d569d7f71026b241d65a6b1757aa409c1a1688
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
-F tool/addopcodes.tcl 0288d5b26b9b35f4cb5affb76eec63f1dfce117bbc2020066708069ef60b86ff
+F tool/addopcodes.tcl 251331be53533f11d9ba36c730b9baaa911df89a08da3043344d0d634dc4a48e
 F tool/build-all-msvc.bat c12328d06c45fec8baada5949e3d5af54bf8c887 x
 F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367
 F tool/cg_anno.tcl f95b0006c52cf7f0496b506343415b6ee3cdcdd3 x
@@ -1710,12 +1724,12 @@ F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439
 F tool/mkautoconfamal.sh 422fc365358a2e92876ffc62971a0ff28ed472fc8bcf9de0df921c736fdeca5e
 F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x
 F tool/mkctimec.tcl dd183b73ae1c28249669741c250525f0407e579a70482371668fd5f130d9feb3
-F tool/mkkeywordhash.c 1f7f2ac1d9f262c08b67faaca47e6a68262ff39113fa4b27d1db2843b4c33e73
+F tool/mkkeywordhash.c 1e5cfcf4cd692ae21e7555604218097a1ee79dad3e366496e5236876255bfcc1
 F tool/mkmsvcmin.tcl cad0c7b54d7dd92bc87d59f36d4cc4f070eb2e625f14159dc2f5c4204e6a13ea
 F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c
 F tool/mkopcodeh.tcl 352a4319c0ad869eb26442bf7c3b015aa15594c21f1cce5a6420dbe999367c21
 F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa
-F tool/mkpragmatab.tcl a1334e70a08fdf5de32cd0093613212bb11ac8f880487540987175c536ac335f
+F tool/mkpragmatab.tcl d5112b04ad7bd35a09a16fda9d968292fd00e11b0a5b32b0664ff6e444a5a9b3
 F tool/mkshellc.tcl 1f45770aea226ac093a9c72f718efbb88a2a2833409ec2e1c4cecae4202626f5
 F tool/mksourceid.c d458f9004c837bee87a6382228ac20d3eae3c49ea3b0a5aace936f8b60748d3b
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
@@ -1782,7 +1796,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3a2c047989facc3461c63a2f9eed412014c951035a80da47c52a70139fb552de
-R e33573679d2b83476b49b39d100df865
-U drh
-Z 9b63335dca170db3a75ac781c84ad8b2
+P 6007abfe0935a47edc90ef34a4f0a8c4263365b4e2e790e015a195be87c9f6b4 b9a741519055a8560ec1d0ab7d0c15ff8db53e740011825506cd2ede11c956f0
+R 0d4f69b7321e017a524ef7e6c799dc7e
+U dan
+Z 7b310a2bc179bf5031c1fdfa8fe699cc

manifest.uuid

@@ -1 +1 @@
-ec63d3506bd429560077f82a4c5ed9d189780789fe1c134fff4f3b8733be1a3f
+186b376ec3713f9c8a1499a9675b3c1b42ef3a123848506a48d3a196ba6461f3

src/bitvec.c

@@ -171,6 +171,12 @@ int sqlite3BitvecSet(Bitvec *p, u32 i){
   if( p==0 ) return SQLITE_OK;
   assert( i>0 );
   assert( i<=p->iSize );
+  if( i>p->iSize || i==0 ){
+    sqlite3_log(SQLITE_ERROR, 
+        "Bitvec: setting bit %d of bitvec size %d\n", (int)i, (int)p->iSize
+    );
+    abort();
+  }
   i--;
   while((p->iSize > BITVEC_NBIT) && p->iDivisor) {
     u32 bin = i/p->iDivisor;

src/btree.c

@@ -467,7 +467,240 @@ static void downgradeAllSharedCacheTableLocks(Btree *p){
 
 #endif /* SQLITE_OMIT_SHARED_CACHE */
 
-static void releasePage(MemPage *pPage);         /* Forward reference */
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** The following structure - BtreePtrmap - stores the in-memory pointer map
+** used for newly allocated pages in CONCURRENT transactions. Such pages are
+** always allocated in a contiguous block (from the end of the file) starting
+** with page BtreePtrmap.iFirst.
+*/
+typedef struct RollbackEntry RollbackEntry;
+typedef struct PtrmapEntry PtrmapEntry;
+struct PtrmapEntry {
+  Pgno parent;
+  u8 eType;
+};
+struct RollbackEntry {
+  Pgno pgno;
+  Pgno parent;
+  u8 eType;
+};
+struct BtreePtrmap {
+  Pgno iFirst;                    /* First new page number aPtr[0] */
+
+  int nPtrAlloc;                  /* Allocated size of aPtr[] array */
+  PtrmapEntry *aPtr;              /* Array of parent page numbers */
+
+  int nSvpt;                      /* Used size of aSvpt[] array */
+  int nSvptAlloc;                 /* Allocated size of aSvpt[] */
+  int *aSvpt;                     /* First aRollback[] entry for savepoint i */
+
+  int nRollback;                  /* Used size of aRollback[] array */
+  int nRollbackAlloc;             /* Allocated size of aRollback[] array */
+  RollbackEntry *aRollback;       /* Array of rollback entries */
+};
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** If page number pgno is greater than or equal to BtreePtrmap.iFirst, 
+** store an entry for it in the pointer-map structure.
+*/
+static int btreePtrmapStore(
+  BtShared *pBt,
+  Pgno pgno,
+  u8 eType, 
+  Pgno parent
+){
+  BtreePtrmap *pMap = pBt->pMap;
+  if( pgno>=pMap->iFirst ){
+    int iEntry = pgno - pMap->iFirst;
+
+    /* Grow the aPtr[] array as required */
+    while( iEntry>=pMap->nPtrAlloc ){
+      int nNew = pMap->nPtrAlloc ? pMap->nPtrAlloc*2 : 16;
+      PtrmapEntry *aNew = (PtrmapEntry*)sqlite3_realloc(
+          pMap->aPtr, nNew*sizeof(PtrmapEntry)
+      );
+      if( aNew==0 ){
+        return SQLITE_NOMEM;
+      }else{
+        int nByte = (nNew-pMap->nPtrAlloc)*sizeof(PtrmapEntry);
+        memset(&aNew[pMap->nPtrAlloc], 0, nByte);
+        pMap->aPtr = aNew;
+        pMap->nPtrAlloc = nNew;
+      }
+    }
+
+    /* Add an entry to the rollback log if required */
+    if( pMap->nSvpt>0 && pMap->aPtr[iEntry].parent ){
+      if( pMap->nRollback>=pMap->nRollbackAlloc ){
+        int nNew = pMap->nRollback ? pMap->nRollback*2 : 16;
+        RollbackEntry *aNew = (RollbackEntry*)sqlite3_realloc(
+            pMap->aRollback, nNew*sizeof(RollbackEntry)
+        );
+        if( aNew==0 ){
+          return SQLITE_NOMEM;
+        }else{
+          pMap->aRollback = aNew;
+          pMap->nRollbackAlloc = nNew;
+        }
+      }
+
+      pMap->aRollback[pMap->nRollback].pgno = pgno;
+      pMap->aRollback[pMap->nRollback].parent = pMap->aPtr[iEntry].parent;
+      pMap->aRollback[pMap->nRollback].eType = pMap->aPtr[iEntry].eType;
+      pMap->nRollback++;
+    }
+
+    /* Update the aPtr[] array */
+    pMap->aPtr[iEntry].parent = parent;
+    pMap->aPtr[iEntry].eType = eType;
+  }
+
+  return SQLITE_OK;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Open savepoint iSavepoint, if it is not already open.
+*/
+static int btreePtrmapBegin(BtShared *pBt, int nSvpt){
+  BtreePtrmap *pMap = pBt->pMap;
+  if( pMap && nSvpt>pMap->nSvpt ){
+    int i;
+    if( nSvpt>=pMap->nSvptAlloc ){
+      int nNew = pMap->nSvptAlloc ? pMap->nSvptAlloc*2 : 16;
+      int *aNew = sqlite3_realloc(pMap->aSvpt, sizeof(int) * nNew);
+      if( aNew==0 ){
+        return SQLITE_NOMEM;
+      }else{
+        pMap->aSvpt = aNew;
+        pMap->nSvptAlloc = nNew;
+      }
+    }
+
+    for(i=pMap->nSvpt; i<nSvpt; i++){
+      pMap->aSvpt[i] = pMap->nRollback;
+    }
+    pMap->nSvpt = nSvpt;
+  }
+
+  return SQLITE_OK;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Rollback (if op==SAVEPOINT_ROLLBACK) or release (if op==SAVEPOINT_RELEASE)
+** savepoint iSvpt.
+*/
+static void btreePtrmapEnd(BtShared *pBt, int op, int iSvpt){
+  BtreePtrmap *pMap = pBt->pMap;
+  if( pMap ){
+    assert( op==SAVEPOINT_ROLLBACK || op==SAVEPOINT_RELEASE );
+    assert( iSvpt>=0 || (iSvpt==-1 && op==SAVEPOINT_ROLLBACK) );
+    if( iSvpt<0 ){
+      pMap->nSvpt = 0;
+      pMap->nRollback = 0;
+      memset(pMap->aPtr, 0, sizeof(Pgno) * pMap->nPtrAlloc);
+    }else if( iSvpt<pMap->nSvpt ){
+      if( op==SAVEPOINT_ROLLBACK ){
+        int ii;
+        for(ii=pMap->nRollback-1; ii>=pMap->aSvpt[iSvpt]; ii--){
+          RollbackEntry *p = &pMap->aRollback[ii];
+          PtrmapEntry *pEntry = &pMap->aPtr[p->pgno - pMap->iFirst];
+          pEntry->parent = p->parent;
+          pEntry->eType = p->eType;
+        }
+      }
+      pMap->nSvpt = iSvpt + (op==SAVEPOINT_ROLLBACK);
+      pMap->nRollback = pMap->aSvpt[iSvpt];
+    }
+  }
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** This function is called after an CONCURRENT transaction is opened on the
+** database. It allocates the BtreePtrmap structure used to track pointers
+** to allocated pages and zeroes the nFree/iTrunk fields in the database 
+** header on page 1.
+*/
+static int btreePtrmapAllocate(BtShared *pBt){
+  int rc = SQLITE_OK;
+  if( pBt->pMap==0 ){
+    BtreePtrmap *pMap = sqlite3_malloc(sizeof(BtreePtrmap));
+    if( pMap==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      memset(&pBt->pPage1->aData[32], 0, sizeof(u32)*2);
+      memset(pMap, 0, sizeof(BtreePtrmap));
+      pMap->iFirst = pBt->nPage + 1;
+      pBt->pMap = pMap;
+    }
+  }
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Free any BtreePtrmap structure allocated by an earlier call to
+** btreePtrmapAllocate().
+*/
+static void btreePtrmapDelete(BtShared *pBt){
+  BtreePtrmap *pMap = pBt->pMap;
+  if( pMap ){
+    sqlite3_free(pMap->aRollback);
+    sqlite3_free(pMap->aPtr);
+    sqlite3_free(pMap->aSvpt);
+    sqlite3_free(pMap);
+    pBt->pMap = 0;
+  }
+}
+
+/*
+** Check that the pointer-map does not contain any entries with a parent
+** page of 0. Call sqlite3_log() multiple times to output the entire
+** data structure if it does.
+*/
+static void btreePtrmapCheck(BtShared *pBt, Pgno nPage){
+  Pgno i;
+  int bProblem = 0;
+  BtreePtrmap *p = pBt->pMap;
+
+  for(i=p->iFirst; i<=nPage; i++){
+    PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst];
+    if( pEntry->eType==PTRMAP_OVERFLOW1
+     || pEntry->eType==PTRMAP_OVERFLOW2
+     || pEntry->eType==PTRMAP_BTREE
+    ){
+      if( pEntry->parent==0 ){
+        bProblem = 1;
+        break;
+      }
+    }
+  }
+
+  if( bProblem ){
+    for(i=p->iFirst; i<=nPage; i++){
+      PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst];
+      sqlite3_log(SQLITE_CORRUPT, 
+          "btreePtrmapCheck: pgno=%d eType=%d parent=%d", 
+          (int)i, (int)pEntry->eType, (int)pEntry->parent
+      );
+    }
+    abort();
+  }
+}
+
+#else  /* SQLITE_OMIT_CONCURRENT */
+# define btreePtrmapAllocate(x) SQLITE_OK
+# define btreePtrmapDelete(x) 
+# define btreePtrmapBegin(x,y)  SQLITE_OK
+# define btreePtrmapEnd(x,y,z) 
+# define btreePtrmapCheck(y,z) 
+#endif /* SQLITE_OMIT_CONCURRENT */
+
+static void releasePage(MemPage *pPage);  /* Forward reference */
 static void releasePageOne(MemPage *pPage);      /* Forward reference */
 static void releasePageNotNull(MemPage *pPage);  /* Forward reference */
 
@@ -978,9 +1211,16 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){
   if( *pRC ) return;
 
   assert( sqlite3_mutex_held(pBt->mutex) );
-  /* The master-journal page number must never be used as a pointer map page */
+  /* The master-journal page number is never added to a pointer-map page */
   assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) );
 
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( pBt->pMap ){
+    *pRC = btreePtrmapStore(pBt, key, eType, parent);
+    return;
+  }
+#endif
+
   assert( pBt->autoVacuum );
   if( key==0 ){
     *pRC = SQLITE_CORRUPT_BKPT;
@@ -2142,6 +2382,17 @@ getAndInitPage_error:
   return rc;
 }
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/* 
+** Set the value of the MemPage.pgnoRoot variable, if it exists.
+*/
+static void setMempageRoot(MemPage *pPg, u32 pgnoRoot){
+  pPg->pgnoRoot = pgnoRoot;
+}
+#else
+# define setMempageRoot(x,y)
+#endif
+
 /*
 ** Release a MemPage.  This should be called once for each prior
 ** call to btreeGetPage.
@@ -3303,6 +3554,7 @@ int sqlite3BtreeNewDb(Btree *p){
 int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){
   BtShared *pBt = p->pBt;
   int rc = SQLITE_OK;
+  int bConcurrent = (p->db->bConcurrent && !ISAUTOVACUUM);
 
   sqlite3BtreeEnter(p);
   btreeIntegrity(p);
@@ -3378,7 +3630,8 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){
       if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){
         rc = SQLITE_READONLY;
       }else{
-        rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db));
+        int exFlag = bConcurrent ? -1 : (wrflag>1);
+        rc = sqlite3PagerBegin(pBt->pPager, exFlag, sqlite3TempInMemory(p->db));
         if( rc==SQLITE_OK ){
           rc = newDatabase(pBt);
         }else if( rc==SQLITE_BUSY_SNAPSHOT && pBt->inTransaction==TRANS_NONE ){
@@ -3438,6 +3691,15 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){
   }
 
 trans_begun:
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( bConcurrent && rc==SQLITE_OK && sqlite3PagerIsWal(pBt->pPager) ){
+    rc = sqlite3PagerBeginConcurrent(pBt->pPager);
+    if( rc==SQLITE_OK && wrflag ){
+      rc = btreePtrmapAllocate(pBt);
+    }
+  }
+#endif
+
   if( rc==SQLITE_OK ){
     if( pSchemaVersion ){
       *pSchemaVersion = get4byte(&pBt->pPage1->aData[40]);
@@ -3447,7 +3709,11 @@ trans_begun:
       ** open savepoints. If the second parameter is greater than 0 and
       ** the sub-journal is not already open, then it will be opened here.
       */
-      rc = sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint);
+      int nSavepoint = p->db->nSavepoint;
+      rc = sqlite3PagerOpenSavepoint(pBt->pPager, nSavepoint);
+      if( rc==SQLITE_OK && nSavepoint ){
+        rc = btreePtrmapBegin(pBt, nSavepoint);
+      }
     }
   }
 
@@ -3880,6 +4146,175 @@ static int autoVacuumCommit(BtShared *pBt){
 # define setChildPtrmaps(x) SQLITE_OK
 #endif
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** This function is called as part of merging an CONCURRENT transaction with
+** the snapshot at the head of the wal file. It relocates all pages in the
+** range iFirst..iLast, inclusive. It is assumed that the BtreePtrmap 
+** structure at BtShared.pMap contains the location of the pointers to each
+** page in the range.
+**
+** If pnCurrent is NULL, then all pages in the range are moved to currently
+** free locations (i.e. free-list entries) within the database file before page
+** iFirst.
+**
+** Or, if pnCurrent is not NULL, then it points to a value containing the
+** current size of the database file in pages. In this case, all pages are
+** relocated to the end of the database file - page iFirst is relocated to
+** page (*pnCurrent+1), page iFirst+1 to page (*pnCurrent+2), and so on.
+** Value *pnCurrent is set to the new size of the database before this 
+** function returns.
+**
+** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error code.
+*/
+static int btreeRelocateRange(
+  BtShared *pBt,                  /* B-tree handle */
+  Pgno iFirst,                    /* First page to relocate */
+  Pgno iLast,                     /* Last page to relocate */
+  Pgno *pnCurrent                 /* If not NULL, IN/OUT: Database size */
+){
+  int rc = SQLITE_OK;
+  BtreePtrmap *pMap = pBt->pMap;
+  Pgno iPg;
+
+  for(iPg=iFirst; iPg<=iLast && rc==SQLITE_OK; iPg++){
+    MemPage *pFree = 0;     /* Page allocated from free-list */
+    MemPage *pPg = 0;
+    Pgno iNew;              /* New page number for pPg */
+    PtrmapEntry *pEntry;    /* Pointer map entry for page iPg */
+
+    if( iPg==PENDING_BYTE_PAGE(pBt) ) continue;
+    pEntry = &pMap->aPtr[iPg - pMap->iFirst];
+
+    if( pEntry->eType==PTRMAP_FREEPAGE ){
+      Pgno dummy;
+      rc = allocateBtreePage(pBt, &pFree, &dummy, iPg, BTALLOC_EXACT);
+      if( pFree ){
+        assert( sqlite3PagerPageRefcount(pFree->pDbPage)==1 );
+        sqlite3PcacheDrop(pFree->pDbPage);
+      }
+      assert( rc!=SQLITE_OK || dummy==iPg );
+    }else if( pnCurrent ){
+      btreeGetPage(pBt, iPg, &pPg, 0);
+      assert( sqlite3PagerIswriteable(pPg->pDbPage) );
+      assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 );
+      iNew = ++(*pnCurrent);
+      if( iNew==PENDING_BYTE_PAGE(pBt) ) iNew = ++(*pnCurrent);
+      rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent, iNew, 1);
+      releasePageNotNull(pPg);
+    }else{
+      rc = allocateBtreePage(pBt, &pFree, &iNew, iFirst-1, BTALLOC_LE);
+      assert( rc!=SQLITE_OK || iNew<iFirst );
+      if( rc==SQLITE_OK ){
+        releasePage(pFree);
+        btreeGetPage(pBt, iPg, &pPg, 0);
+        rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent,iNew,1);
+        releasePage(pPg);
+      }
+    }
+  }
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** The b-tree handle passed as the only argument is about to commit an
+** CONCURRENT transaction. At this point it is guaranteed that this is 
+** possible - the wal WRITER lock is held and it is known that there are 
+** no conflicts with committed transactions.
+*/
+static int btreeFixUnlocked(Btree *p){
+  BtShared *pBt = p->pBt;
+  MemPage *pPage1 = pBt->pPage1;
+  u8 *p1 = pPage1->aData;
+  Pager *pPager = pBt->pPager;
+  int rc = SQLITE_OK;
+
+  /* If page 1 of the database is not writable, then no pages were allocated
+  ** or freed by this transaction. In this case no special handling is 
+  ** required. Otherwise, if page 1 is dirty, proceed.  */
+  BtreePtrmap *pMap = pBt->pMap;
+  Pgno iTrunk = get4byte(&p1[32]);
+  Pgno nPage = btreePagecount(pBt);
+  u32 nFree = get4byte(&p1[36]);
+
+  assert( pBt->pMap );
+  rc = sqlite3PagerUpgradeSnapshot(pPager, pPage1->pDbPage);
+  assert( p1==pPage1->aData );
+
+  if( rc==SQLITE_OK ){
+    Pgno nHPage = get4byte(&p1[28]);
+    Pgno nFin = nHPage;         /* Size of db after transaction merge */
+
+    if( sqlite3PagerIswriteable(pPage1->pDbPage) ){
+      Pgno iHTrunk = get4byte(&p1[32]);
+      u32 nHFree = get4byte(&p1[36]);
+
+      btreePtrmapCheck(pBt, nPage);
+
+      /* Attach the head database free list to the end of the current
+      ** transactions free-list (if any).  */
+      if( iTrunk!=0 ){
+        put4byte(&p1[36], nHFree + nFree);
+        put4byte(&p1[32], iTrunk);
+        while( iTrunk ){
+          DbPage *pTrunk = sqlite3PagerLookup(pPager, iTrunk);
+          iTrunk = get4byte((u8*)pTrunk->pData);
+          if( iTrunk==0 ){
+            put4byte((u8*)pTrunk->pData, iHTrunk);
+          }
+          sqlite3PagerUnref(pTrunk);
+        };
+      }
+
+      if( nHPage<(pMap->iFirst-1) ){
+        /* The database consisted of (pMap->iFirst-1) pages when the current
+        ** concurrent transaction was opened. And an concurrent transaction may
+        ** not be executed on an auto-vacuum database - so the db should 
+        ** not have shrunk since the transaction was opened. Therefore nHPage
+        ** should be set to (pMap->iFirst-1) or greater. */
+        rc = SQLITE_CORRUPT_BKPT;
+      }else{
+        /* The current transaction allocated pages pMap->iFirst through
+        ** nPage (inclusive) at the end of the database file. Meanwhile,
+        ** other transactions have allocated (iFirst..nHPage). So move
+        ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1). */
+        Pgno iLast = MIN(nPage, nHPage);    /* Last page to move */
+        Pgno nCurrent;                      /* Current size of db */
+
+        nCurrent = MAX(nPage, nHPage);
+        pBt->nPage = nCurrent;
+        rc = btreeRelocateRange(pBt, pMap->iFirst, iLast, &nCurrent);
+
+        /* There are now no collisions with the snapshot at the head of the
+        ** database file. So at this point it would be possible to write
+        ** the transaction out to disk. Before doing so though, attempt to
+        ** relocate some of the new pages to free locations within the body
+        ** of the database file (i.e. free-list entries). */
+        if( rc==SQLITE_OK ){
+          assert( nCurrent!=PENDING_BYTE_PAGE(pBt) );
+          sqlite3PagerSetDbsize(pBt->pPager, nCurrent);
+          nFree = get4byte(&p1[36]);
+          nFin = nCurrent-nFree;
+          if( nCurrent>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){
+            nFin--;
+          }
+          nFin = MAX(nFin, nHPage);
+          rc = btreeRelocateRange(pBt, nFin+1, nCurrent, 0);
+        }
+
+        put4byte(&p1[28], nFin);
+      }
+    }
+    sqlite3PagerSetDbsize(pPager, nFin);
+  }
+
+  return rc;
+}
+#else
+# define btreeFixUnlocked(X)  SQLITE_OK
+#endif /* SQLITE_OMIT_CONCURRENT */
+
 /*
 ** This routine does the first phase of a two-phase commit.  This routine
 ** causes a rollback journal to be created (if it does not already exist)
@@ -3911,8 +4346,10 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){
   if( p->inTrans==TRANS_WRITE ){
     BtShared *pBt = p->pBt;
     sqlite3BtreeEnter(p);
+
 #ifndef SQLITE_OMIT_AUTOVACUUM
     if( pBt->autoVacuum ){
+      assert( ISCONCURRENT==0 );
       rc = autoVacuumCommit(pBt);
       if( rc!=SQLITE_OK ){
         sqlite3BtreeLeave(p);
@@ -3923,7 +4360,12 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){
       sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage);
     }
 #endif
-    rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0);
+    if( rc==SQLITE_OK && ISCONCURRENT ){
+      rc = btreeFixUnlocked(p);
+    }
+    if( rc==SQLITE_OK ){
+      rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0);
+    }
     sqlite3BtreeLeave(p);
   }
   return rc;
@@ -3966,6 +4408,11 @@ static void btreeEndTransaction(Btree *p){
     unlockBtreeIfUnused(pBt);
   }
 
+  /* If this was an CONCURRENT transaction, delete the pBt->pMap object.
+  ** Also call PagerEndConcurrent() to ensure that the pager has discarded
+  ** the record of all pages read within the transaction.  */
+  btreePtrmapDelete(pBt);
+  sqlite3PagerEndConcurrent(pBt->pPager);
   btreeIntegrity(p);
 }
 
@@ -4187,6 +4634,9 @@ int sqlite3BtreeBeginStmt(Btree *p, int iStatement){
   ** such savepoints while the statement transaction savepoint is active.
   */
   rc = sqlite3PagerOpenSavepoint(pBt->pPager, iStatement);
+  if( rc==SQLITE_OK ){
+    rc = btreePtrmapBegin(pBt, iStatement);
+  }
   sqlite3BtreeLeave(p);
   return rc;
 }
@@ -4210,6 +4660,7 @@ int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){
     assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK );
     assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) );
     sqlite3BtreeEnter(p);
+    btreePtrmapEnd(pBt, op, iSavepoint);
     if( op==SAVEPOINT_ROLLBACK ){
       rc = saveAllCursors(pBt, 0, 0);
     }
@@ -4951,6 +5402,7 @@ const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){
 */
 static int moveToChild(BtCursor *pCur, u32 newPgno){
   BtShared *pBt = pCur->pBt;
+  int rc;
 
   assert( cursorOwnsBtShared(pCur) );
   assert( pCur->eState==CURSOR_VALID );
@@ -4965,7 +5417,12 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){
   pCur->apPage[pCur->iPage] = pCur->pPage;
   pCur->ix = 0;
   pCur->iPage++;
-  return getAndInitPage(pBt, newPgno, &pCur->pPage, pCur, pCur->curPagerFlags);
+  rc = getAndInitPage(pBt, newPgno, &pCur->pPage,
+                        pCur, pCur->curPagerFlags);
+  if( rc==SQLITE_OK ){
+    setMempageRoot(pCur->pPage, pCur->pgnoRoot);
+  }
+  return rc;
 }
 
 #ifdef SQLITE_DEBUG
@@ -5077,6 +5534,7 @@ static int moveToRoot(BtCursor *pCur){
       pCur->eState = CURSOR_INVALID;
       return rc;
     }
+    setMempageRoot(pCur->pPage, pCur->pgnoRoot);
     pCur->iPage = 0;
     pCur->curIntKey = pCur->pPage->intKey;
   }
@@ -5808,7 +6266,7 @@ static int allocateBtreePage(
   Pgno mxPage;     /* Total size of the database file */
 
   assert( sqlite3_mutex_held(pBt->mutex) );
-  assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) );
+  assert( eMode==BTALLOC_ANY || (nearby>0 && REQUIRE_PTRMAP ) );
   pPage1 = pBt->pPage1;
   mxPage = btreePagecount(pBt);
   /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36
@@ -5818,6 +6276,15 @@ static int allocateBtreePage(
   if( n>=mxPage ){
     return SQLITE_CORRUPT_BKPT;
   }
+
+  /* Ensure page 1 is writable. This function will either change the number
+  ** of pages in the free-list or the size of the database file. Since both
+  ** of these operations involve modifying page 1 header fields, page 1
+  ** will definitely be written by this transaction. If this is an CONCURRENT
+  ** transaction, ensure the BtreePtrmap structure has been allocated.  */
+  rc = sqlite3PagerWrite(pPage1->pDbPage);
+  if( rc ) return rc;
+
   if( n>0 ){
     /* There are pages on the freelist.  Reuse one of those pages. */
     Pgno iTrunk;
@@ -5828,28 +6295,29 @@ static int allocateBtreePage(
     ** shows that the page 'nearby' is somewhere on the free-list, then
     ** the entire-list will be searched for that page.
     */
-#ifndef SQLITE_OMIT_AUTOVACUUM
     if( eMode==BTALLOC_EXACT ){
-      if( nearby<=mxPage ){
-        u8 eType;
-        assert( nearby>0 );
-        assert( pBt->autoVacuum );
-        rc = ptrmapGet(pBt, nearby, &eType, 0);
-        if( rc ) return rc;
-        if( eType==PTRMAP_FREEPAGE ){
-          searchList = 1;
+      assert( ISAUTOVACUUM!=ISCONCURRENT );
+      if( ISAUTOVACUUM ){
+        if( nearby<=mxPage ){
+          u8 eType;
+          assert( nearby>0 );
+          assert( pBt->autoVacuum );
+          rc = ptrmapGet(pBt, nearby, &eType, 0);
+          if( rc ) return rc;
+          if( eType==PTRMAP_FREEPAGE ){
+            searchList = 1;
+          }
         }
+      }else{
+        searchList = 1;
       }
     }else if( eMode==BTALLOC_LE ){
       searchList = 1;
     }
-#endif
 
     /* Decrement the free-list count by 1. Set iTrunk to the index of the
     ** first free-list trunk page. iPrevTrunk is initially 1.
     */
-    rc = sqlite3PagerWrite(pPage1->pDbPage);
-    if( rc ) return rc;
     put4byte(&pPage1->aData[36], n-1);
 
     /* The code within this loop is run only once if the 'searchList' variable
@@ -6155,7 +6623,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){
   /* If the database supports auto-vacuum, write an entry in the pointer-map
   ** to indicate that the page is free.
   */
-  if( ISAUTOVACUUM ){
+  if( REQUIRE_PTRMAP ){
     ptrmapPut(pBt, iPage, PTRMAP_FREEPAGE, 0, &rc);
     if( rc ) goto freepage_out;
   }
@@ -6471,7 +6939,7 @@ static int fillInCell(
       }
 #endif
       rc = allocateBtreePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl, 0);
-#ifndef SQLITE_OMIT_AUTOVACUUM
+
       /* If the database supports auto-vacuum, and the second or subsequent
       ** overflow page is being allocated, add an entry to the pointer-map
       ** for that page now. 
@@ -6482,14 +6950,13 @@ static int fillInCell(
       ** may misinterpret the uninitialized values and delete the
       ** wrong pages from the database.
       */
-      if( pBt->autoVacuum && rc==SQLITE_OK ){
+      if( REQUIRE_PTRMAP && rc==SQLITE_OK ){
         u8 eType = (pgnoPtrmap?PTRMAP_OVERFLOW2:PTRMAP_OVERFLOW1);
         ptrmapPut(pBt, pgnoOvfl, eType, pgnoPtrmap, &rc);
         if( rc ){
           releasePage(pOvfl);
         }
       }
-#endif
       if( rc ){
         releasePage(pToRelease);
         return rc;
@@ -6631,6 +7098,7 @@ static void insertCell(
     assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */
     assert( j==0 || i==pPage->aiOvfl[j-1]+1 );   /* Overflows are sequential */
   }else{
+    BtShared *pBt = pPage->pBt;
     int rc = sqlite3PagerWrite(pPage->pDbPage);
     if( rc!=SQLITE_OK ){
       *pRC = rc;
@@ -6645,7 +7113,7 @@ static void insertCell(
     ** if it returns successfully */
     assert( idx >= 0 );
     assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB );
-    assert( idx+sz <= (int)pPage->pBt->usableSize );
+    assert( idx+sz <= (int)pBt->usableSize );
     pPage->nFree -= (u16)(2 + sz);
     memcpy(&data[idx], pCell, sz);
     if( iChild ){
@@ -6658,14 +7126,12 @@ static void insertCell(
     /* increment the cell count */
     if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++;
     assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell );
-#ifndef SQLITE_OMIT_AUTOVACUUM
-    if( pPage->pBt->autoVacuum ){
+    if( REQUIRE_PTRMAP ){
       /* The cell may contain a pointer to an overflow page. If so, write
       ** the entry for the overflow page into the pointer map.
       */
       ptrmapPutOvflPtr(pPage, pCell, pRC);
     }
-#endif
   }
 }
 
@@ -7076,7 +7542,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){
     ** be marked as dirty. Returning an error code will cause a
     ** rollback, undoing any changes made to the parent page.
     */
-    if( ISAUTOVACUUM ){
+    if( REQUIRE_PTRMAP ){
       ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc);
       if( szCell>pNew->minLocal ){
         ptrmapPutOvflPtr(pNew, pCell, &rc);
@@ -7213,7 +7679,7 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){
     /* If this is an auto-vacuum database, update the pointer-map entries
     ** for any b-tree or overflow pages that pTo now contains the pointers to.
     */
-    if( ISAUTOVACUUM ){
+    if( REQUIRE_PTRMAP ){
       *pRC = setChildPtrmaps(pTo);
     }
   }
@@ -7264,7 +7730,8 @@ static int balance_nonroot(
   int iParentIdx,                 /* Index of "the page" in pParent */
   u8 *aOvflSpace,                 /* page-size bytes of space for parent ovfl */
   int isRoot,                     /* True if pParent is a root-page */
-  int bBulk                       /* True if this call is part of a bulk load */
+  int bBulk,                      /* True if this call is part of a bulk load */
+  Pgno pgnoRoot                   /* Root page of b-tree being balanced */
 ){
   BtShared *pBt;               /* The whole database */
   int nMaxCells = 0;           /* Allocated size of apCell, szCell, aFrom. */
@@ -7356,6 +7823,8 @@ static int balance_nonroot(
       memset(apOld, 0, (i+1)*sizeof(MemPage*));
       goto balance_cleanup;
     }
+    setMempageRoot(apOld[i], pgnoRoot);
+
     nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow;
     if( (i--)==0 ) break;
 
@@ -7676,7 +8145,7 @@ static int balance_nonroot(
       cntOld[i] = b.nCell;
 
       /* Set the pointer-map entry for the new sibling page. */
-      if( ISAUTOVACUUM ){
+      if( REQUIRE_PTRMAP ){
         ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc);
         if( rc!=SQLITE_OK ){
           goto balance_cleanup;
@@ -7770,7 +8239,7 @@ static int balance_nonroot(
   ** updated. This happens below, after the sibling pages have been 
   ** populated, not here.
   */
-  if( ISAUTOVACUUM ){
+  if( REQUIRE_PTRMAP ){
     MemPage *pNew = apNew[0];
     u8 *aOld = pNew->aData;
     int cntOldNext = pNew->nCell + pNew->nOverflow;
@@ -7954,7 +8423,7 @@ static int balance_nonroot(
     );
     copyNodeContent(apNew[0], pParent, &rc);
     freePage(apNew[0], &rc);
-  }else if( ISAUTOVACUUM && !leafCorrection ){
+  }else if( REQUIRE_PTRMAP && !leafCorrection ){
     /* Fix the pointer map entries associated with the right-child of each
     ** sibling page. All other pointer map entries have already been taken
     ** care of.  */
@@ -8037,7 +8506,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){
   if( rc==SQLITE_OK ){
     rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0);
     copyNodeContent(pRoot, pChild, &rc);
-    if( ISAUTOVACUUM ){
+    if( REQUIRE_PTRMAP ){
       ptrmapPut(pBt, pgnoChild, PTRMAP_BTREE, pRoot->pgno, &rc);
     }
   }
@@ -8164,7 +8633,7 @@ static int balance(BtCursor *pCur){
           */
           u8 *pSpace = sqlite3PageMalloc(pCur->pBt->pageSize);
           rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1,
-                               pCur->hints&BTREE_BULKLOAD);
+                               pCur->hints&BTREE_BULKLOAD, pCur->pgnoRoot);
           if( pFree ){
             /* If pFree is not NULL, it points to the pSpace buffer used 
             ** by a previous call to balance_nonroot(). Its contents are
@@ -8485,7 +8954,7 @@ int sqlite3BtreeInsert(
     }
     rc = clearCell(pPage, oldCell, &info);
     if( info.nSize==szNew && info.nLocal==info.nPayload 
-     && (!ISAUTOVACUUM || szNew<pPage->minLocal)
+     && (!REQUIRE_PTRMAP || szNew<pPage->minLocal)
     ){
       /* Overwrite the old cell with the new if they are the same size.
       ** We could also try to do this if the old cell is smaller, then add
@@ -8918,7 +9387,8 @@ static int clearDatabasePage(
   BtShared *pBt,           /* The BTree that contains the table */
   Pgno pgno,               /* Page number to clear */
   int freePageFlag,        /* Deallocate page if true */
-  int *pnChange            /* Add number of Cells freed to this counter */
+  int *pnChange,           /* Add number of Cells freed to this counter */
+  Pgno pgnoRoot
 ){
   MemPage *pPage;
   int rc;
@@ -8933,6 +9403,7 @@ static int clearDatabasePage(
   }
   rc = getAndInitPage(pBt, pgno, &pPage, 0, 0);
   if( rc ) return rc;
+  setMempageRoot(pPage, pgnoRoot);
   if( pPage->bBusy ){
     rc = SQLITE_CORRUPT_BKPT;
     goto cleardatabasepage_out;
@@ -8942,14 +9413,16 @@ static int clearDatabasePage(
   for(i=0; i<pPage->nCell; i++){
     pCell = findCell(pPage, i);
     if( !pPage->leaf ){
-      rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange);
+      rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange, pgnoRoot);
       if( rc ) goto cleardatabasepage_out;
     }
     rc = clearCell(pPage, pCell, &info);
     if( rc ) goto cleardatabasepage_out;
   }
   if( !pPage->leaf ){
-    rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange);
+    rc = clearDatabasePage(
+        pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange, pgnoRoot
+    );
     if( rc ) goto cleardatabasepage_out;
   }else if( pnChange ){
     assert( pPage->intKey || CORRUPT_DB );
@@ -8994,7 +9467,7 @@ int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){
     ** is the root of a table b-tree - if it is not, the following call is
     ** a no-op).  */
     invalidateIncrblobCursors(p, (Pgno)iTable, 0, 1);
-    rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange);
+    rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange, (Pgno)iTable);
   }
   sqlite3BtreeLeave(p);
   return rc;
@@ -9869,9 +10342,11 @@ char *sqlite3BtreeIntegrityCheck(
   }
   pBt->db->flags = savedDbFlags;
 
-  /* Make sure every page in the file is referenced
-  */
-  for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){
+  /* Make sure every page in the file is referenced. Skip this if the
+  ** database is currently being written by a CONCURRENT transaction (it 
+  ** may fail as pages that were part of the free-list when the transaction
+  ** was opened cannot be counted).  */
+  for(i=1; ISCONCURRENT==0 && i<=sCheck.nPage && sCheck.mxErr; i++){
 #ifdef SQLITE_OMIT_AUTOVACUUM
     if( getPageReferenced(&sCheck, i)==0 ){
       checkAppendMsg(&sCheck, "Page %d is never used", i);
@@ -10175,6 +10650,99 @@ int sqlite3BtreeIsReadonly(Btree *p){
 */
 int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); }
 
+/*
+** This function is called to ensure that all locks required to commit the
+** current write-transaction to the database file are held. If the db is
+** in rollback mode, this means the EXCLUSIVE lock on the database file.
+**
+** Or, if this is an CONCURRENT transaction on a wal-mode database, the WRITER
+** lock on the wal file. In this case this function also checks that the
+** CONCURRENT transaction can be safely committed (does not commit with any
+** other transaction committed since it was opened).
+**
+** SQLITE_OK is returned if successful. SQLITE_BUSY if the required locks
+** cannot be obtained due to a conflicting lock. If the locks cannot be
+** obtained for an CONCURRENT transaction due to a conflict with an already
+** committed transaction, SQLITE_BUSY_SNAPSHOT is returned. Otherwise, if
+** some other error (OOM, IO, etc.) occurs, the relevant SQLite error code
+** is returned.
+*/
+int sqlite3BtreeExclusiveLock(Btree *p){
+  int rc;
+  Pgno pgno = 0;
+  BtShared *pBt = p->pBt;
+  assert( p->inTrans==TRANS_WRITE && pBt->pPage1 );
+  sqlite3BtreeEnter(p);
+  rc = sqlite3PagerExclusiveLock(pBt->pPager, pBt->pPage1->pDbPage, &pgno);
+#ifdef SQLITE_OMIT_CONCURRENT
+  assert( pgno==0 );
+#else
+  if( rc==SQLITE_BUSY_SNAPSHOT && pgno ){
+    PgHdr *pPg = 0;
+    int rc2 = sqlite3PagerGet(pBt->pPager, pgno, &pPg, 0);
+    if( rc2==SQLITE_OK ){
+      int bWrite = -1;
+      const char *zObj = 0;
+      const char *zTab = 0;
+      char zContent[17];
+
+      if( pPg ){
+        Pgno pgnoRoot = 0;
+        HashElem *pE;
+        Schema *pSchema;
+        u8 *aData = (u8*)sqlite3PagerGetData(pPg);
+        int i;
+        for(i=0; i<8; i++){
+          static const char hexdigits[] = {
+            '0', '1', '2', '3', '4', '5', '6', '7',
+            '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' 
+          };
+          zContent[i*2] = hexdigits[(aData[i] >> 4)];
+          zContent[i*2+1] = hexdigits[(aData[i] & 0xF)];
+        }
+        zContent[16] = '\0';
+
+        pgnoRoot = ((MemPage*)sqlite3PagerGetExtra(pPg))->pgnoRoot;
+        bWrite = sqlite3PagerIswriteable(pPg);
+        sqlite3PagerUnref(pPg);
+
+        pSchema = sqlite3SchemaGet(p->db, p);
+        if( pSchema ){
+          for(pE=sqliteHashFirst(&pSchema->tblHash); pE; pE=sqliteHashNext(pE)){
+            Table *pTab = (Table *)sqliteHashData(pE);
+            if( pTab->tnum==(int)pgnoRoot ){
+              zObj = pTab->zName;
+              zTab = 0;
+            }else{
+              Index *pIdx;
+              for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
+                if( pIdx->tnum==(int)pgnoRoot ){
+                  zObj = pIdx->zName;
+                  zTab = pTab->zName;
+                }
+              }
+            }
+          }
+        }
+      }
+
+      sqlite3_log(SQLITE_OK,
+          "cannot commit CONCURRENT transaction "
+          "- conflict at page %d "
+          "(%s page; part of db %s %s%s%s; content=%s...)",
+          (int)pgno,
+          (bWrite==0?"read-only":(bWrite>0?"read/write":"unknown")),
+          (zTab ? "index" : "table"),
+          (zTab ? zTab : ""), (zTab ? "." : ""), (zObj ? zObj : "UNKNOWN"),
+          zContent
+      );
+    }
+  }
+#endif
+  sqlite3BtreeLeave(p);
+  return rc;
+}
+
 #if !defined(SQLITE_OMIT_SHARED_CACHE)
 /*
 ** Return true if the Btree passed as the only argument is sharable.

src/btree.h

@@ -331,6 +331,8 @@ int sqlite3BtreeCursorHasHint(BtCursor*, unsigned int mask);
 int sqlite3BtreeIsReadonly(Btree *pBt);
 int sqlite3HeaderSizeBtree(void);
 
+int sqlite3BtreeExclusiveLock(Btree *pBt);
+
 #ifndef NDEBUG
 int sqlite3BtreeCursorIsValid(BtCursor*);
 #endif
@@ -389,5 +391,4 @@ void sqlite3BtreeCursorList(Btree*);
 # define sqlite3SchemaMutexHeld(X,Y,Z) 1
 #endif
 
-
 #endif /* SQLITE_BTREE_H */

src/btreeInt.h

@@ -232,6 +232,7 @@
 typedef struct MemPage MemPage;
 typedef struct BtLock BtLock;
 typedef struct CellInfo CellInfo;
+typedef struct BtreePtrmap BtreePtrmap;
 
 /*
 ** This is a magic string that appears at the beginning of every
@@ -276,6 +277,9 @@ struct MemPage {
   u8 intKey;           /* True if table b-trees.  False for index b-trees */
   u8 intKeyLeaf;       /* True if the leaf of an intKey table */
   Pgno pgno;           /* Page number for this page */
+#ifndef SQLITE_OMIT_CONCURRENT
+  Pgno pgnoRoot;       /* Root page of b-tree that this page belongs to */
+#endif
   /* Only the first 8 bytes (above) are zeroed by pager.c when a new page
   ** is allocated. All fields that follow must be initialized before use */
   u8 leaf;             /* True if a leaf page */
@@ -440,6 +444,9 @@ struct BtShared {
   Btree *pWriter;       /* Btree with currently open write transaction */
 #endif
   u8 *pTmpSpace;        /* Temp space sufficient to hold a single cell */
+#ifndef SQLITE_OMIT_CONCURRENT
+  BtreePtrmap *pMap;
+#endif
 };
 
 /*
@@ -648,12 +655,19 @@ struct BtCursor {
 ** (sqliteMallocRaw), it is not possible to use conditional compilation.
 ** So, this macro is defined instead.
 */
-#ifndef SQLITE_OMIT_AUTOVACUUM
-#define ISAUTOVACUUM (pBt->autoVacuum)
-#else
+#ifdef SQLITE_OMIT_AUTOVACUUM
 #define ISAUTOVACUUM 0
+#else
+#define ISAUTOVACUUM (pBt->autoVacuum)
 #endif
 
+#ifdef SQLITE_OMIT_CONCURRENT
+# define ISCONCURRENT 0
+#else
+# define ISCONCURRENT (pBt->pMap!=0)
+#endif
+
+#define REQUIRE_PTRMAP (ISAUTOVACUUM || ISCONCURRENT)
 
 /*
 ** This structure is passed around through all the sanity checking routines

src/build.c

@@ -59,8 +59,10 @@ void sqlite3TableLock(
   TableLock *p;
   assert( iDb>=0 );
 
+#ifdef SQLITE_OMIT_CONCURRENT
   if( iDb==1 ) return;
   if( !sqlite3BtreeSharable(pParse->db->aDb[iDb].pBt) ) return;
+#endif
   for(i=0; i<pToplevel->nTableLock; i++){
     p = &pToplevel->aTableLock[i];
     if( p->iDb==iDb && p->iTab==iTab ){
@@ -4133,13 +4135,13 @@ void sqlite3BeginTransaction(Parse *pParse, int type){
   }
   v = sqlite3GetVdbe(pParse);
   if( !v ) return;
-  if( type!=TK_DEFERRED ){
+  if( type==TK_IMMEDIATE || type==TK_EXCLUSIVE ){
     for(i=0; i<db->nDb; i++){
       sqlite3VdbeAddOp2(v, OP_Transaction, i, (type==TK_EXCLUSIVE)+1);
       sqlite3VdbeUsesBtree(v, i);
     }
   }
-  sqlite3VdbeAddOp0(v, OP_AutoCommit);
+  sqlite3VdbeAddOp3(v, OP_AutoCommit, 0, 0, (type==TK_CONCURRENT));
 }
 
 /*

src/func.c

@@ -485,8 +485,9 @@ static void randomFunc(
   sqlite3_value **NotUsed2
 ){
   sqlite_int64 r;
+  sqlite3 *db = sqlite3_context_db_handle(context);
   UNUSED_PARAMETER2(NotUsed, NotUsed2);
-  sqlite3_randomness(sizeof(r), &r);
+  sqlite3FastRandomness(&db->sPrng, sizeof(r), &r);
   if( r<0 ){
     /* We need to prevent a random number of 0x8000000000000000 
     ** (or -9223372036854775808) since when you do abs() of that
@@ -512,6 +513,7 @@ static void randomBlob(
 ){
   int n;
   unsigned char *p;
+  sqlite3 *db = sqlite3_context_db_handle(context);
   assert( argc==1 );
   UNUSED_PARAMETER(argc);
   n = sqlite3_value_int(argv[0]);
@@ -520,7 +522,7 @@ static void randomBlob(
   }
   p = contextMalloc(context, n);
   if( p ){
-    sqlite3_randomness(n, p);
+    sqlite3FastRandomness(&db->sPrng, n, p);
     sqlite3_result_blob(context, (char*)p, n, sqlite3_free);
   }
 }

src/main.c

@@ -3025,7 +3025,7 @@ static int openDatabase(
   db->magic = SQLITE_MAGIC_BUSY;
   db->aDb = db->aDbStatic;
   db->lookaside.bDisable = 1;
-
+  sqlite3FastPrngInit(&db->sPrng);
   assert( sizeof(db->aLimit)==sizeof(aHardLimit) );
   memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit));
   db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS;
@@ -4301,6 +4301,35 @@ void sqlite3_snapshot_free(sqlite3_snapshot *pSnapshot){
 }
 #endif /* SQLITE_ENABLE_SNAPSHOT */
 
+SQLITE_EXPERIMENTAL int sqlite3_wal_info(
+  sqlite3 *db, const char *zDb, 
+  unsigned int *pnPrior, unsigned int *pnFrame
+){
+  int rc = SQLITE_OK;
+
+#ifndef SQLITE_OMIT_WAL
+  Btree *pBt;
+  int iDb;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( !sqlite3SafetyCheckOk(db) ){
+    return SQLITE_MISUSE_BKPT;
+  }
+#endif
+
+  sqlite3_mutex_enter(db->mutex);
+  iDb = sqlite3FindDbName(db, zDb);
+  if( iDb<0 ){
+    return SQLITE_ERROR;
+  }
+  pBt = db->aDb[iDb].pBt;
+  rc = sqlite3PagerWalInfo(sqlite3BtreePager(pBt), pnPrior, pnFrame);
+  sqlite3_mutex_leave(db->mutex);
+#endif   /* SQLITE_OMIT_WAL */
+
+  return rc;
+}
+
 #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS
 /*
 ** Given the name of a compile-time option, return true if that option

src/os_unix.c

@@ -46,6 +46,16 @@
 #include "sqliteInt.h"
 #if SQLITE_OS_UNIX              /* This file is used on unix only */
 
+/* Turn this feature on in all builds for now */
+#define SQLITE_MUTEXFREE_SHMLOCK 1
+#define SQLITE_MFS_EXCLUSIVE     255
+#ifndef SQLITE_MFS_NSHARD
+# define SQLITE_MFS_NSHARD       8
+#endif
+#if SQLITE_MFS_NSHARD<1
+# error "SQLITE_MFS_NSHARD must be greater than 0"
+#endif
+
 /*
 ** There are various methods for file locking used for concurrency
 ** control:
@@ -1159,6 +1169,10 @@ struct unixInodeInfo {
   sem_t *pSem;                    /* Named POSIX semaphore */
   char aSemName[MAX_PATHNAME+2];  /* Name of that semaphore */
 #endif
+#ifdef SQLITE_SHARED_MAPPING
+  sqlite3_int64 nSharedMapping;   /* Size of mapped region in bytes */
+  void *pSharedMapping;           /* Memory mapped region */
+#endif
 };
 
 /*
@@ -1311,6 +1325,13 @@ static void releaseInodeInfo(unixFile *pFile){
     pInode->nRef--;
     if( pInode->nRef==0 ){
       assert( pInode->pShmNode==0 );
+#ifdef SQLITE_SHARED_MAPPING
+      if( pInode->pSharedMapping ){
+        osMunmap(pInode->pSharedMapping, pInode->nSharedMapping);
+        pInode->pSharedMapping = 0;
+        pInode->nSharedMapping = 0;
+      }
+#endif
       sqlite3_mutex_enter(pInode->pLockMutex);
       closePendingFds(pFile);
       sqlite3_mutex_leave(pInode->pLockMutex);
@@ -2169,6 +2190,14 @@ static int nolockUnlock(sqlite3_file *NotUsed, int NotUsed2){
 ** Close the file.
 */
 static int nolockClose(sqlite3_file *id) {
+#ifdef SQLITE_SHARED_MAPPING
+  unixFile *pFd = (unixFile*)id;
+  if( pFd->pInode ){
+    unixEnterMutex();
+    releaseInodeInfo(pFd);
+    unixLeaveMutex();
+  }
+#endif
   return closeUnixFile(id);
 }
 
@@ -4000,6 +4029,9 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){
       *(i64*)pArg = pFile->mmapSizeMax;
       if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){
         pFile->mmapSizeMax = newLimit;
+#ifdef SQLITE_SHARED_MAPPING
+        if( pFile->pInode==0 )
+#endif
         if( pFile->mmapSize>0 ){
           unixUnmapfile(pFile);
           rc = unixMapfile(pFile, -1);
@@ -4239,8 +4271,41 @@ struct unixShmNode {
   u8 sharedMask;             /* Mask of shared locks held */
   u8 nextShmId;              /* Next available unixShm.id value */
 #endif
+
+#ifdef SQLITE_MUTEXFREE_SHMLOCK
+  /* In unix-excl mode, if SQLITE_MUTEXFREE_SHMLOCK is defined, all locks
+  ** are stored in the following 64-bit value. There are in total 8 
+  ** shm-locking slots, each of which are assigned 8-bits from the 64-bit
+  ** value. The least-significant 8 bits correspond to shm-locking slot
+  ** 0, and so on.
+  **
+  ** If the 8-bits corresponding to a shm-locking locking slot are set to
+  ** 0xFF, then a write-lock is held on the slot. Or, if they are set to
+  ** a non-zero value smaller than 0xFF, then they represent the total 
+  ** number of read-locks held on the slot. There is no way to distinguish
+  ** between a write-lock and 255 read-locks.  */
+  struct LockingSlot {
+    u32 nLock;
+    u64 aPadding[7];
+  } aMFSlot[3 + SQLITE_MFS_NSHARD*5];
+#endif
 };
 
+/*
+** Atomic CAS primitive used in multi-process mode. Equivalent to:
+** 
+**   int unixCompareAndSwap(u32 *ptr, u32 oldval, u32 newval){
+**     if( *ptr==oldval ){
+**       *ptr = newval;
+**       return 1;
+**     }
+**     return 0;
+**   }
+*/
+#define unixCompareAndSwap(ptr,oldval,newval) \
+    __sync_bool_compare_and_swap(ptr,oldval,newval)
+
+
 /*
 ** Structure used internally by this VFS to record the state of an
 ** open shared memory connection.
@@ -4261,6 +4326,9 @@ struct unixShm {
   u8 id;                     /* Id of this connection within its unixShmNode */
   u16 sharedMask;            /* Mask of shared locks held */
   u16 exclMask;              /* Mask of exclusive locks held */
+#ifdef SQLITE_MUTEXFREE_SHMLOCK
+  u8 aMFCurrent[8];          /* Current slot used for each shared lock */
+#endif
 };
 
 /*
@@ -4765,6 +4833,83 @@ shmpage_out:
   return rc;
 }
 
+#ifdef SQLITE_MUTEXFREE_SHMLOCK
+static int unixMutexFreeShmlock(
+  unixFile *pFd,             /* Database file holding the shared memory */
+  int ofst,                  /* First lock to acquire or release */
+  int n,                     /* Number of locks to acquire or release */
+  int flags                  /* What to do with the lock */
+){
+  struct LockMapEntry {
+    int iFirst;
+    int nSlot;
+  } aMap[9] = {
+    { 0, 1 },
+    { 1, 1 },
+    { 2, 1 },
+    { 3+0*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+1*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+2*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+3*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+4*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+5*SQLITE_MFS_NSHARD, 0 },
+  };
+
+  unixShm *p = pFd->pShm;               /* The shared memory being locked */
+  unixShm *pX;                          /* For looping over all siblings */
+  unixShmNode *pShmNode = p->pShmNode;  /* The underlying file iNode */
+  int rc = SQLITE_OK;
+  int iIncr;
+  u16 mask;                             /* Mask of locks to take or release */
+
+  if( flags & SQLITE_SHM_SHARED ){
+    /* SHARED locks */
+    u32 iOld, iNew, *ptr;
+    int iIncr = -1;
+    if( (flags & SQLITE_SHM_UNLOCK)==0 ){
+      p->aMFCurrent[ofst] = (p->aMFCurrent[ofst] + 1) % aMap[ofst].nSlot;
+      iIncr = 1;
+    }
+    ptr = &pShmNode->aMFSlot[aMap[ofst].iFirst + p->aMFCurrent[ofst]].nLock;
+    do {
+      iOld = *ptr;
+      iNew = iOld + iIncr;
+      if( iNew>SQLITE_MFS_EXCLUSIVE ){
+        return SQLITE_BUSY;
+      }
+    }while( 0==unixCompareAndSwap(ptr, iOld, iNew) );
+  }else{
+    /* EXCLUSIVE locks */
+    int iFirst = aMap[ofst].iFirst;
+    int iLast = aMap[ofst+n].iFirst;
+    int i;
+    for(i=iFirst; i<iLast; i++){
+      u32 *ptr = &pShmNode->aMFSlot[i].nLock;
+      if( flags & SQLITE_SHM_UNLOCK ){
+        assert( (*ptr)==SQLITE_MFS_EXCLUSIVE );
+        *ptr = 0;
+      }else{
+        u32 iOld;
+        do {
+          iOld = *ptr;
+          if( iOld>0 ){
+            while( i>iFirst ){
+              i--;
+              pShmNode->aMFSlot[i].nLock = 0;
+            }
+            return SQLITE_BUSY;
+          }
+        }while( 0==unixCompareAndSwap(ptr, iOld, SQLITE_MFS_EXCLUSIVE) );
+      }
+    }
+  }
+
+  return SQLITE_OK;
+}
+#else
+# define unixMutexFreeShmlock(a,b,c,d) SQLITE_OK
+#endif
+
 /*
 ** Change the lock state for a shared-memory segment.
 **
@@ -4798,8 +4943,20 @@ static int unixShmLock(
   assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 );
   assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 );
 
+  if( pDbFd->pInode->bProcessLock ){
+    return unixMutexFreeShmlock(pDbFd, ofst, n, flags);
+  }
+
   mask = (1<<(ofst+n)) - (1<<ofst);
   assert( n>1 || mask==(1<<ofst) );
+  if( flags & SQLITE_SHM_LOCK ){
+    assert( !(flags&SQLITE_SHM_SHARED) || (p->sharedMask&mask)==0 );
+    assert( !(flags&SQLITE_SHM_EXCLUSIVE) || !(p->exclMask&mask) );
+  }else{
+    assert( !(flags&SQLITE_SHM_SHARED) || (p->sharedMask&mask)==mask );
+    assert( !(flags&SQLITE_SHM_EXCLUSIVE) || (p->exclMask&mask)==mask );
+  }
+
   sqlite3_mutex_enter(pShmNode->pShmMutex);
   if( flags & SQLITE_SHM_UNLOCK ){
     u16 allMask = 0; /* Mask of locks held by siblings */
@@ -4889,12 +5046,16 @@ static void unixShmBarrier(
   sqlite3_file *fd                /* Database file holding the shared memory */
 ){
   UNUSED_PARAMETER(fd);
+#ifdef SQLITE_MUTEXFREE_SHMLOCK
+  __sync_synchronize();
+#else
   sqlite3MemoryBarrier();         /* compiler-defined memory barrier */
   assert( fd->pMethods->xLock==nolockLock 
        || unixFileMutexNotheld((unixFile*)fd) 
   );
   unixEnterMutex();               /* Also mutex, for redundancy */
   unixLeaveMutex();
+#endif
 }
 
 /*
@@ -4963,6 +5124,9 @@ static int unixShmUnmap(
 */
 static void unixUnmapfile(unixFile *pFd){
   assert( pFd->nFetchOut==0 );
+#ifdef SQLITE_SHARED_MAPPING
+  if( pFd->pInode ) return;
+#endif
   if( pFd->pMapRegion ){
     osMunmap(pFd->pMapRegion, pFd->mmapSizeActual);
     pFd->pMapRegion = 0;
@@ -5094,6 +5258,28 @@ static int unixMapfile(unixFile *pFd, i64 nMap){
     nMap = pFd->mmapSizeMax;
   }
 
+#ifdef SQLITE_SHARED_MAPPING
+  if( pFd->pInode ){
+    unixInodeInfo *pInode = pFd->pInode;
+    if( pFd->pMapRegion ) return SQLITE_OK;
+    unixEnterMutex();
+    if( pInode->pSharedMapping==0 ){
+      u8 *pNew = osMmap(0, nMap, PROT_READ, MAP_SHARED, pFd->h, 0);
+      if( pNew==MAP_FAILED ){
+        unixLogError(SQLITE_OK, "mmap", pFd->zPath);
+        pFd->mmapSizeMax = 0;
+      }else{
+        pInode->pSharedMapping = pNew;
+        pInode->nSharedMapping = nMap;
+      }
+    }
+    pFd->pMapRegion = pInode->pSharedMapping;
+    pFd->mmapSizeActual = pFd->mmapSize = pInode->nSharedMapping;
+    unixLeaveMutex();
+    return SQLITE_OK;
+  }
+#endif
+
   assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) );
   if( nMap!=pFd->mmapSize ){
     unixRemapfile(pFd, nMap);
@@ -5531,6 +5717,9 @@ static int fillInUnixFile(
   if( pLockingStyle == &posixIoMethods
 #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE
     || pLockingStyle == &nfsIoMethods
+#endif
+#ifdef SQLITE_SHARED_MAPPING
+    || pLockingStyle == &nolockIoMethods
 #endif
   ){
     unixEnterMutex();

src/pager.c

@@ -670,6 +670,9 @@ struct Pager {
   u32 cksumInit;              /* Quasi-random value added to every checksum */
   u32 nSubRec;                /* Number of records written to sub-journal */
   Bitvec *pInJournal;         /* One bit for each page in the database file */
+#ifndef SQLITE_OMIT_CONCURRENT
+  Bitvec *pAllRead;           /* Pages read within current CONCURRENT trans. */
+#endif
   sqlite3_file *fd;           /* File descriptor for database */
   sqlite3_file *jfd;          /* File descriptor for main journal */
   sqlite3_file *sjfd;         /* File descriptor for sub-journal */
@@ -937,7 +940,9 @@ static int assert_pager_state(Pager *p){
       if( !pagerUseWal(pPager) ){
         assert( p->eLock>=RESERVED_LOCK );
       }
-      assert( pPager->dbSize==pPager->dbOrigSize );
+#ifndef SQLITE_OMIT_CONCURRENT
+      assert( pPager->dbSize==pPager->dbOrigSize || pPager->pAllRead );
+#endif
       assert( pPager->dbOrigSize==pPager->dbFileSize );
       assert( pPager->dbOrigSize==pPager->dbHintSize );
       assert( pPager->setMaster==0 );
@@ -1821,6 +1826,53 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){
   return rc;
 }
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** If they are not already, begin recording all pages read from the pager layer
+** by the b-tree layer This is used by concurrent transactions. Return
+** SQLITE_OK if successful, or an SQLite error code (SQLITE_NOMEM) if an error
+** occurs.
+*/
+int sqlite3PagerBeginConcurrent(Pager *pPager){
+  int rc = SQLITE_OK;
+  if( pPager->pAllRead==0 ){
+    pPager->pAllRead = sqlite3BitvecCreate(pPager->dbSize);
+    pPager->dbOrigSize = pPager->dbSize;
+    if( pPager->pAllRead==0 ){
+      rc = SQLITE_NOMEM;
+    }
+  }
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Stop recording all pages read from the pager layer by the b-tree layer
+** and discard any current records.
+*/
+void sqlite3PagerEndConcurrent(Pager *pPager){
+  sqlite3BitvecDestroy(pPager->pAllRead);
+  pPager->pAllRead = 0;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Return true if the database is in wal mode. False otherwise.
+*/
+int sqlite3PagerIsWal(Pager *pPager){
+  return pPager->pWal!=0;
+}
+#endif /* SQLITE_OMIT_CONCURRENT */
+
+/*
+** Free the Pager.pInJournal and Pager.pAllRead bitvec objects.
+*/
+static void pagerFreeBitvecs(Pager *pPager){
+  sqlite3BitvecDestroy(pPager->pInJournal);
+  pPager->pInJournal = 0;
+  sqlite3PagerEndConcurrent(pPager);
+}
+
 /*
 ** This function is a no-op if the pager is in exclusive mode and not
 ** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN
@@ -1845,8 +1897,7 @@ static void pager_unlock(Pager *pPager){
        || pPager->eState==PAGER_ERROR 
   );
 
-  sqlite3BitvecDestroy(pPager->pInJournal);
-  pPager->pInJournal = 0;
+  pagerFreeBitvecs(pPager);
   releaseAllSavepoints(pPager);
 
   if( pagerUseWal(pPager) ){
@@ -2114,8 +2165,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){
   }
 #endif
 
-  sqlite3BitvecDestroy(pPager->pInJournal);
-  pPager->pInJournal = 0;
+  pagerFreeBitvecs(pPager);
   pPager->nRec = 0;
   if( rc==SQLITE_OK ){
     if( MEMDB || pagerFlushOnCommit(pPager, bCommit) ){
@@ -3165,6 +3215,16 @@ static int pagerRollbackWal(Pager *pPager){
   pPager->dbSize = pPager->dbOrigSize;
   rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager);
   pList = sqlite3PcacheDirtyList(pPager->pPCache);
+
+#ifndef SQLITE_OMIT_CONCURRENT
+  /* If this is an CONCURRENT transaction, then page 1 must be reread from 
+  ** the db file, even if it is not dirty. This is because the b-tree layer 
+  ** may have already zeroed the nFree and iTrunk header fields.  */
+  if( rc==SQLITE_OK && (pList==0 || pList->pgno!=1) && pPager->pAllRead ){
+    rc = pagerUndoCallback((void*)pPager, 1);
+  }
+#endif
+
   while( pList && rc==SQLITE_OK ){
     PgHdr *pNext = pList->pDirty;
     rc = pagerUndoCallback((void *)pPager, pList->pgno);
@@ -3214,6 +3274,8 @@ static int pagerWalFrames(
       if( p->pgno<=nTruncate ){
         ppNext = &p->pDirty;
         nList++;
+        PAGERTRACE(("TO-WAL %d page %d hash(%08x)\n",
+                     PAGERID(pPager), p->pgno, pager_pagehash(p)));
       }
     }
     assert( pList );
@@ -4274,7 +4336,7 @@ static int syncJournal(Pager *pPager, int newHdr){
   assert( assert_pager_state(pPager) );
   assert( !pagerUseWal(pPager) );
 
-  rc = sqlite3PagerExclusiveLock(pPager);
+  rc = sqlite3PagerExclusiveLock(pPager, 0, 0);
   if( rc!=SQLITE_OK ) return rc;
 
   if( !pPager->noSync ){
@@ -4632,6 +4694,12 @@ static int pagerStress(void *p, PgHdr *pPg){
   pPager->aStat[PAGER_STAT_SPILL]++;
   pPg->pDirty = 0;
   if( pagerUseWal(pPager) ){
+#ifndef SQLITE_OMIT_CONCURRENT
+    /* If the transaction is a "BEGIN CONCURRENT" transaction, the page 
+    ** cannot be flushed to disk. Return early in this case. */
+    if( pPager->pAllRead ) return SQLITE_OK;
+#endif
+
     /* Write a single frame for this page to the log. */
     rc = subjournalPageIfRequired(pPg); 
     if( rc==SQLITE_OK ){
@@ -5465,6 +5533,18 @@ static int getPageNormal(
   assert( assert_pager_state(pPager) );
   assert( pPager->hasHeldSharedLock==1 );
 
+#ifndef SQLITE_OMIT_CONCURRENT
+  /* If this is an CONCURRENT transaction and the page being read was
+  ** present in the database file when the transaction was opened,
+  ** mark it as read in the pAllRead vector.  */
+  pPg = 0;
+  if( pPager->pAllRead && pgno<=pPager->dbOrigSize ){
+    PAGERTRACE(("USING page %d\n", pgno));
+    rc = sqlite3BitvecSet(pPager->pAllRead, pgno);
+    if( rc!=SQLITE_OK ) goto pager_acquire_err;
+  }
+#endif
+
   if( pgno==0 ) return SQLITE_CORRUPT_BKPT;
   pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3);
   if( pBase==0 ){
@@ -5806,11 +5886,14 @@ static int pager_open_journal(Pager *pPager){
 ** Begin a write-transaction on the specified pager object. If a 
 ** write-transaction has already been opened, this function is a no-op.
 **
-** If the exFlag argument is false, then acquire at least a RESERVED
-** lock on the database file. If exFlag is true, then acquire at least
+** If the exFlag argument is 0, then acquire at least a RESERVED
+** lock on the database file. If exFlag is >0, then acquire at least
 ** an EXCLUSIVE lock. If such a lock is already held, no locking 
 ** functions need be called.
 **
+** If (exFlag<0) and the database is in WAL mode, do not take any locks.
+** The transaction will run in CONCURRENT mode instead.
+**
 ** If the subjInMemory argument is non-zero, then any sub-journal opened
 ** within this transaction will be opened as an in-memory file. This
 ** has no effect if the sub-journal is already opened (as it may be when
@@ -5828,7 +5911,6 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){
 
   if( ALWAYS(pPager->eState==PAGER_READER) ){
     assert( pPager->pInJournal==0 );
-
     if( pagerUseWal(pPager) ){
       /* If the pager is configured to use locking_mode=exclusive, and an
       ** exclusive lock on the database is not already held, obtain it now.
@@ -5844,9 +5926,10 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){
       /* Grab the write lock on the log file. If successful, upgrade to
       ** PAGER_RESERVED state. Otherwise, return an error code to the caller.
       ** The busy-handler is not invoked if another connection already
-      ** holds the write-lock. If possible, the upper layer will call it.
-      */
-      rc = sqlite3WalBeginWriteTransaction(pPager->pWal);
+      ** holds the write-lock. If possible, the upper layer will call it.  */
+      if( exFlag>=0 ){
+        rc = sqlite3WalBeginWriteTransaction(pPager->pWal);
+      }
     }else{
       /* Obtain a RESERVED lock on the database file. If the exFlag parameter
       ** is true, then immediately upgrade this to an EXCLUSIVE lock. The
@@ -5854,7 +5937,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){
       ** lock, but not when obtaining the RESERVED lock.
       */
       rc = pagerLockDb(pPager, RESERVED_LOCK);
-      if( rc==SQLITE_OK && exFlag ){
+      if( rc==SQLITE_OK && exFlag>0 ){
         rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK);
       }
     }
@@ -6154,7 +6237,7 @@ int sqlite3PagerWrite(PgHdr *pPg){
 ** to sqlite3PagerWrite().  In other words, return TRUE if it is ok
 ** to change the content of the page.
 */
-#ifndef NDEBUG
+#if !defined(SQLITE_OMIT_CONCURRENT) || !defined(NDEBUG)
 int sqlite3PagerIswriteable(DbPage *pPg){
   return pPg->flags & PGHDR_WRITEABLE;
 }
@@ -6310,17 +6393,26 @@ int sqlite3PagerSync(Pager *pPager, const char *zMaster){
 }
 
 /*
-** This function may only be called while a write-transaction is active in
-** rollback. If the connection is in WAL mode, this call is a no-op. 
-** Otherwise, if the connection does not already have an EXCLUSIVE lock on 
-** the database file, an attempt is made to obtain one.
+** This function is called to ensure that all locks required to commit the
+** current write-transaction to the database file are held. If the db is
+** in rollback mode, this means the EXCLUSIVE lock on the database file.
 **
-** If the EXCLUSIVE lock is already held or the attempt to obtain it is
-** successful, or the connection is in WAL mode, SQLITE_OK is returned.
-** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is 
-** returned.
+** Or, if this is a non-CONCURRENT transaction on a wal-mode database, this
+** function is a no-op.
+**
+** If this is an CONCURRENT transaction on a wal-mode database, this function
+** attempts to obtain the WRITER lock on the wal file and also checks to
+** see that the transaction can be safely committed (does not commit with 
+** any other transaction committed since it was opened).
+**
+** If the required locks are already held or successfully obtained and
+** the transaction can be committed, SQLITE_OK is returned. If a required lock
+** cannot be obtained, SQLITE_BUSY is returned. Or, if the current transaction
+** is CONCURRENT and cannot be committed due to a conflict, SQLITE_BUSY_SNAPSHOT
+** is returned. Otherwise, if some other error occurs (IO error, OOM etc.),
+** and SQLite error code is returned.
 */
-int sqlite3PagerExclusiveLock(Pager *pPager){
+int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1, Pgno *piConflict){
   int rc = pPager->errCode;
   assert( assert_pager_state(pPager) );
   if( rc==SQLITE_OK ){
@@ -6332,10 +6424,73 @@ int sqlite3PagerExclusiveLock(Pager *pPager){
     if( 0==pagerUseWal(pPager) ){
       rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK);
     }
+#ifndef SQLITE_OMIT_CONCURRENT
+    else{
+      if( pPager->pAllRead ){
+        /* This is an CONCURRENT transaction. Attempt to lock the wal database
+        ** here. If SQLITE_BUSY (but not SQLITE_BUSY_SNAPSHOT) is returned,
+        ** invoke the busy-handler and try again for as long as it returns
+        ** non-zero.  */
+        do {
+          rc = sqlite3WalLockForCommit(
+              pPager->pWal, pPage1, pPager->pAllRead, piConflict
+          );
+        }while( rc==SQLITE_BUSY 
+             && pPager->xBusyHandler(pPager->pBusyHandlerArg) 
+        );
+      }
+    }
+#endif /* SQLITE_OMIT_CONCURRENT */
   }
   return rc;
 }
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** This function is called as part of committing an CONCURRENT transaction.
+** At this point the wal WRITER lock is held, and all pages in the cache 
+** except for page 1 are compatible with the snapshot at the head of the
+** wal file. 
+**
+** This function updates the in-memory data structures and reloads the
+** contents of page 1 so that the client is operating on the snapshot 
+** at the head of the wal file.
+**
+** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
+*/
+int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage *pPage1){
+  int rc;
+
+  assert( pPager->pWal && pPager->pAllRead );
+  rc = sqlite3WalUpgradeSnapshot(pPager->pWal);
+  if( rc==SQLITE_OK ){
+    rc = readDbPage(pPage1);
+  }
+
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Set the in-memory cache of the database file size to nSz pages.
+*/
+void sqlite3PagerSetDbsize(Pager *pPager, Pgno nSz){
+  pPager->dbSize = nSz;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** If this is a WAL mode connection and the WRITER lock is currently held,
+** relinquish it.
+*/
+void sqlite3PagerDropExclusiveLock(Pager *pPager){
+  if( pagerUseWal(pPager) ){
+    sqlite3WalEndWriteTransaction(pPager->pWal);
+  }
+}
+#endif  /* SQLITE_OMIT_CONCURRENT */
+
+
 /*
 ** Sync the database file for the pager pPager. zMaster points to the name
 ** of a master journal file that should be written into the individual
@@ -7698,6 +7853,11 @@ void sqlite3PagerSnapshotUnlock(Pager *pPager){
 }
 
 #endif /* SQLITE_ENABLE_SNAPSHOT */
+
+int sqlite3PagerWalInfo(Pager *pPager, u32 *pnPrior, u32 *pnFrame){
+  return sqlite3WalInfo(pPager->pWal, pnPrior, pnFrame);
+}
+
 #endif /* !SQLITE_OMIT_WAL */
 
 #ifdef SQLITE_ENABLE_ZIPVFS

src/pager.h

@@ -165,7 +165,7 @@ void *sqlite3PagerGetExtra(DbPage *);
 void sqlite3PagerPagecount(Pager*, int*);
 int sqlite3PagerBegin(Pager*, int exFlag, int);
 int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int);
-int sqlite3PagerExclusiveLock(Pager*);
+int sqlite3PagerExclusiveLock(Pager*, DbPage *pPage1, Pgno*);
 int sqlite3PagerSync(Pager *pPager, const char *zMaster);
 int sqlite3PagerCommitPhaseTwo(Pager*);
 int sqlite3PagerRollback(Pager*);
@@ -173,6 +173,7 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int n);
 int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint);
 int sqlite3PagerSharedLock(Pager *pPager);
 
+
 #ifndef SQLITE_OMIT_WAL
   int sqlite3PagerCheckpoint(Pager *pPager, sqlite3*, int, int*, int*);
   int sqlite3PagerWalSupported(Pager *pPager);
@@ -224,6 +225,23 @@ void sqlite3PagerTruncateImage(Pager*,Pgno);
 
 void sqlite3PagerRekey(DbPage*, Pgno, u16);
 
+#ifndef SQLITE_OMIT_CONCURRENT
+void sqlite3PagerEndConcurrent(Pager*);
+int sqlite3PagerBeginConcurrent(Pager*);
+void sqlite3PagerDropExclusiveLock(Pager*);
+int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage*);
+void sqlite3PagerSetDbsize(Pager *pPager, Pgno);
+int sqlite3PagerIsWal(Pager*);
+#else
+# define sqlite3PagerEndConcurrent(x)
+#endif
+
+#if defined(SQLITE_DEBUG) || !defined(SQLITE_OMIT_CONCURRENT)
+int sqlite3PagerIswriteable(DbPage*);
+#endif
+
+int sqlite3PagerWalInfo(Pager*, u32 *pnPrior, u32 *pnFrame);
+
 #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL)
 void *sqlite3PagerCodec(DbPage *);
 #endif
@@ -231,7 +249,6 @@ void *sqlite3PagerCodec(DbPage *);
 /* Functions to support testing and debugging. */
 #if !defined(NDEBUG) || defined(SQLITE_TEST)
   Pgno sqlite3PagerPagenumber(DbPage*);
-  int sqlite3PagerIswriteable(DbPage*);
 #endif
 #ifdef SQLITE_TEST
   int *sqlite3PagerStats(Pager*);

src/parse.y

@@ -99,6 +99,13 @@
 */
 struct TrigEvent { int a; IdList * b; };
 
+/*
+** Generate a syntax error
+*/
+static void parserSyntaxError(Parse *pParse, Token *p){
+  sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", p);
+}
+
 struct FrameBound     { int eType; Expr *pExpr; };
 
 /*
@@ -136,7 +143,16 @@ trans_opt ::= TRANSACTION nm.
 transtype(A) ::= .             {A = TK_DEFERRED;}
 transtype(A) ::= DEFERRED(X).  {A = @X; /*A-overwrites-X*/}
 transtype(A) ::= IMMEDIATE(X). {A = @X; /*A-overwrites-X*/}
-transtype(A) ::= EXCLUSIVE(X). {A = @X; /*A-overwrites-X*/}
+transtype(A) ::= ID(X). {
+   Token *p = &X;
+   if( p->n==9 && sqlite3_strnicmp(p->z,"exclusive",9)==0 ){
+     A = TK_EXCLUSIVE;
+   }else if( p->n==10 && sqlite3_strnicmp(p->z,"concurrent",10)==0 ){
+     A = TK_CONCURRENT;  /*A-overwrites-X*/
+   }else{
+     parserSyntaxError(pParse, p);
+   }
+}
 cmd ::= COMMIT|END(X) trans_opt.   {sqlite3EndTransaction(pParse,@X);}
 cmd ::= ROLLBACK(X) trans_opt.     {sqlite3EndTransaction(pParse,@X);}
 
@@ -253,7 +269,6 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);}
 //
 %token_class id  ID|INDEXED.
 
-
 // And "ids" is an identifer-or-string.
 //
 %token_class ids  ID|STRING.
@@ -1011,7 +1026,7 @@ expr(A) ::= VARIABLE(X).     {
     Token t = X; /*A-overwrites-X*/
     assert( t.n>=2 );
     if( pParse->nested==0 ){
-      sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t);
+      parserSyntaxError(pParse, &t);
       A = 0;
     }else{
       A = sqlite3PExpr(pParse, TK_REGISTER, 0, 0);

src/pragma.h

@@ -443,6 +443,15 @@ static const PragmaName aPragmaName[] = {
   /* iArg:      */ 0 },
 #endif
 #endif
+#endif
+#if !defined(SQLITE_OMIT_FLAG_PRAGMAS)
+#if defined(SQLITE_ENABLE_NOOP_UPDATE)
+ {/* zName:     */ "noop_update",
+  /* ePragTyp:  */ PragTyp_FLAG,
+  /* ePragFlg:  */ PragFlg_Result0|PragFlg_NoColumns1,
+  /* ColNames:  */ 0, 0,
+  /* iArg:      */ SQLITE_NoopUpdate },
+#endif
 #endif
  {/* zName:     */ "optimize",
   /* ePragTyp:  */ PragTyp_OPTIMIZE,

src/random.c

@@ -106,6 +106,28 @@ void sqlite3_randomness(int N, void *pBuf){
   sqlite3_mutex_leave(mutex);
 }
 
+/*
+** Initialize a fast PRNG.  A Fast PRNG is called "fast" because it does
+** not need a mutex to operate, though it does use a mutex to initialize.
+** The quality of the randomness is not as good as the global PRNG.
+*/
+void sqlite3FastPrngInit(FastPrng *pPrng){
+  sqlite3_randomness(sizeof(*pPrng), pPrng);
+  pPrng->x |= 1;
+}
+
+/*
+** Generate N bytes of pseudo-randomness using a FastPrng
+*/
+void sqlite3FastRandomness(FastPrng *pPrng, int N, void *P){
+  unsigned char *pOut = (unsigned char*)P;
+  while( N-->0 ){
+    pPrng->x = ((pPrng->x)>>1) ^ ((1+~((pPrng->x)&1)) & 0xd0000001);
+    pPrng->y = (pPrng->y)*1103515245 + 12345;
+    *(pOut++) = (pPrng->x ^ pPrng->y) & 0xff;
+  }
+}
+
 #ifndef SQLITE_UNTESTABLE
 /*
 ** For testing purposes, we sometimes want to preserve the state of

src/select.c

@@ -1993,7 +1993,7 @@ int sqlite3ColumnsFromExprList(
         if( zName[j]==':' ) nName = j;
       }
       zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt);
-      if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt);
+      if( cnt>3 ) sqlite3FastRandomness(&db->sPrng, sizeof(cnt), &cnt);
     }
     pCol->zName = zName;
     sqlite3ColumnPropertiesFromName(0, pCol);

src/sqlite.h.in

@@ -9192,6 +9192,31 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp(
 */
 SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);
 
+/*
+** CAPI3REF: Wal related information regarding the most recent COMMIT
+** EXPERIMENTAL
+**
+** This function reports on the state of the wal file (if any) for database 
+** zDb, which should be "main", "temp", or the name of the attached database.
+** Its results - the values written to the output parameters - are only
+** defined if the most recent SQL command on the connection was a successful 
+** COMMIT that wrote data to wal-mode database zDb.
+**
+** Assuming the above conditions are met, output parameter (*pnFrame) is set
+** to the total number of frames in the wal file. Parameter (*pnPrior) is
+** set to the number of frames that were present in the wal file before the
+** most recent transaction was committed. So that the number of frames written
+** by the most recent transaction is (*pnFrame)-(*pnPrior).
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. It
+** is not an error if this function is called at a time when the results
+** are undefined.
+*/
+SQLITE_EXPERIMENTAL int sqlite3_wal_info(
+  sqlite3 *db, const char *zDb, 
+  unsigned int *pnPrior, unsigned int *pnFrame
+);
+
 /*
 ** CAPI3REF: Serialize a database
 **

src/sqliteInt.h

@@ -1089,6 +1089,7 @@ typedef struct Parse Parse;
 typedef struct PreUpdate PreUpdate;
 typedef struct PrintfArguments PrintfArguments;
 typedef struct RenameToken RenameToken;
+typedef struct FastPrng FastPrng;
 typedef struct RowSet RowSet;
 typedef struct Savepoint Savepoint;
 typedef struct Select Select;
@@ -1189,6 +1190,14 @@ typedef int VList;
 # define SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_SYNCHRONOUS
 #endif
 
+/*
+** State of a simple PRNG used for the per-connection and per-pager
+** pseudo-random number generators.
+*/
+struct FastPrng {
+  unsigned int x, y;
+};
+
 /*
 ** Each database file to be accessed by the system is an instance
 ** of the following structure.  There are normally two of these structures
@@ -1387,6 +1396,7 @@ struct sqlite3 {
   u16 dbOptFlags;               /* Flags to enable/disable optimizations */
   u8 enc;                       /* Text encoding */
   u8 autoCommit;                /* The auto-commit flag. */
+  u8 bConcurrent;               /* Current transaction is "CONCURRENT" */
   u8 temp_store;                /* 1: file 2: memory 0: default */
   u8 mallocFailed;              /* True if we have seen a malloc failure */
   u8 bBenignMalloc;             /* Do not require OOMs if true */
@@ -1400,6 +1410,7 @@ struct sqlite3 {
   u8 nSqlExec;                  /* Number of pending OP_SqlExec opcodes */
   int nextPagesize;             /* Pagesize after VACUUM if >0 */
   u32 magic;                    /* Magic number for detect library misuse */
+  FastPrng sPrng;               /* State of the per-connection PRNG */
   int nChange;                  /* Value returned by sqlite3_changes() */
   int nTotalChange;             /* Value returned by sqlite3_total_changes() */
   int aLimit[SQLITE_N_LIMIT];   /* Limits */
@@ -1547,6 +1558,7 @@ struct sqlite3 {
 #define SQLITE_NoSchemaError  0x08000000  /* Do not report schema parse errors*/
 #define SQLITE_Defensive      0x10000000  /* Input SQL is likely hostile */
 
+#define SQLITE_NoopUpdate     0x01000000  /* UPDATE operations are no-ops */
 /* Flags used only if debugging */
 #define HI(X)  ((u64)(X)<<32)
 #ifdef SQLITE_DEBUG
@@ -4002,6 +4014,8 @@ Vdbe *sqlite3GetVdbe(Parse*);
 void sqlite3PrngSaveState(void);
 void sqlite3PrngRestoreState(void);
 #endif
+void sqlite3FastPrngInit(FastPrng*);
+void sqlite3FastRandomness(FastPrng*, int N, void *P);
 void sqlite3RollbackAll(sqlite3*,int);
 void sqlite3CodeVerifySchema(Parse*, int);
 void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb);

src/test1.c

@@ -7612,6 +7612,41 @@ static int SQLITE_TCLAPI test_dbconfig_maindbname_icecube(
   }
 }
 
+/*
+** Usage: sqlite3_wal_info DB DBNAME
+*/
+static int SQLITE_TCLAPI test_wal_info(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int rc;
+  sqlite3 *db;
+  char *zName;
+  unsigned int nPrior;
+  unsigned int nFrame;
+
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME");
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+  zName = Tcl_GetString(objv[2]);
+
+  rc = sqlite3_wal_info(db, zName, &nPrior, &nFrame);
+  if( rc!=SQLITE_OK ){
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+    return TCL_ERROR;
+  }else{
+    Tcl_Obj *pNew = Tcl_NewObj();
+    Tcl_ListObjAppendElement(interp, pNew, Tcl_NewWideIntObj((i64)nPrior));
+    Tcl_ListObjAppendElement(interp, pNew, Tcl_NewWideIntObj((i64)nFrame));
+    Tcl_SetObjResult(interp, pNew);
+  }
+  return TCL_OK;
+}
+
 /*
 ** Usage: sqlite3_mmap_warm DB DBNAME
 */
@@ -7916,8 +7951,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_snapshot_open_blob", test_snapshot_open_blob, 0 },
      { "sqlite3_snapshot_cmp_blob", test_snapshot_cmp_blob, 0 },
 #endif
-     { "sqlite3_delete_database", test_delete_database,    0 },
-     { "atomic_batch_write",      test_atomic_batch_write, 0 },
+     { "sqlite3_delete_database", test_delete_database, 0 },
+     { "sqlite3_wal_info", test_wal_info, 0 },
+     { "atomic_batch_write",      test_atomic_batch_write,     0   },
      { "sqlite3_mmap_warm",       test_mmap_warm,          0 },
      { "sqlite3_config_sorterref", test_config_sorterref,   0 },
   };

src/test_config.c

@@ -672,6 +672,12 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
   Tcl_SetVar2(interp, "sqlite_options", "truncate_opt", "1", TCL_GLOBAL_ONLY);
 #endif
 
+#ifndef SQLITE_OMIT_CONCURRENT
+  Tcl_SetVar2(interp, "sqlite_options", "concurrent", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "concurrent", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_OMIT_UTF16
   Tcl_SetVar2(interp, "sqlite_options", "utf16", "0", TCL_GLOBAL_ONLY);
 #else

src/test_superlock.c

@@ -41,6 +41,8 @@ typedef struct SuperlockBusy SuperlockBusy;
 struct Superlock {
   sqlite3 *db;                    /* Database handle used to lock db */
   int bWal;                       /* True if db is a WAL database */
+  int bRecoveryLocked;            /* True if WAL RECOVERY lock is held */
+  int bReaderLocked;              /* True if WAL READER locks are held */
 };
 typedef struct Superlock Superlock;
 
@@ -107,12 +109,13 @@ static int superlockShmLock(
 ** Invoke the supplied busy-handler as required.
 */
 static int superlockWalLock(
-  sqlite3 *db,                    /* Database handle open on WAL database */
+  Superlock *pLock,               /* Superlock handle */
   SuperlockBusy *pBusy            /* Busy handler wrapper object */
 ){
   int rc;                         /* Return code */
   sqlite3_file *fd = 0;           /* Main database file handle */
   void volatile *p = 0;           /* Pointer to first page of shared memory */
+  sqlite3 *db = pLock->db;
 
   /* Obtain a pointer to the sqlite3_file object open on the main db file. */
   rc = sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, (void *)&fd);
@@ -121,8 +124,10 @@ static int superlockWalLock(
   /* Obtain the "recovery" lock. Normally, this lock is only obtained by
   ** clients running database recovery.  
   */
+  assert( pLock->bRecoveryLocked==0 );
   rc = superlockShmLock(fd, 2, 1, pBusy);
   if( rc!=SQLITE_OK ) return rc;
+  pLock->bRecoveryLocked = 1;
 
   /* Zero the start of the first shared-memory page. This means that any
   ** clients that open read or write transactions from this point on will
@@ -139,7 +144,9 @@ static int superlockWalLock(
   ** are held, it is guaranteed that there are no active reader, writer or 
   ** checkpointer clients.
   */
+  assert( pLock->bReaderLocked==0 );
   rc = superlockShmLock(fd, 3, SQLITE_SHM_NLOCK-3, pBusy);
+  if( rc==SQLITE_OK ) pLock->bReaderLocked = 1;
   return rc;
 }
 
@@ -156,8 +163,14 @@ void sqlite3demo_superunlock(void *pLock){
     sqlite3_file *fd = 0;
     rc = sqlite3_file_control(p->db, "main", SQLITE_FCNTL_FILE_POINTER, (void *)&fd);
     if( rc==SQLITE_OK ){
-      fd->pMethods->xShmLock(fd, 2, 1, flags);
-      fd->pMethods->xShmLock(fd, 3, SQLITE_SHM_NLOCK-3, flags);
+      if( p->bRecoveryLocked ){
+        fd->pMethods->xShmLock(fd, 2, 1, flags);
+        p->bRecoveryLocked = 0;
+      }
+      if( p->bReaderLocked ){
+        fd->pMethods->xShmLock(fd, 3, SQLITE_SHM_NLOCK-3, flags);
+        p->bReaderLocked = 0;
+      }
     }
   }
   sqlite3_close(p->db);
@@ -232,7 +245,7 @@ int sqlite3demo_superlock(
     if( SQLITE_OK==(rc = superlockIsWal(pLock)) && pLock->bWal ){
       rc = sqlite3_exec(pLock->db, "COMMIT", 0, 0, 0);
       if( rc==SQLITE_OK ){
-        rc = superlockWalLock(pLock->db, &busy);
+        rc = superlockWalLock(pLock, &busy);
       }
     }
   }

src/test_vfs.c

@@ -1563,8 +1563,62 @@ static int SQLITE_TCLAPI testvfs_cmd(
   return TCL_ERROR;
 }
 
+extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb);
+extern const char *sqlite3ErrName(int);
+
+/*
+** tclcmd: vfs_shmlock DB DBNAME (shared|exclusive) (lock|unlock) OFFSET N
+*/
+static int SQLITE_TCLAPI test_vfs_shmlock(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  const char *azArg1[] = {"shared", "exclusive", 0};
+  const char *azArg2[] = {"lock", "unlock", 0};
+  sqlite3 *db = 0;
+  int rc = SQLITE_OK;
+  const char *zDbname = 0;
+  int iArg1 = 0;
+  int iArg2 = 0;
+  int iOffset = 0;
+  int n = 0;
+  sqlite3_file *pFd;
+
+  if( objc!=7 ){
+    Tcl_WrongNumArgs(interp, 1, objv, 
+        "DB DBNAME (shared|exclusive) (lock|unlock) OFFSET N"
+    );
+    return TCL_ERROR;
+  }
+
+  zDbname = Tcl_GetString(objv[2]);
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) 
+   || Tcl_GetIndexFromObj(interp, objv[3], azArg1, "ARG", 0, &iArg1) 
+   || Tcl_GetIndexFromObj(interp, objv[4], azArg2, "ARG", 0, &iArg2) 
+   || Tcl_GetIntFromObj(interp, objv[5], &iOffset)
+   || Tcl_GetIntFromObj(interp, objv[6], &n)
+  ){
+    return TCL_ERROR;
+  }
+
+  sqlite3_file_control(db, zDbname, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd);
+  if( pFd==0 ){
+    return TCL_ERROR;
+  }
+  rc = pFd->pMethods->xShmLock(pFd, iOffset, n, 
+      (iArg1==0 ? SQLITE_SHM_SHARED : SQLITE_SHM_EXCLUSIVE)
+    | (iArg2==0 ? SQLITE_SHM_LOCK : SQLITE_SHM_UNLOCK)
+  );
+  Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+  return TCL_OK;
+}
+
+
 int Sqlitetestvfs_Init(Tcl_Interp *interp){
   Tcl_CreateObjCommand(interp, "testvfs", testvfs_cmd, 0, 0);
+  Tcl_CreateObjCommand(interp, "vfs_shmlock", test_vfs_shmlock, 0, 0);
   return TCL_OK;
 }
 

src/update.c

@@ -296,6 +296,17 @@ void sqlite3Update(
   */
   chngRowid = chngPk = 0;
   for(i=0; i<pChanges->nExpr; i++){
+#if defined(SQLITE_ENABLE_NOOP_UPDATE) && !defined(SQLITE_OMIT_FLAG_PRAGMAS)
+    if( db->flags & SQLITE_NoopUpdate ){
+      Token x;
+      sqlite3ExprDelete(db, pChanges->a[i].pExpr);
+      x.z = pChanges->a[i].zName;
+      x.n = sqlite3Strlen30(x.z);
+      pChanges->a[i].pExpr =
+         sqlite3PExpr(pParse, TK_UPLUS, sqlite3ExprAlloc(db, TK_ID, &x, 0), 0);
+      if( db->mallocFailed ) goto update_cleanup;
+    }
+#endif
     if( sqlite3ResolveExprNames(&sNC, pChanges->a[i].pExpr) ){
       goto update_cleanup;
     }

src/vacuum.c

@@ -361,6 +361,7 @@ end_of_vacuum:
   ** is closed by the DETACH.
   */
   db->autoCommit = 1;
+  assert( db->bConcurrent==0 );
 
   if( pDb ){
     sqlite3BtreeClose(pDb->pBt);

src/vdbe.c

@@ -3068,6 +3068,7 @@ case OP_Savepoint: {
       ** is committed. 
       */
       int isTransaction = pSavepoint->pNext==0 && db->isTransactionSavepoint;
+      assert( db->bConcurrent==0 || db->isTransactionSavepoint==0 );
       if( isTransaction && p1==SAVEPOINT_RELEASE ){
         if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){
           goto vdbe_return;
@@ -3144,23 +3145,31 @@ case OP_Savepoint: {
   break;
 }
 
-/* Opcode: AutoCommit P1 P2 * * *
+/* Opcode: AutoCommit P1 P2 P3 * *
 **
 ** Set the database auto-commit flag to P1 (1 or 0). If P2 is true, roll
 ** back any currently active btree transactions. If there are any active
 ** VMs (apart from this one), then a ROLLBACK fails.  A COMMIT fails if
 ** there are active writing VMs or active VMs that use shared cache.
 **
+** If P3 is non-zero, then this instruction is being executed as part of
+** a "BEGIN CONCURRENT" command.
+**
 ** This instruction causes the VM to halt.
 */
 case OP_AutoCommit: {
   int desiredAutoCommit;
   int iRollback;
+  int bConcurrent;
+  int hrc;
 
   desiredAutoCommit = pOp->p1;
   iRollback = pOp->p2;
+  bConcurrent = pOp->p3;
   assert( desiredAutoCommit==1 || desiredAutoCommit==0 );
   assert( desiredAutoCommit==1 || iRollback==0 );
+  assert( desiredAutoCommit==0 || bConcurrent==0 );
+  assert( db->autoCommit==0 || db->bConcurrent==0 );
   assert( db->nVdbeActive>0 );  /* At least this one VM is active */
   assert( p->bIsReader );
 
@@ -3169,10 +3178,17 @@ case OP_AutoCommit: {
       assert( desiredAutoCommit==1 );
       sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK);
       db->autoCommit = 1;
-    }else if( desiredAutoCommit && db->nVdbeWrite>0 ){
-      /* If this instruction implements a COMMIT and other VMs are writing
-      ** return an error indicating that the other VMs must complete first. 
-      */
+      db->bConcurrent = 0;
+    }else if( desiredAutoCommit
+            && (db->nVdbeWrite>0 || (db->bConcurrent && db->nVdbeActive>1)) ){
+      /* A transaction may only be committed if there are no other active
+      ** writer VMs. If the transaction is CONCURRENT, then it may only be
+      ** committed if there are no active VMs at all (readers or writers).
+      **
+      ** If this instruction is a COMMIT and the transaction may not be
+      ** committed due to one of the conditions above, return an error
+      ** indicating that other VMs must complete before the COMMIT can 
+      ** be processed.  */
       sqlite3VdbeError(p, "cannot commit transaction - "
                           "SQL statements in progress");
       rc = SQLITE_BUSY;
@@ -3182,12 +3198,15 @@ case OP_AutoCommit: {
     }else{
       db->autoCommit = (u8)desiredAutoCommit;
     }
-    if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){
+    hrc = sqlite3VdbeHalt(p);
+    if( (hrc & 0xFF)==SQLITE_BUSY ){
       p->pc = (int)(pOp - aOp);
       db->autoCommit = (u8)(1-desiredAutoCommit);
-      p->rc = rc = SQLITE_BUSY;
+      p->rc = hrc;
+      rc = SQLITE_BUSY;
       goto vdbe_return;
     }
+    db->bConcurrent = (u8)bConcurrent;
     assert( db->nStatement==0 );
     sqlite3CloseSavepoints(db);
     if( p->rc==SQLITE_OK ){
@@ -3378,10 +3397,22 @@ case OP_SetCookie: {
   pDb = &db->aDb[pOp->p1];
   assert( pDb->pBt!=0 );
   assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) );
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( db->bConcurrent 
+   && (pOp->p2==BTREE_USER_VERSION || pOp->p2==BTREE_APPLICATION_ID)
+  ){
+    rc = SQLITE_ERROR;
+    sqlite3VdbeError(p, "cannot modify %s within CONCURRENT transaction",
+        pOp->p2==BTREE_USER_VERSION ? "user_version" : "application_id"
+    );
+    goto abort_due_to_error;
+  }
+#endif
   /* See note about index shifting on OP_ReadCookie */
   rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, pOp->p3);
   if( pOp->p2==BTREE_SCHEMA_VERSION ){
     /* When the schema cookie changes, record the new cookie internally */
+    assert( pOp->p1==1 || db->bConcurrent==0 );
     pDb->pSchema->schema_cookie = pOp->p3;
     db->mDbFlags |= DBFLAG_SchemaChange;
   }else if( pOp->p2==BTREE_FILE_FORMAT ){
@@ -6766,6 +6797,14 @@ case OP_Expire: {
 */
 case OP_TableLock: {
   u8 isWriteLock = (u8)pOp->p3;
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( isWriteLock && db->bConcurrent && pOp->p2==1 && pOp->p1!=1 ){
+    rc = SQLITE_ERROR;
+    sqlite3VdbeError(p, 
+        "cannot modify database schema within CONCURRENT transaction");
+    goto abort_due_to_error;
+  }
+#endif
   if( isWriteLock || 0==(db->flags&SQLITE_ReadUncommit) ){
     int p1 = pOp->p1; 
     assert( p1>=0 && p1<db->nDb );

src/vdbeaux.c

@@ -2411,10 +2411,27 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
         assert( i!=1 );
         nTrans++;
       }
-      rc = sqlite3PagerExclusiveLock(pPager);
+      rc = sqlite3BtreeExclusiveLock(pBt);
       sqlite3BtreeLeave(pBt);
     }
   }
+
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( db->bConcurrent && (rc & 0xFF)==SQLITE_BUSY ){
+    /* An SQLITE_BUSY or SQLITE_BUSY_SNAPSHOT was encountered while 
+    ** attempting to take the WRITER lock on a wal file. Release the
+    ** WRITER locks on all wal files and return early.  */
+    for(i=0; i<db->nDb; i++){
+      Btree *pBt = db->aDb[i].pBt;
+      if( sqlite3BtreeIsInTrans(pBt) ){
+        sqlite3BtreeEnter(pBt);
+        sqlite3PagerDropExclusiveLock(sqlite3BtreePager(pBt));
+        sqlite3BtreeLeave(pBt);
+      }
+    }
+  }
+#endif
+
   if( rc!=SQLITE_OK ){
     return rc;
   }
@@ -2810,6 +2827,7 @@ int sqlite3VdbeHalt(Vdbe *p){
           sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK);
           sqlite3CloseSavepoints(db);
           db->autoCommit = 1;
+          db->bConcurrent = 0;
           p->nChange = 0;
         }
       }
@@ -2845,9 +2863,9 @@ int sqlite3VdbeHalt(Vdbe *p){
           ** is required. */
           rc = vdbeCommit(db, p);
         }
-        if( rc==SQLITE_BUSY && p->readOnly ){
+        if( (rc & 0xFF)==SQLITE_BUSY && p->readOnly ){
           sqlite3VdbeLeave(p);
-          return SQLITE_BUSY;
+          return rc;
         }else if( rc!=SQLITE_OK ){
           p->rc = rc;
           sqlite3RollbackAll(db, SQLITE_OK);
@@ -2872,6 +2890,7 @@ int sqlite3VdbeHalt(Vdbe *p){
         sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK);
         sqlite3CloseSavepoints(db);
         db->autoCommit = 1;
+        db->bConcurrent = 0;
         p->nChange = 0;
       }
     }
@@ -2893,6 +2912,7 @@ int sqlite3VdbeHalt(Vdbe *p){
         sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK);
         sqlite3CloseSavepoints(db);
         db->autoCommit = 1;
+        db->bConcurrent = 0;
         p->nChange = 0;
       }
     }

src/wal.c

@@ -471,8 +471,10 @@ struct Wal {
   WalIndexHdr hdr;           /* Wal-index header for current transaction */
   u32 minFrame;              /* Ignore wal frames before this one */
   u32 iReCksum;              /* On commit, recalculate checksums from here */
+  u32 nPriorFrame;           /* For sqlite3WalInfo() */
   const char *zWalName;      /* Name of WAL file */
   u32 nCkpt;                 /* Checkpoint sequence counter in the wal-header */
+  FastPrng sPrng;            /* Random number generator */
 #ifdef SQLITE_DEBUG
   u8 lockError;              /* True if a locking error has occurred */
 #endif
@@ -840,7 +842,7 @@ static const char *walLockName(int lockIdx){
 /*
 ** Set or release locks on the WAL.  Locks are either shared or exclusive.
 ** A lock cannot be moved directly between shared and exclusive - it must go
-** through the unlocked state first.
+** through the concurrent state first.
 **
 ** In locking_mode=EXCLUSIVE, all of these routines become no-ops.
 */
@@ -1140,7 +1142,7 @@ static int walIndexRecover(Wal *pWal){
   /* Obtain an exclusive lock on all byte in the locking range not already
   ** locked by the caller. The caller is guaranteed to have locked the
   ** WAL_WRITE_LOCK byte, and may have also locked the WAL_CKPT_LOCK byte.
-  ** If successful, the same bytes that are locked here are unlocked before
+  ** If successful, the same bytes that are locked here are concurrent before
   ** this function returns.
   */
   assert( pWal->ckptLock==1 || pWal->ckptLock==0 );
@@ -1379,6 +1381,7 @@ int sqlite3WalOpen(
   pRet->syncHeader = 1;
   pRet->padToSectorBoundary = 1;
   pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE);
+  sqlite3FastPrngInit(&pRet->sPrng);
 
   /* Open file handle on the write-ahead log file. */
   flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL);
@@ -1925,7 +1928,7 @@ static int walCheckpoint(
       rc = SQLITE_BUSY;
     }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){
       u32 salt1;
-      sqlite3_randomness(4, &salt1);
+      sqlite3FastRandomness(&pWal->sPrng, 4, &salt1);
       assert( pInfo->nBackfill==pWal->hdr.mxFrame );
       rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1);
       if( rc==SQLITE_OK ){
@@ -2041,6 +2044,49 @@ int sqlite3WalClose(
   return rc;
 }
 
+/*
+** Try to copy the wal-index header from shared-memory into (*pHdr). Return
+** zero if successful or non-zero otherwise. If the header is corrupted
+** (either because the two copies are inconsistent or because the checksum 
+** values are incorrect), the read fails and non-zero is returned.
+*/
+static int walIndexLoadHdr(Wal *pWal, WalIndexHdr *pHdr){
+  u32 aCksum[2];                  /* Checksum on the header content */
+  WalIndexHdr h2;                 /* Second copy of the header content */
+  WalIndexHdr volatile *aHdr;     /* Header in shared memory */
+
+  /* The first page of the wal-index must be mapped at this point. */
+  assert( pWal->nWiData>0 && pWal->apWiData[0] );
+
+  /* Read the header. This might happen concurrently with a write to the
+  ** same area of shared memory on a different CPU in a SMP,
+  ** meaning it is possible that an inconsistent snapshot is read
+  ** from the file. If this happens, return non-zero.
+  **
+  ** There are two copies of the header at the beginning of the wal-index.
+  ** When reading, read [0] first then [1].  Writes are in the reverse order.
+  ** Memory barriers are used to prevent the compiler or the hardware from
+  ** reordering the reads and writes.
+  */
+  aHdr = walIndexHdr(pWal);
+  memcpy(pHdr, (void *)&aHdr[0], sizeof(h2));
+  walShmBarrier(pWal);
+  memcpy(&h2, (void *)&aHdr[1], sizeof(h2));
+
+  if( memcmp(&h2, pHdr, sizeof(h2))!=0 ){
+    return 1;   /* Dirty read */
+  }  
+  if( h2.isInit==0 ){
+    return 1;   /* Malformed header - probably all zeros */
+  }
+  walChecksumBytes(1, (u8*)&h2, sizeof(h2)-sizeof(h2.aCksum), 0, aCksum);
+  if( aCksum[0]!=h2.aCksum[0] || aCksum[1]!=h2.aCksum[1] ){
+    return 1;   /* Checksum does not match */
+  }
+
+  return 0;
+}
+
 /*
 ** Try to read the wal-index header.  Return 0 on success and 1 if
 ** there is a problem.
@@ -2059,37 +2105,10 @@ int sqlite3WalClose(
 ** is read successfully and the checksum verified, return zero.
 */
 static int walIndexTryHdr(Wal *pWal, int *pChanged){
-  u32 aCksum[2];                  /* Checksum on the header content */
-  WalIndexHdr h1, h2;             /* Two copies of the header content */
-  WalIndexHdr volatile *aHdr;     /* Header in shared memory */
+  WalIndexHdr h1;                 /* Copy of the header content */
 
-  /* The first page of the wal-index must be mapped at this point. */
-  assert( pWal->nWiData>0 && pWal->apWiData[0] );
-
-  /* Read the header. This might happen concurrently with a write to the
-  ** same area of shared memory on a different CPU in a SMP,
-  ** meaning it is possible that an inconsistent snapshot is read
-  ** from the file. If this happens, return non-zero.
-  **
-  ** There are two copies of the header at the beginning of the wal-index.
-  ** When reading, read [0] first then [1].  Writes are in the reverse order.
-  ** Memory barriers are used to prevent the compiler or the hardware from
-  ** reordering the reads and writes.
-  */
-  aHdr = walIndexHdr(pWal);
-  memcpy(&h1, (void *)&aHdr[0], sizeof(h1));
-  walShmBarrier(pWal);
-  memcpy(&h2, (void *)&aHdr[1], sizeof(h2));
-
-  if( memcmp(&h1, &h2, sizeof(h1))!=0 ){
-    return 1;   /* Dirty read */
-  }  
-  if( h1.isInit==0 ){
-    return 1;   /* Malformed header - probably all zeros */
-  }
-  walChecksumBytes(1, (u8*)&h1, sizeof(h1)-sizeof(h1.aCksum), 0, aCksum);
-  if( aCksum[0]!=h1.aCksum[0] || aCksum[1]!=h1.aCksum[1] ){
-    return 1;   /* Checksum does not match */
+  if( walIndexLoadHdr(pWal, &h1) ){
+    return 1;
   }
 
   if( memcmp(&pWal->hdr, &h1, sizeof(WalIndexHdr)) ){
@@ -2756,6 +2775,7 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){
   testcase( rc==SQLITE_PROTOCOL );
   testcase( rc==SQLITE_OK );
 
+  pWal->nPriorFrame = pWal->hdr.mxFrame;
 #ifdef SQLITE_ENABLE_SNAPSHOT
   if( rc==SQLITE_OK ){
     if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){
@@ -2777,19 +2797,14 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){
       assert( pWal->readLock>0 || pWal->hdr.mxFrame==0 );
       assert( pInfo->aReadMark[pWal->readLock]<=pSnapshot->mxFrame );
 
-      /* It is possible that there is a checkpointer thread running 
-      ** concurrent with this code. If this is the case, it may be that the
-      ** checkpointer has already determined that it will checkpoint 
+      /* If it were possible for a checkpointer thread to run concurrent 
+      ** with this code, it would be a problem. In this case, it could be
+      ** that the checkpointer has already determined that it will checkpoint 
       ** snapshot X, where X is later in the wal file than pSnapshot, but 
       ** has not yet set the pInfo->nBackfillAttempted variable to indicate 
-      ** its intent. To avoid the race condition this leads to, ensure that
-      ** there is no checkpointer process by taking a shared CKPT lock 
-      ** before checking pInfo->nBackfillAttempted.  
-      **
-      ** TODO: Does the aReadMark[] lock prevent a checkpointer from doing
-      **       this already?
-      */
-      rc = walLockShared(pWal, WAL_CKPT_LOCK);
+      ** its intent. Fortunately this is not possible, as the call to
+      ** sqlite3WalSnapshotOpen() that sets pWal->pSnapshot also takes a
+      ** SHARED lock on the checkpointer slot.  */
 
       if( rc==SQLITE_OK ){
         /* Check that the wal file has not been wrapped. Assuming that it has
@@ -2808,8 +2823,6 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){
           rc = SQLITE_ERROR_SNAPSHOT;
         }
 
-        /* Release the shared CKPT lock obtained above. */
-        walUnlockShared(pWal, WAL_CKPT_LOCK);
         pWal->minFrame = 1;
       }
 
@@ -2867,8 +2880,7 @@ int sqlite3WalFindFrame(
     return SQLITE_OK;
   }
 
-  /* Search the hash table or tables for an entry matching page number
-  ** pgno. Each iteration of the following for() loop searches one
+  /* Each iteration of the following for() loop searches one
   ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames).
   **
   ** This code might run concurrently to the code in walIndexAppend()
@@ -2972,6 +2984,35 @@ Pgno sqlite3WalDbsize(Wal *pWal){
   return 0;
 }
 
+/*
+** Take the WRITER lock on the WAL file. Return SQLITE_OK if successful,
+** or an SQLite error code otherwise. This routine does not invoke any
+** busy-handler callbacks, that is done at a higher level.
+*/
+static int walWriteLock(Wal *pWal){
+  int rc;
+
+  /* Cannot start a write transaction without first holding a read lock */
+  assert( pWal->readLock>=0 );
+  assert( pWal->writeLock==0 );
+  assert( pWal->iReCksum==0 );
+
+  /* If this is a read-only connection, obtaining a write-lock is not
+  ** possible. In this case return SQLITE_READONLY. Otherwise, attempt
+  ** to grab the WRITER lock. Set Wal.writeLock to true and return
+  ** SQLITE_OK if successful, or leave Wal.writeLock clear and return 
+  ** an SQLite error code (possibly SQLITE_BUSY) otherwise. */
+  if( pWal->readOnly ){
+    rc = SQLITE_READONLY;
+  }else{
+    rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1);
+    if( rc==SQLITE_OK ){
+      pWal->writeLock = 1;
+    }
+  }
+
+  return rc;
+}
 
 /* 
 ** This function starts a write transaction on the WAL.
@@ -2987,39 +3028,203 @@ Pgno sqlite3WalDbsize(Wal *pWal){
 ** There can only be a single writer active at a time.
 */
 int sqlite3WalBeginWriteTransaction(Wal *pWal){
-  int rc;
-
-  /* Cannot start a write transaction without first holding a read
-  ** transaction. */
-  assert( pWal->readLock>=0 );
-  assert( pWal->writeLock==0 && pWal->iReCksum==0 );
-
-  if( pWal->readOnly ){
-    return SQLITE_READONLY;
+  int rc = walWriteLock(pWal);
+  if( rc==SQLITE_OK ){
+    /* If another connection has written to the database file since the
+    ** time the read transaction on this connection was started, then
+    ** the write is disallowed. Release the WRITER lock and return
+    ** SQLITE_BUSY_SNAPSHOT in this case.  */
+    if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){
+      walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1);
+      pWal->writeLock = 0;
+      rc = SQLITE_BUSY_SNAPSHOT;
+    }
   }
-
-  /* Only one writer allowed at a time.  Get the write lock.  Return
-  ** SQLITE_BUSY if unable.
-  */
-  rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1);
-  if( rc ){
-    return rc;
-  }
-  pWal->writeLock = 1;
-
-  /* If another connection has written to the database file since the
-  ** time the read transaction on this connection was started, then
-  ** the write is disallowed.
-  */
-  if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){
-    walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1);
-    pWal->writeLock = 0;
-    rc = SQLITE_BUSY_SNAPSHOT;
-  }
-
   return rc;
 }
 
+/*
+** This function is called by a writer that has a read-lock on aReadmark[0]
+** (pWal->readLock==0). This function relinquishes that lock and takes a
+** lock on a different aReadmark[] slot. 
+**
+** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
+*/
+static int walUpgradeReadlock(Wal *pWal){
+  int cnt;
+  int rc;
+  assert( pWal->writeLock && pWal->readLock==0 );
+  walUnlockShared(pWal, WAL_READ_LOCK(0));
+  pWal->readLock = -1;
+  cnt = 0;
+  do{
+    int notUsed;
+    rc = walTryBeginRead(pWal, &notUsed, 1, ++cnt);
+  }while( rc==WAL_RETRY );
+  assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */
+  testcase( (rc&0xff)==SQLITE_IOERR );
+  testcase( rc==SQLITE_PROTOCOL );
+  testcase( rc==SQLITE_OK );
+  return rc;
+}
+
+
+#ifndef SQLITE_OMIT_CONCURRENT
+/* 
+** This function is only ever called when committing a "BEGIN CONCURRENT"
+** transaction. It may be assumed that no frames have been written to
+** the wal file. The second parameter is a pointer to the in-memory 
+** representation of page 1 of the database (which may or may not be
+** dirty). The third is a bitvec with a bit set for each page in the
+** database file that was read by the current concurrent transaction.
+**
+** This function performs three tasks:
+**
+**   1) It obtains the WRITER lock on the wal file,
+**
+**   2) It checks that there are no conflicts between the current
+**      transaction and any transactions committed to the wal file since
+**      it was opened, and
+**
+**   3) It ejects any non-dirty pages from the page-cache that have been
+**      written by another client since the CONCURRENT transaction was started
+**      (so as to avoid ending up with an inconsistent cache after the
+**      current transaction is committed).
+**
+** If no error occurs and the caller may proceed with committing the 
+** transaction, SQLITE_OK is returned. SQLITE_BUSY is returned if the WRITER
+** lock cannot be obtained. Or, if the WRITER lock can be obtained but there
+** are conflicts with a committed transaction, SQLITE_BUSY_SNAPSHOT. Finally,
+** if an error (i.e. an OOM condition or IO error), an SQLite error code
+** is returned.
+*/
+int sqlite3WalLockForCommit(
+  Wal *pWal, 
+  PgHdr *pPage1, 
+  Bitvec *pAllRead, 
+  Pgno *piConflict
+){
+  Pager *pPager = pPage1->pPager;
+  int rc = walWriteLock(pWal);
+
+  /* If the database has been modified since this transaction was started,
+  ** check if it is still possible to commit. The transaction can be 
+  ** committed if:
+  **
+  **   a) None of the pages in pList have been modified since the 
+  **      transaction opened, and
+  **
+  **   b) The database schema cookie has not been modified since the
+  **      transaction was started.
+  */
+  if( rc==SQLITE_OK ){
+    WalIndexHdr head;
+
+    if( walIndexLoadHdr(pWal, &head) ){
+      /* This branch is taken if the wal-index header is corrupted. This 
+      ** occurs if some other writer has crashed while committing a 
+      ** transaction to this database since the current concurrent transaction
+      ** was opened.  */
+      rc = SQLITE_BUSY_SNAPSHOT;
+    }else if( memcmp(&pWal->hdr, (void*)&head, sizeof(WalIndexHdr))!=0 ){
+      int iHash;
+      int iLastHash = walFramePage(head.mxFrame);
+      u32 iFirst = pWal->hdr.mxFrame+1;     /* First wal frame to check */
+      if( memcmp(pWal->hdr.aSalt, (u32*)head.aSalt, sizeof(u32)*2) ){
+        assert( pWal->readLock==0 );
+        iFirst = 1;
+      }
+      for(iHash=walFramePage(iFirst); iHash<=iLastHash; iHash++){
+        WalHashLoc sLoc;
+
+        rc = walHashGet(pWal, iHash, &sLoc);
+        if( rc==SQLITE_OK ){
+          u32 i, iMin, iMax;
+          assert( head.mxFrame>=sLoc.iZero );
+          iMin = (sLoc.iZero >= iFirst) ? 1 : (iFirst - sLoc.iZero);
+          iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE;
+          if( iMax>(head.mxFrame-sLoc.iZero) ) iMax = (head.mxFrame-sLoc.iZero);
+          for(i=iMin; rc==SQLITE_OK && i<=iMax; i++){
+            PgHdr *pPg;
+            if( sLoc.aPgno[i]==1 ){
+              /* Check that the schema cookie has not been modified. If
+              ** it has not, the commit can proceed. */
+              u8 aNew[4];
+              u8 *aOld = &((u8*)pPage1->pData)[40];
+              int sz;
+              i64 iOffset;
+              sz = pWal->hdr.szPage;
+              sz = (sz&0xfe00) + ((sz&0x0001)<<16);
+              iOffset = walFrameOffset(i+sLoc.iZero, sz) + WAL_FRAME_HDRSIZE+40;
+              rc = sqlite3OsRead(pWal->pWalFd, aNew, sizeof(aNew), iOffset);
+              if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){
+                rc = SQLITE_BUSY_SNAPSHOT;
+              }
+            }else if( sqlite3BitvecTestNotNull(pAllRead, sLoc.aPgno[i]) ){
+              *piConflict = sLoc.aPgno[i];
+              rc = SQLITE_BUSY_SNAPSHOT;
+            }else if( (pPg = sqlite3PagerLookup(pPager, sLoc.aPgno[i])) ){
+              /* Page aPgno[i], which is present in the pager cache, has been
+              ** modified since the current CONCURRENT transaction was started.
+              ** However it was not read by the current transaction, so is not
+              ** a conflict. There are two possibilities: (a) the page was
+              ** allocated at the of the file by the current transaction or 
+              ** (b) was present in the cache at the start of the transaction.
+              **
+              ** For case (a), do nothing. This page will be moved within the
+              ** database file by the commit code to avoid the conflict. The
+              ** call to PagerUnref() is to release the reference grabbed by
+              ** the sqlite3PagerLookup() above.  
+              **
+              ** In case (b), drop the page from the cache - otherwise
+              ** following the snapshot upgrade the cache would be inconsistent
+              ** with the database as stored on disk. */
+              if( sqlite3PagerIswriteable(pPg) ){
+                sqlite3PagerUnref(pPg);
+              }else{
+                sqlite3PcacheDrop(pPg);
+              }
+            }
+          }
+        }
+        if( rc!=SQLITE_OK ) break;
+      }
+    }
+  }
+
+  pWal->nPriorFrame = pWal->hdr.mxFrame;
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** This function is called as part of committing an CONCURRENT transaction.
+** It is assumed that sqlite3WalLockForCommit() has already been successfully
+** called and so (a) the WRITER lock is held and (b) it is known that the
+** wal-index-header stored in shared memory is not corrupt.
+**
+** Before returning, this function upgrades the client so that it is 
+** operating on the database snapshot currently at the head of the wal file
+** (even if the CONCURRENT transaction ran against an older snapshot).
+**
+** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
+*/
+int sqlite3WalUpgradeSnapshot(Wal *pWal){
+  int rc = SQLITE_OK;
+  assert( pWal->writeLock );
+  memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr));
+
+  /* If this client has its read-lock on slot aReadmark[0] and the entire
+  ** wal has not been checkpointed, switch it to a different slot. Otherwise
+  ** any reads performed between now and committing the transaction will
+  ** read from the old snapshot - not the one just upgraded to.  */
+  if( pWal->readLock==0 && pWal->hdr.mxFrame!=walCkptInfo(pWal)->nBackfill ){
+    rc = walUpgradeReadlock(pWal);
+  }
+  return rc;
+}
+#endif   /* SQLITE_OMIT_CONCURRENT */
+
 /*
 ** End a write transaction.  The commit has already been done.  This
 ** routine merely releases the lock.
@@ -3048,7 +3253,7 @@ int sqlite3WalEndWriteTransaction(Wal *pWal){
 */
 int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){
   int rc = SQLITE_OK;
-  if( ALWAYS(pWal->writeLock) ){
+  if( pWal->writeLock ){
     Pgno iMax = pWal->hdr.mxFrame;
     Pgno iFrame;
   
@@ -3087,7 +3292,6 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){
 ** point in the event of a savepoint rollback (via WalSavepointUndo()).
 */
 void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){
-  assert( pWal->writeLock );
   aWalData[0] = pWal->hdr.mxFrame;
   aWalData[1] = pWal->hdr.aFrameCksum[0];
   aWalData[2] = pWal->hdr.aFrameCksum[1];
@@ -3103,7 +3307,7 @@ void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){
 int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){
   int rc = SQLITE_OK;
 
-  assert( pWal->writeLock );
+  assert( pWal->writeLock || aWalData[0]==pWal->hdr.mxFrame );
   assert( aWalData[3]!=pWal->nCkpt || aWalData[0]<=pWal->hdr.mxFrame );
 
   if( aWalData[3]!=pWal->nCkpt ){
@@ -3139,14 +3343,13 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){
 */
 static int walRestartLog(Wal *pWal){
   int rc = SQLITE_OK;
-  int cnt;
 
   if( pWal->readLock==0 ){
     volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
     assert( pInfo->nBackfill==pWal->hdr.mxFrame );
     if( pInfo->nBackfill>0 ){
       u32 salt1;
-      sqlite3_randomness(4, &salt1);
+      sqlite3FastRandomness(&pWal->sPrng, 4, &salt1);
       rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
       if( rc==SQLITE_OK ){
         /* If all readers are using WAL_READ_LOCK(0) (in other words if no
@@ -3160,21 +3363,21 @@ static int walRestartLog(Wal *pWal){
         ** to handle if this transaction is rolled back.  */
         walRestartHdr(pWal, salt1);
         walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
+        pWal->nPriorFrame = 0;
       }else if( rc!=SQLITE_BUSY ){
         return rc;
       }
     }
-    walUnlockShared(pWal, WAL_READ_LOCK(0));
-    pWal->readLock = -1;
-    cnt = 0;
-    do{
-      int notUsed;
-      rc = walTryBeginRead(pWal, &notUsed, 1, ++cnt);
-    }while( rc==WAL_RETRY );
-    assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */
-    testcase( (rc&0xff)==SQLITE_IOERR );
-    testcase( rc==SQLITE_PROTOCOL );
-    testcase( rc==SQLITE_OK );
+
+    /* Regardless of whether or not the wal file was restarted, change the
+    ** read-lock held by this client to a slot other than aReadmark[0]. 
+    ** Clients with a lock on aReadmark[0] read from the database file 
+    ** only - never from the wal file. This means that if a writer holding
+    ** a lock on aReadmark[0] were to commit a transaction but not close the
+    ** read-transaction, subsequent read operations would read directly from
+    ** the database file - ignoring the new pages just appended
+    ** to the wal file. */
+    rc = walUpgradeReadlock(pWal);
   }
   return rc;
 }
@@ -3362,7 +3565,7 @@ int sqlite3WalFrames(
     sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION);
     sqlite3Put4byte(&aWalHdr[8], szPage);
     sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt);
-    if( pWal->nCkpt==0 ) sqlite3_randomness(8, pWal->hdr.aSalt);
+    if( pWal->nCkpt==0 ) sqlite3FastRandomness(&pWal->sPrng, 8, pWal->hdr.aSalt);
     memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8);
     walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum);
     sqlite3Put4byte(&aWalHdr[24], aCksum[0]);
@@ -3443,6 +3646,7 @@ int sqlite3WalFrames(
     p->flags |= PGHDR_WAL_APPEND;
   }
 
+
   /* Recalculate checksums within the wal file if required. */
   if( isCommit && pWal->iReCksum ){
     rc = walRewriteChecksums(pWal, iFrame);
@@ -3617,7 +3821,6 @@ int sqlite3WalCheckpoint(
 
   /* Copy data from the log to the database file. */
   if( rc==SQLITE_OK ){
-
     if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){
       rc = SQLITE_CORRUPT_BKPT;
     }else{
@@ -3839,4 +4042,16 @@ sqlite3_file *sqlite3WalFile(Wal *pWal){
   return pWal->pWalFd;
 }
 
+/* 
+** Return the values required by sqlite3_wal_info().
+*/
+int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame){
+  int rc = SQLITE_OK;
+  if( pWal ){
+    *pnFrame = pWal->hdr.mxFrame;
+    *pnPrior = pWal->nPriorFrame;
+  }
+  return rc;
+}
+
 #endif /* #ifndef SQLITE_OMIT_WAL */

src/wal.h

@@ -136,6 +136,15 @@ int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapshot);
 void sqlite3WalSnapshotUnlock(Wal *pWal);
 #endif
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/* Tell the wal layer that we want to commit a concurrent transaction */
+int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPg, Bitvec *pRead, Pgno*);
+
+/* Upgrade the state of the client to take into account changes written
+** by other connections */
+int sqlite3WalUpgradeSnapshot(Wal *pWal);
+#endif /* SQLITE_OMIT_CONCURRENT */
+
 #ifdef SQLITE_ENABLE_ZIPVFS
 /* If the WAL file is not empty, return the number of bytes of content
 ** stored in each frame (i.e. the db page-size when the WAL was created).
@@ -146,5 +155,8 @@ int sqlite3WalFramesize(Wal *pWal);
 /* Return the sqlite3_file object for the WAL file */
 sqlite3_file *sqlite3WalFile(Wal *pWal);
 
+/* sqlite3_wal_info() data */
+int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame);
+
 #endif /* ifndef SQLITE_OMIT_WAL */
 #endif /* SQLITE_WAL_H */

test/bc_test1.c

@@ -0,0 +1,556 @@
+/*
+** 2016-05-07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+
+#include <sqlite3.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include "tt3_core.c"
+
+#ifdef USE_OSINST
+# include "../src/test_osinst.c"
+#else
+# define vfslog_time() 0
+#endif
+
+typedef struct Config Config;
+typedef struct ThreadCtx ThreadCtx;
+
+#define THREAD_TIME_INSERT   0
+#define THREAD_TIME_COMMIT   1
+#define THREAD_TIME_ROLLBACK 2
+#define THREAD_TIME_WRITER   3
+#define THREAD_TIME_CKPT     4
+
+struct ThreadCtx {
+  Config *pConfig;
+  Sqlite *pDb;
+  Error *pErr;
+  sqlite3_int64 aTime[5];
+};
+
+struct Config {
+  int nIPT;                       /* --inserts-per-transaction */
+  int nThread;                    /* --threads */
+  int nSecond;                    /* --seconds */
+  int bMutex;                     /* --mutex */
+  int nAutoCkpt;                  /* --autockpt */
+  int bRm;                        /* --rm */
+  int bClearCache;                /* --clear-cache */
+  int nMmap;                      /* mmap limit in MB */
+  char *zFile;
+  int bOsinst;                    /* True to use osinst */
+
+  ThreadCtx *aCtx;                /* Array of size nThread */
+
+  pthread_cond_t cond;
+  pthread_mutex_t mutex;
+  int nCondWait;                  /* Number of threads waiting on hCond */
+  sqlite3_vfs *pVfs;
+};
+
+
+typedef struct VfsWrapperFd VfsWrapperFd;
+struct VfsWrapperFd {
+  sqlite3_file base;              /* Base class */
+  int bWriter;                    /* True if holding shm WRITER lock */
+  int iTid;
+  Config *pConfig;
+  sqlite3_file *pFd;              /* Underlying file descriptor */
+};
+
+/* Methods of the wrapper VFS */
+static int vfsWrapOpen(sqlite3_vfs*, const char*, sqlite3_file*, int, int*);
+static int vfsWrapDelete(sqlite3_vfs*, const char*, int);
+static int vfsWrapAccess(sqlite3_vfs*, const char*, int, int*);
+static int vfsWrapFullPathname(sqlite3_vfs*, const char *, int, char*);
+static void *vfsWrapDlOpen(sqlite3_vfs*, const char*);
+static void vfsWrapDlError(sqlite3_vfs*, int, char*);
+static void (*vfsWrapDlSym(sqlite3_vfs*,void*, const char*))(void);
+static void vfsWrapDlClose(sqlite3_vfs*, void*);
+static int vfsWrapRandomness(sqlite3_vfs*, int, char*);
+static int vfsWrapSleep(sqlite3_vfs*, int);
+static int vfsWrapCurrentTime(sqlite3_vfs*, double*);
+static int vfsWrapGetLastError(sqlite3_vfs*, int, char*);
+static int vfsWrapCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
+static int vfsWrapSetSystemCall(sqlite3_vfs*, const char*, sqlite3_syscall_ptr);
+static sqlite3_syscall_ptr vfsWrapGetSystemCall(sqlite3_vfs*, const char*);
+static const char *vfsWrapNextSystemCall(sqlite3_vfs*, const char*);
+
+/* Methods of wrapper sqlite3_io_methods object (see vfsWrapOpen()) */
+static int vfsWrapClose(sqlite3_file*);
+static int vfsWrapRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
+static int vfsWrapWrite(sqlite3_file*, const void*, int iAmt, sqlite3_int64);
+static int vfsWrapTruncate(sqlite3_file*, sqlite3_int64 size);
+static int vfsWrapSync(sqlite3_file*, int flags);
+static int vfsWrapFileSize(sqlite3_file*, sqlite3_int64 *pSize);
+static int vfsWrapLock(sqlite3_file*, int);
+static int vfsWrapUnlock(sqlite3_file*, int);
+static int vfsWrapCheckReservedLock(sqlite3_file*, int *pResOut);
+static int vfsWrapFileControl(sqlite3_file*, int op, void *pArg);
+static int vfsWrapSectorSize(sqlite3_file*);
+static int vfsWrapDeviceCharacteristics(sqlite3_file*);
+static int vfsWrapShmMap(sqlite3_file*, int iPg, int, int, void volatile**);
+static int vfsWrapShmLock(sqlite3_file*, int offset, int n, int flags);
+static void vfsWrapShmBarrier(sqlite3_file*);
+static int vfsWrapShmUnmap(sqlite3_file*, int deleteFlag);
+static int vfsWrapFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **);
+static int vfsWrapUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
+
+static int vfsWrapOpen(
+  sqlite3_vfs *pVfs, 
+  const char *zName, 
+  sqlite3_file *pFd, 
+  int flags, 
+  int *fout
+){
+  static sqlite3_io_methods methods = {
+    3,
+    vfsWrapClose, vfsWrapRead, vfsWrapWrite,
+    vfsWrapTruncate, vfsWrapSync, vfsWrapFileSize,
+    vfsWrapLock, vfsWrapUnlock, vfsWrapCheckReservedLock,
+    vfsWrapFileControl, vfsWrapSectorSize, vfsWrapDeviceCharacteristics,
+    vfsWrapShmMap, vfsWrapShmLock, vfsWrapShmBarrier,
+    vfsWrapShmUnmap, vfsWrapFetch, vfsWrapUnfetch
+  };
+
+  Config *pConfig = (Config*)pVfs->pAppData;
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  int rc;
+
+  memset(pWrapper, 0, sizeof(VfsWrapperFd));
+  if( flags & SQLITE_OPEN_MAIN_DB ){
+    pWrapper->iTid = (int)sqlite3_uri_int64(zName, "tid", 0);
+  }
+
+  pWrapper->pFd = (sqlite3_file*)&pWrapper[1];
+  pWrapper->pConfig = pConfig;
+  rc = pConfig->pVfs->xOpen(pConfig->pVfs, zName, pWrapper->pFd, flags, fout);
+  if( rc==SQLITE_OK ){
+    pWrapper->base.pMethods = &methods;
+  }
+  return rc;
+}
+
+static int vfsWrapDelete(sqlite3_vfs *pVfs, const char *a, int b){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xDelete(pConfig->pVfs, a, b);
+}
+static int vfsWrapAccess(sqlite3_vfs *pVfs, const char *a, int b, int *c){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xAccess(pConfig->pVfs, a, b, c);
+}
+static int vfsWrapFullPathname(sqlite3_vfs *pVfs, const char *a, int b, char*c){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xFullPathname(pConfig->pVfs, a, b, c);
+}
+static void *vfsWrapDlOpen(sqlite3_vfs *pVfs, const char *a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xDlOpen(pConfig->pVfs, a);
+}
+static void vfsWrapDlError(sqlite3_vfs *pVfs, int a, char *b){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xDlError(pConfig->pVfs, a, b);
+}
+static void (*vfsWrapDlSym(sqlite3_vfs *pVfs, void *a, const char *b))(void){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xDlSym(pConfig->pVfs, a, b);
+}
+static void vfsWrapDlClose(sqlite3_vfs *pVfs, void *a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xDlClose(pConfig->pVfs, a);
+}
+static int vfsWrapRandomness(sqlite3_vfs *pVfs, int a, char *b){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xRandomness(pConfig->pVfs, a, b);
+}
+static int vfsWrapSleep(sqlite3_vfs *pVfs, int a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xSleep(pConfig->pVfs, a);
+}
+static int vfsWrapCurrentTime(sqlite3_vfs *pVfs, double *a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xCurrentTime(pConfig->pVfs, a);
+}
+static int vfsWrapGetLastError(sqlite3_vfs *pVfs, int a, char *b){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xGetLastError(pConfig->pVfs, a, b);
+}
+static int vfsWrapCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xCurrentTimeInt64(pConfig->pVfs, a);
+}
+static int vfsWrapSetSystemCall(
+  sqlite3_vfs *pVfs, 
+  const char *a, 
+  sqlite3_syscall_ptr b
+){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xSetSystemCall(pConfig->pVfs, a, b);
+}
+static sqlite3_syscall_ptr vfsWrapGetSystemCall(
+  sqlite3_vfs *pVfs, 
+  const char *a
+){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xGetSystemCall(pConfig->pVfs, a);
+}
+static const char *vfsWrapNextSystemCall(sqlite3_vfs *pVfs, const char *a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xNextSystemCall(pConfig->pVfs, a);
+}
+
+static int vfsWrapClose(sqlite3_file *pFd){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  pWrapper->pFd->pMethods->xClose(pWrapper->pFd);
+  pWrapper->pFd = 0;
+  return SQLITE_OK;
+}
+static int vfsWrapRead(sqlite3_file *pFd, void *a, int b, sqlite3_int64 c){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xRead(pWrapper->pFd, a, b, c);
+}
+static int vfsWrapWrite(
+  sqlite3_file *pFd, 
+  const void *a, int b, 
+  sqlite3_int64 c
+){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xWrite(pWrapper->pFd, a, b, c);
+}
+static int vfsWrapTruncate(sqlite3_file *pFd, sqlite3_int64 a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xTruncate(pWrapper->pFd, a);
+}
+static int vfsWrapSync(sqlite3_file *pFd, int a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xSync(pWrapper->pFd, a);
+}
+static int vfsWrapFileSize(sqlite3_file *pFd, sqlite3_int64 *a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xFileSize(pWrapper->pFd, a);
+}
+static int vfsWrapLock(sqlite3_file *pFd, int a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xLock(pWrapper->pFd, a);
+}
+static int vfsWrapUnlock(sqlite3_file *pFd, int a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xUnlock(pWrapper->pFd, a);
+}
+static int vfsWrapCheckReservedLock(sqlite3_file *pFd, int *a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xCheckReservedLock(pWrapper->pFd, a);
+}
+static int vfsWrapFileControl(sqlite3_file *pFd, int a, void *b){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xFileControl(pWrapper->pFd, a, b);
+}
+static int vfsWrapSectorSize(sqlite3_file *pFd){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xSectorSize(pWrapper->pFd);
+}
+static int vfsWrapDeviceCharacteristics(sqlite3_file *pFd){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xDeviceCharacteristics(pWrapper->pFd);
+}
+static int vfsWrapShmMap(
+  sqlite3_file *pFd, 
+  int a, int b, int c, 
+  void volatile **d
+){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xShmMap(pWrapper->pFd, a, b, c, d);
+}
+static int vfsWrapShmLock(sqlite3_file *pFd, int offset, int n, int flags){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  Config *pConfig = pWrapper->pConfig;
+  int bMutex = 0;
+  int rc;
+
+  if(  (offset==0 && n==1)
+    && (flags & SQLITE_SHM_LOCK) && (flags & SQLITE_SHM_EXCLUSIVE)
+  ){
+    pthread_mutex_lock(&pConfig->mutex);
+    pWrapper->bWriter = 1;
+    bMutex = 1;
+    if( pWrapper->iTid ){
+      sqlite3_int64 t = vfslog_time();
+      pConfig->aCtx[pWrapper->iTid-1].aTime[THREAD_TIME_WRITER] -= t;
+    }
+  }
+
+  rc = pWrapper->pFd->pMethods->xShmLock(pWrapper->pFd, offset, n, flags);
+
+  if( (rc!=SQLITE_OK && bMutex)
+   || (offset==0 && (flags & SQLITE_SHM_UNLOCK) && pWrapper->bWriter)
+  ){
+    assert( pWrapper->bWriter );
+    pthread_mutex_unlock(&pConfig->mutex);
+    pWrapper->bWriter = 0;
+    if( pWrapper->iTid ){
+      sqlite3_int64 t = vfslog_time();
+      pConfig->aCtx[pWrapper->iTid-1].aTime[THREAD_TIME_WRITER] += t;
+    }
+  }
+
+  return rc;
+}
+static void vfsWrapShmBarrier(sqlite3_file *pFd){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xShmBarrier(pWrapper->pFd);
+}
+static int vfsWrapShmUnmap(sqlite3_file *pFd, int a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xShmUnmap(pWrapper->pFd, a);
+}
+static int vfsWrapFetch(sqlite3_file *pFd, sqlite3_int64 a, int b, void **c){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xFetch(pWrapper->pFd, a, b, c);
+}
+static int vfsWrapUnfetch(sqlite3_file *pFd, sqlite3_int64 a, void *b){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xUnfetch(pWrapper->pFd, a, b);
+}
+
+static void create_vfs(Config *pConfig){
+  static sqlite3_vfs vfs = {
+    3, 0, 0, 0, "wrapper", 0,
+    vfsWrapOpen, vfsWrapDelete, vfsWrapAccess,
+    vfsWrapFullPathname, vfsWrapDlOpen, vfsWrapDlError,
+    vfsWrapDlSym, vfsWrapDlClose, vfsWrapRandomness,
+    vfsWrapSleep, vfsWrapCurrentTime, vfsWrapGetLastError,
+    vfsWrapCurrentTimeInt64, vfsWrapSetSystemCall, vfsWrapGetSystemCall,
+    vfsWrapNextSystemCall
+  };
+  sqlite3_vfs *pVfs;
+
+  pVfs = sqlite3_vfs_find(0);
+  vfs.mxPathname = pVfs->mxPathname;
+  vfs.szOsFile = pVfs->szOsFile + sizeof(VfsWrapperFd);
+  vfs.pAppData = (void*)pConfig;
+  pConfig->pVfs = pVfs;
+
+  sqlite3_vfs_register(&vfs, 1);
+}
+
+
+/*
+** Wal hook used by connections in thread_main().
+*/
+static int thread_wal_hook(
+  void *pArg,                     /* Pointer to ThreadCtx object */
+  sqlite3 *db,
+  const char *zDb, 
+  int nFrame
+){
+  ThreadCtx *pCtx = (ThreadCtx*)pArg;
+  Config *pConfig = pCtx->pConfig;
+
+  if( pConfig->nAutoCkpt && nFrame>=pConfig->nAutoCkpt ){
+    pCtx->aTime[THREAD_TIME_CKPT] -= vfslog_time();
+    pthread_mutex_lock(&pConfig->mutex);
+    if( pConfig->nCondWait>=0 ){
+      pConfig->nCondWait++;
+      if( pConfig->nCondWait==pConfig->nThread ){
+        execsql(pCtx->pErr, pCtx->pDb, "PRAGMA wal_checkpoint");
+        pthread_cond_broadcast(&pConfig->cond);
+      }else{
+        pthread_cond_wait(&pConfig->cond, &pConfig->mutex);
+      }
+      pConfig->nCondWait--;
+    }
+    pthread_mutex_unlock(&pConfig->mutex);
+    pCtx->aTime[THREAD_TIME_CKPT] += vfslog_time();
+  }
+
+  return SQLITE_OK;
+}
+
+
+static char *thread_main(int iTid, void *pArg){
+  Config *pConfig = (Config*)pArg;
+  Error err = {0};                /* Error code and message */
+  Sqlite db = {0};                /* SQLite database connection */
+  int nAttempt = 0;               /* Attempted transactions */
+  int nCommit = 0;                /* Successful transactions */
+  int j;
+  ThreadCtx *pCtx = &pConfig->aCtx[iTid-1];
+  char *zUri = 0;
+
+#ifdef USE_OSINST
+  char *zOsinstName = 0;
+  char *zLogName = 0;
+  if( pConfig->bOsinst ){
+    zOsinstName = sqlite3_mprintf("osinst%d", iTid);
+    zLogName = sqlite3_mprintf("bc_test1.log.%d.%d", (int)getpid(), iTid);
+    zUri = sqlite3_mprintf(
+        "file:%s?vfs=%s&tid=%d", pConfig->zFile, zOsinstName, iTid
+    );
+    sqlite3_vfslog_new(zOsinstName, 0, zLogName);
+    opendb(&err, &db, zUri, 0);
+  }else
+#endif
+  {
+    zUri = sqlite3_mprintf("file:%s?tid=%d", pConfig->zFile, iTid);
+    opendb(&err, &db, zUri, 0);
+  }
+
+  sqlite3_busy_handler(db.db, 0, 0);
+  sql_script_printf(&err, &db, 
+      "PRAGMA wal_autocheckpoint = 0;"
+      "PRAGMA synchronous = 0;"
+      "PRAGMA mmap_size = %lld;",
+      (i64)(pConfig->nMmap) * 1024 * 1024
+  );
+
+  pCtx->pConfig = pConfig;
+  pCtx->pErr = &err;
+  pCtx->pDb = &db;
+  sqlite3_wal_hook(db.db, thread_wal_hook, (void*)pCtx);
+
+  while( !timetostop(&err) ){
+    execsql(&err, &db, "BEGIN CONCURRENT");
+
+    pCtx->aTime[THREAD_TIME_INSERT] -= vfslog_time();
+    for(j=0; j<pConfig->nIPT; j++){
+      execsql(&err, &db, 
+          "INSERT INTO t1 VALUES"
+          "(randomblob(10), randomblob(20), randomblob(30), randomblob(200))"
+      );
+    }
+    pCtx->aTime[THREAD_TIME_INSERT] += vfslog_time();
+
+    pCtx->aTime[THREAD_TIME_COMMIT] -= vfslog_time();
+    execsql(&err, &db, "COMMIT");
+    pCtx->aTime[THREAD_TIME_COMMIT] += vfslog_time();
+
+    pCtx->aTime[THREAD_TIME_ROLLBACK] -= vfslog_time();
+    nAttempt++;
+    if( err.rc==SQLITE_OK ){
+      nCommit++;
+    }else{
+      clear_error(&err, SQLITE_BUSY);
+      execsql(&err, &db, "ROLLBACK");
+    }
+    pCtx->aTime[THREAD_TIME_ROLLBACK] += vfslog_time();
+
+    if( pConfig->bClearCache ){
+      sqlite3_db_release_memory(db.db);
+    }
+  }
+
+  closedb(&err, &db);
+
+#ifdef USE_OSINST
+  if( pConfig->bOsinst ){
+    sqlite3_vfslog_finalize(zOsinstName);
+    sqlite3_free(zOsinstName);
+    sqlite3_free(zLogName);
+  }
+#endif
+  sqlite3_free(zUri);
+
+  pthread_mutex_lock(&pConfig->mutex);
+  pConfig->nCondWait = -1;
+  pthread_cond_broadcast(&pConfig->cond);
+  pthread_mutex_unlock(&pConfig->mutex);
+
+  return sqlite3_mprintf("commits: %d/%d insert: %dms"
+      " commit: %dms"
+      " rollback: %dms" 
+      " writer: %dms"
+      " checkpoint: %dms", 
+      nCommit, nAttempt, 
+      (int)(pCtx->aTime[THREAD_TIME_INSERT]/1000), 
+      (int)(pCtx->aTime[THREAD_TIME_COMMIT]/1000), 
+      (int)(pCtx->aTime[THREAD_TIME_ROLLBACK]/1000),
+      (int)(pCtx->aTime[THREAD_TIME_WRITER]/1000),
+      (int)(pCtx->aTime[THREAD_TIME_CKPT]/1000)
+  );
+}
+
+int main(int argc, const char **argv){
+  Error err = {0};                /* Error code and message */
+  Sqlite db = {0};                /* SQLite database connection */
+  Threadset threads = {0};        /* Test threads */
+  Config conf = {5, 3, 5};
+  int i;
+
+  CmdlineArg apArg[] = {
+    { "-seconds", CMDLINE_INT,  offsetof(Config, nSecond) },
+    { "-inserts", CMDLINE_INT,  offsetof(Config, nIPT) },
+    { "-threads", CMDLINE_INT,  offsetof(Config, nThread) },
+    { "-mutex",   CMDLINE_BOOL, offsetof(Config, bMutex) },
+    { "-rm",      CMDLINE_BOOL, offsetof(Config, bRm) },
+    { "-autockpt",CMDLINE_INT,  offsetof(Config, nAutoCkpt) },
+    { "-mmap",    CMDLINE_INT,  offsetof(Config, nMmap) },
+    { "-clear-cache",    CMDLINE_BOOL,  offsetof(Config, bClearCache) },
+    { "-file",    CMDLINE_STRING,  offsetof(Config, zFile) },
+    { "-osinst",  CMDLINE_BOOL,  offsetof(Config, bOsinst) },
+    { 0, 0, 0 }
+  };
+
+  conf.nAutoCkpt = 1000;
+  cmdline_process(apArg, argc, argv, (void*)&conf);
+  if( err.rc==SQLITE_OK ){
+    char *z = cmdline_construct(apArg, (void*)&conf);
+    printf("With: %s\n", z);
+    sqlite3_free(z);
+  }
+  if( conf.zFile==0 ){
+    conf.zFile = "xyz.db";
+  }
+
+  /* Create the special VFS - "wrapper". And the mutex and condition 
+  ** variable. */
+  create_vfs(&conf);
+  pthread_mutex_init(&conf.mutex, 0);
+  pthread_cond_init(&conf.cond, 0);
+
+  conf.aCtx = sqlite3_malloc(sizeof(ThreadCtx) * conf.nThread);
+  memset(conf.aCtx, 0, sizeof(ThreadCtx) * conf.nThread);
+
+  /* Ensure the schema has been created */
+  opendb(&err, &db, conf.zFile, conf.bRm);
+  sql_script(&err, &db,
+      "PRAGMA journal_mode = wal;"
+      "CREATE TABLE IF NOT EXISTS t1(a PRIMARY KEY, b, c, d) WITHOUT ROWID;"
+      "CREATE INDEX IF NOT EXISTS t1b ON t1(b);"
+      "CREATE INDEX IF NOT EXISTS t1c ON t1(c);"
+  );
+
+  setstoptime(&err, conf.nSecond*1000);
+  if( conf.nThread==1 ){
+    char *z = thread_main(1, (void*)&conf);
+    printf("Thread 0 says: %s\n", (z==0 ? "..." : z));
+    fflush(stdout);
+  }else{
+    for(i=0; i<conf.nThread; i++){
+      launch_thread(&err, &threads, thread_main, (void*)&conf);
+    }
+    join_all_threads(&err, &threads);
+  }
+
+  if( err.rc==SQLITE_OK ){
+    printf("Database is %dK\n", (int)(filesize(&err, conf.zFile) / 1024));
+  }
+  if( err.rc==SQLITE_OK ){
+    char *zWal = sqlite3_mprintf("%s-wal", conf.zFile);
+    printf("Wal file is %dK\n", (int)(filesize(&err, zWal) / 1024));
+  }
+
+  closedb(&err, &db);
+  print_and_free_err(&err);
+  return 0;
+}

test/concfault.test

@@ -0,0 +1,86 @@
+# 2015 Aug 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains fault injection tests designed to test the concurrent
+# transactions feature.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set testprefix concfault
+
+# This test will not work with an in-memory journal, as the database will
+# become corrupt if an error is injected into a transaction after it starts
+# writing data out to the db file.
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+do_test 1-pre1 {
+  execsql {
+    PRAGMA journal_mode = wal;
+    CREATE TABLE t1(a PRIMARY KEY, b);
+    INSERT INTO t1 VALUES(randomblob(1000), randomblob(100));
+    INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
+    INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
+    INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
+    INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
+    DELETE FROM t1 WHERE rowid%2;
+  }
+  faultsim_save_and_close
+} {}
+
+do_faultsim_test 1.1 -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { 
+    BEGIN CONCURRENT;
+      INSERT INTO t1 VALUES(randomblob(1000), randomblob(100));
+    COMMIT;
+  }
+} -test {
+  faultsim_test_result {0 {}} 
+  catchsql { ROLLBACK }
+  faultsim_integrity_check
+}
+
+do_faultsim_test 1.2 -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { 
+    BEGIN CONCURRENT;
+      INSERT INTO t1 VALUES(randomblob(1000), randomblob(100));
+    ROLLBACK;
+  }
+} -test {
+  faultsim_test_result {0 {}} 
+  catchsql { ROLLBACK }
+  faultsim_integrity_check
+}
+
+do_faultsim_test 1.3 -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { 
+    BEGIN CONCURRENT;
+      DELETE FROM t1;
+    COMMIT;
+  }
+} -test {
+  faultsim_test_result {0 {}} 
+  catchsql { ROLLBACK }
+  faultsim_integrity_check
+}
+
+finish_test
+

test/concurrent.test

@@ -0,0 +1,678 @@
+# 2015 July 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set ::testprefix concurrent
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal;
+} {wal}
+
+do_execsql_test 1.1 {
+  CREATE TABLE t1(k INTEGER PRIMARY KEY, v);
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(1, 'abcd');
+  COMMIT;
+}
+
+do_execsql_test 1.2 {
+  SELECT * FROM t1;
+} {1 abcd}
+
+do_execsql_test 1.3 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(2, 'efgh');
+  ROLLBACK;
+}
+
+do_execsql_test 1.4 {
+  SELECT * FROM t1;
+} {1 abcd}
+
+
+#-------------------------------------------------------------------------
+# CONCURRENT transactions cannot do cache spills.
+#
+foreach {tn trans spill} {
+  1 {BEGIN CONCURRENT}  0
+  2 {BEGIN}           1
+} {
+  do_test 1.5.$tn {
+    sqlite3 db2 test.db
+    set walsz [file size test.db-wal]
+
+    execsql { PRAGMA cache_size = 10 } db2
+    execsql $trans db2
+    execsql {
+      WITH cnt(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM cnt WHERE i<50)
+        INSERT INTO t1(v) SELECT randomblob(900) FROM cnt;
+    } db2
+
+    expr {[file size test.db-wal]==$walsz}
+  } [expr !$spill]
+
+  execsql ROLLBACK db2
+  db2 close
+}
+
+#-------------------------------------------------------------------------
+# CONCURRENT transactions man not be committed while there are active
+# readers.
+do_execsql_test 1.6.setup {
+  DROP TABLE t1;
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 2);
+  INSERT INTO t1 VALUES(3, 4);
+  INSERT INTO t1 VALUES(5, 6);
+}
+foreach {tn trans commit_ok} {
+  1 {BEGIN CONCURRENT}  0
+  2 {BEGIN}           1
+} {
+  do_test 1.6.$tn.1 {
+    set stmt [sqlite3_prepare db "SELECT * FROM t1" -1 dummy]
+    sqlite3_step $stmt
+  } SQLITE_ROW
+  do_test 1.6.$tn.2 {
+    execsql $trans
+    execsql { INSERT INTO t1 VALUES(7, 8) }
+  } {}
+
+  if { $commit_ok } {
+    do_test 1.6.$tn.3 { catchsql COMMIT } {0 {}}
+  } else {
+    do_test 1.6.$tn.4 { catchsql COMMIT } {/1 {cannot commit transaction .*}/}
+  }
+
+  sqlite3_finalize $stmt
+  catchsql ROLLBACK
+}
+
+#-------------------------------------------------------------------------
+# CONCURRENT transactions may not modify the db schema.
+#
+foreach {tn sql} {
+  1 { CREATE TABLE xx(a, b) }
+  2 { DROP TABLE t1 }
+  3 { CREATE INDEX i1 ON t1(a) }
+  4 { CREATE VIEW v1 AS SELECT * FROM t1 }
+} {
+  do_catchsql_test 1.7.0.$tn.1 "
+    BEGIN CONCURRENT;
+    $sql
+  " {1 {cannot modify database schema within CONCURRENT transaction}}
+
+  do_execsql_test 1.7.0.$tn.2 {
+    SELECT sql FROM sqlite_master;
+    SELECT sql FROM sqlite_temp_master;
+  } {{CREATE TABLE t1(a, b)}}
+
+  do_execsql_test 1.7.0.$tn.3 COMMIT
+}
+
+# Except the temp db schema.
+foreach {tn sql} {
+  1 { CREATE TEMP TABLE xx(a, b) }
+  2 { DROP TABLE xx }
+  3 { CREATE TEMP TABLE yy(a, b) }
+  4 { CREATE VIEW temp.v1 AS SELECT * FROM t1 }
+  5 { CREATE INDEX yyi1 ON yy(a); }
+  6 { CREATE TABLE temp.zz(a, b) }
+} {
+  do_catchsql_test 1.7.1.$tn.1 "
+    BEGIN CONCURRENT;
+    $sql
+  " {0 {}}
+
+  do_execsql_test 1.7.1.$tn.2 COMMIT
+}
+
+
+do_execsql_test 1.7.1.x {
+  SELECT sql FROM sqlite_master;
+  SELECT sql FROM sqlite_temp_master;
+} {
+  {CREATE TABLE t1(a, b)}
+  {CREATE TABLE yy(a, b)} 
+  {CREATE VIEW v1 AS SELECT * FROM t1} 
+  {CREATE INDEX yyi1 ON yy(a)} 
+  {CREATE TABLE zz(a, b)}
+}
+
+#-------------------------------------------------------------------------
+# If an auto-vacuum database is written within an CONCURRENT transaction, it
+# is handled in the same way as for a non-CONCURRENT transaction.
+#
+reset_db
+do_execsql_test 1.8.1 {
+  PRAGMA auto_vacuum = 1;
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x, y);
+  INSERT INTO t1 VALUES('x', 'y');
+} {wal}
+
+do_execsql_test 1.8.2 {
+  BEGIN CONCURRENT;
+    SELECT * FROM t1;
+  COMMIT;
+} {x y}
+
+do_catchsql_test 1.8.3 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES('a', 'b');
+} {0 {}}
+
+do_test 1.8.4 {
+  sqlite3 db2 test.db
+  catchsql {
+    BEGIN CONCURRENT;
+      INSERT INTO t1 VALUES('c', 'd');
+  } db2
+} {1 {database is locked}}
+
+do_test 1.8.5 {
+  db eval COMMIT
+  db2 eval COMMIT
+} {}
+db close
+db2 close
+
+do_multiclient_test tn {
+
+  #-----------------------------------------------------------------------
+  # 1. Start an CONCURRENT transaction using [db1].
+  #
+  # 2. Start and then rollback a regular transaction using [db2]. This 
+  #    can be done as the ongoing [db1] transaction is CONCURRENT.
+  #
+  # 3. The [db1] transaction can now be committed, as [db2] has relinquished
+  #    the write lock.
+  #
+  do_test 2.$tn.1.1 {
+    sql1 { 
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(k INTEGER PRIMARY KEY, v);
+      INSERT INTO t1 VALUES(1, 'one');
+    }
+    sql1 { 
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(2, 'two');
+    }
+    code1 { sqlite3_get_autocommit db }
+  } 0
+
+  do_test 2.$tn.1.2 {
+    sql2 {
+      BEGIN;
+        INSERT INTO t1 VALUES(3, 'three');
+      ROLLBACK;
+    }
+  } {}
+
+  do_test 2.$tn.1.3 {
+    sql1 COMMIT
+    sql2 { SELECT * FROM t1 }
+  } {1 one 2 two}
+  
+  #-----------------------------------------------------------------------
+  # 1. Start an CONCURRENT transaction using [db1].
+  #
+  # 2. Commit a transaction using [db2].
+  #
+  # 3. Try to commit with [db1]. Check that SQLITE_BUSY_SNAPSHOT is returned,
+  #    and the transaction is not rolled back.
+  #
+  do_test 2.$tn.2.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(-1, 'hello world');
+    }
+  } {}
+
+  do_test 2.$tn.2.2 {
+    sql2 {
+      INSERT INTO t1 VALUES(3, 'three');
+    }
+  } {}
+
+  do_test 2.$tn.2.3.1 {
+    set rc [catch { sql1 COMMIT } msg]
+    list $rc $msg
+  } {1 {database is locked}}
+
+  do_test 2.$tn.2.3.2 {
+    code1 { list [sqlite3_extended_errcode db] [sqlite3_get_autocommit db] }
+  } {SQLITE_BUSY_SNAPSHOT 0}
+
+  do_test 2.$tn.2.3.3 {
+    sql1 {
+      SELECT * FROM t1;
+      ROLLBACK;
+    }
+  } {-1 {hello world} 1 one 2 two}
+  
+  #-----------------------------------------------------------------------
+  # 1. Start an CONCURRENT transaction using [db1].
+  #
+  # 2. Open a transaction using [db2].
+  #
+  # 3. Try to commit with [db1]. Check that SQLITE_BUSY is returned,
+  #    and the transaction is not rolled back.
+  #
+  # 4. Have [db2] roll its transaction back. Then check that [db1] can
+  #    commit.
+  #
+  do_test 2.$tn.3.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(4, 'four');
+    }
+  } {}
+
+  do_test 2.$tn.3.2 {
+    sql2 {
+      BEGIN;
+        INSERT INTO t1 VALUES(-1, 'xyz');
+    }
+  } {}
+
+  do_test 2.$tn.3.3.1 {
+    set rc [catch { sql1 COMMIT } msg]
+    list $rc $msg
+  } {1 {database is locked}}
+
+  do_test 2.$tn.3.3.2 {
+    code1 { list [sqlite3_extended_errcode db] [sqlite3_get_autocommit db] }
+  } {SQLITE_BUSY 0}
+
+  do_test 2.$tn.3.3.3 {
+    sql1 { SELECT * FROM t1; }
+  } {1 one 2 two 3 three 4 four}
+
+  do_test 2.$tn.3.4 {
+    sql2 ROLLBACK
+    sql1 COMMIT
+    sql1 { SELECT * FROM t1; }
+  } {1 one 2 two 3 three 4 four}
+
+  #-----------------------------------------------------------------------
+  # 1. Create a second table - t2.
+  #
+  # 2. Write to t1 with [db] and t2 with [db2].
+  #
+  # 3. See if it worked.
+  #
+  do_test 2.$tn.4.1 {
+    sql1 { CREATE TABLE t2(a, b) }
+  } {}
+  do_test 2.$tn.4.2 {
+    sql2 {
+      BEGIN CONCURRENT;
+        INSERT INTO t2 VALUES('i', 'n');
+    }
+
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(5, 'five');
+      COMMIT;
+    }
+
+    sql2 COMMIT
+  } {}
+
+  do_test 2.$tn.4.3.1 {
+    sql2 {SELECT * FROM t1}
+  } {1 one 2 two 3 three 4 four 5 five}
+  do_test 2.$tn.4.3.2 {
+    sql1 {SELECT * FROM t1}
+  } {1 one 2 two 3 three 4 four 5 five}
+
+  do_test 2.$tn.4.3.3 { sql2 {SELECT * FROM t2} } {i n}
+  do_test 2.$tn.4.3.4 { sql1 {SELECT * FROM t2} } {i n}
+
+  #-----------------------------------------------------------------------
+  # The "schema cookie" issue.
+  #
+  # 1. Begin and CONCURRENT write to "t1" using [db]
+  #
+  # 2. Create an index on t1 using [db2].
+  #
+  # 3. Attempt to commit the CONCURRENT write. This is an SQLITE_BUSY_SNAPSHOT,
+  #    even though there is no page collision.
+  #
+  do_test 2.$tn.5.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(6, 'six');
+    }
+  } {}
+
+  do_test 2.$tn.5.2 {
+    sql2 { CREATE INDEX i1 ON t1(v); }
+  } {}
+
+  do_test 2.$tn.5.3 {
+    list [catch { sql1 { COMMIT } } msg] $msg [sqlite3_errcode db]
+  } {1 {database is locked} SQLITE_BUSY_SNAPSHOT}
+
+  do_test 2.$tn.5.4 {
+    sql2 { PRAGMA integrity_check }
+  } {ok}
+  catch { sql1 ROLLBACK }
+
+  #-----------------------------------------------------------------------
+  #
+  # 1. Begin an CONCURRENT write to "t1" using [db]
+  #
+  # 2. Lots of inserts into t2. Enough to grow the db file and modify page 1.
+  #
+  # 3. Check that the CONCURRENT transaction can not be committed.
+  #
+  do_test 2.$tn.6.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(6, 'six');
+    }
+  } {}
+
+  do_test 2.$tn.6.2 {
+    sql2 { 
+      WITH src(a,b) AS (
+        VALUES(1,1) UNION ALL SELECT a+1,b+1 FROM src WHERE a<10000
+      ) INSERT INTO t2 SELECT * FROM src;
+    }
+  } {}
+
+  do_test 2.$tn.6.3 {
+    sql1 { SELECT count(*) FROM t2 }
+    list [catch { sql1 { COMMIT } } msg] $msg [sqlite3_errcode db]
+  } {1 {database is locked} SQLITE_BUSY_SNAPSHOT}
+  sql1 ROLLBACK
+
+  do_test 2.$tn.6.4 {
+    sql1 {
+      SELECT count(*) FROM t1;
+      SELECT count(*) FROM t2;
+    }
+  } {5 10001}
+
+  #-----------------------------------------------------------------------
+  # 
+  # 1. Begin an big CONCURRENT write to "t1" using [db] - large enough to
+  #    grow the db file.
+  #
+  # 2. Lots of inserts into t2. Also enough to grow the db file.
+  #
+  # 3. Check that the CONCURRENT transaction cannot be committed (due to a clash
+  #    on page 1 - the db size field).
+  #
+  do_test 2.$tn.7.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        WITH src(a,b) AS (
+          VALUES(10000,10000) UNION ALL SELECT a+1,b+1 FROM src WHERE a<20000
+        ) INSERT INTO t1 SELECT * FROM src;
+    }
+  } {}
+
+  do_test 2.$tn.7.2 {
+    sql2 { 
+      WITH src(a,b) AS (
+        VALUES(1,1) UNION ALL SELECT a+1,b+1 FROM src WHERE a<10000
+      ) INSERT INTO t2 SELECT * FROM src;
+    }
+  } {}
+
+  do_test 2.$tn.7.3 {
+    list [catch { sql1 { COMMIT } } msg] $msg [sqlite3_errcode db]
+  } {0 {} SQLITE_OK}
+
+  do_test 2.$tn.7.4 { sql3 { PRAGMA integrity_check } } ok
+}
+
+#-------------------------------------------------------------------------
+# Concurrent transactions may not modify the user_version or application_id.
+#
+reset_db
+do_execsql_test 3.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x, y);
+  INSERT INTO t1 VALUES('a', 'b');
+  PRAGMA user_version = 10;
+} {wal}
+do_execsql_test 3.1 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES('c', 'd');
+    SELECT * FROM t1;
+} {a b c d}
+do_catchsql_test 3.2 {
+  PRAGMA user_version = 11;
+} {1 {cannot modify user_version within CONCURRENT transaction}}
+do_execsql_test 3.3 {
+  PRAGMA user_version;
+  SELECT * FROM t1;
+} {10 a b c d}
+do_catchsql_test 3.4 {
+  PRAGMA application_id = 11;
+} {1 {cannot modify application_id within CONCURRENT transaction}}
+do_execsql_test 3.5 {
+  COMMIT;
+  PRAGMA user_version;
+  PRAGMA application_id;
+  SELECT * FROM t1;
+} {10 0 a b c d}
+
+#-------------------------------------------------------------------------
+# However, another transaction modifying the user_version or application_id
+# should not cause a conflict. And committing a concurrent transaction does not
+# clobber the modification - even if the concurrent transaction allocates or
+# frees database pages.
+#
+do_multiclient_test tn {
+  do_test 4.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE ttt(y UNIQUE, z UNIQUE);
+      PRAGMA user_version = 14;
+      BEGIN CONCURRENT;
+        INSERT INTO ttt VALUES('y', 'z');
+    }
+  } {wal}
+  do_test 4.$tn.2 {
+    sql2 { PRAGMA user_version = 16 }
+    sql1 COMMIT
+    sql1 { PRAGMA user_version }
+  } {16}
+
+  do_test 4.$tn.3 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO ttt VALUES(randomblob(10000), randomblob(4));
+        PRAGMA user_version;
+    }
+  } {16}
+  do_test 4.$tn.4 {
+    sql2 { PRAGMA user_version = 1234 }
+    sql1 {
+        PRAGMA user_version;
+      COMMIT;
+      PRAGMA user_version;
+      PRAGMA integrity_check;
+    }
+  } {16 1234 ok}
+
+  do_test 4.$tn.5 {
+    sql1 {
+      BEGIN CONCURRENT;
+        DELETE FROM ttt;
+        PRAGMA user_version;
+    }
+  } {1234}
+  do_test 4.$tn.4 {
+    sql2 { PRAGMA user_version = 5678 }
+    sql1 {
+        PRAGMA user_version;
+      COMMIT;
+      PRAGMA user_version;
+      PRAGMA integrity_check;
+    }
+  } {1234 5678 ok}
+}
+
+do_multiclient_test tn {
+  do_test 5.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE tt(a INTEGER PRIMARY KEY, b);
+      CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
+      INSERT INTO tt VALUES(1, randomblob(400));
+      BEGIN CONCURRENT;
+    }
+  } {wal}
+
+  do_test 5.$tn.2 {
+    sql1 { UPDATE t2 SET b=5 WHERE a=3 }
+    sql2 { INSERT INTO tt VALUES(2, randomblob(6000)) }
+  } {}
+
+  do_test 5.$tn.3 {
+    sql1 { COMMIT }
+  } {}
+}
+
+do_multiclient_test tn {
+  do_test 6.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+      CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
+      INSERT INTO t1 VALUES(1, 'one');
+      INSERT INTO t2 VALUES(2, 'two');
+    }
+  } {wal}
+
+  do_test 6.$tn.2 {
+    sql2 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t2;
+        INSERT INTO t1 VALUES(3, 'three');
+    }
+  } {2 two}
+
+  do_test 6.$tn.3 {
+    sql1 {
+      INSERT INTO t2 VALUES(3, 'three');
+    }
+  } {}
+
+  do_test 6.$tn.2 {
+    list [catch { sql2 { COMMIT } } msg] $msg
+  } {1 {database is locked}}
+}
+
+do_multiclient_test tn {
+  do_test 7.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+      WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100) 
+      INSERT INTO t1 SELECT NULL, randomblob(400) FROM s;
+
+      CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
+      WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<50000) 
+      INSERT INTO t2 SELECT NULL, randomblob(400) FROM s;
+
+      CREATE TABLE t3(a INTEGER PRIMARY KEY, b);
+      WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100) 
+      INSERT INTO t3 SELECT NULL, randomblob(400) FROM s;
+
+      CREATE TABLE t4(a INTEGER PRIMARY KEY, b);
+
+    }
+    set {} {}
+  } {}
+
+  do_test 7.$tn.2 {
+    sql2 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t1;
+        INSERT INTO t4 VALUES(1, 2);
+    }
+    set {} {}
+  } {}
+
+  do_test 7.$tn.3 {
+    sql3 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t3;
+        INSERT INTO t4 VALUES(1, 2);
+    }
+    set {} {}
+  } {}
+
+  do_test 7.$tn.4 {
+    sql1 {
+      UPDATE t1 SET b=randomblob(400);
+      UPDATE t2 SET b=randomblob(400);
+      UPDATE t3 SET b=randomblob(400);
+    }
+  } {}
+
+  do_test 7.$tn.5 {
+    csql2 { COMMIT } 
+  } {1 {database is locked}}
+
+  do_test 7.$tn.6 {
+    csql3 { COMMIT } 
+  } {1 {database is locked}}
+
+
+  csql2 ROLLBACK
+  csql3 ROLLBACK
+
+  # The following test works with $tn==1 (sql2 and sql3 use separate 
+  # processes), but is quite slow. So only run it with $tn==2 (all
+  # connections in the same process).
+  #
+  if {$tn==2} {
+    do_test 7.$tn.7 {
+      for {set i 1} {$i < 10000} {incr i} {
+        sql3 { 
+          PRAGMA wal_checkpoint;
+          BEGIN CONCURRENT;
+          SELECT * FROM t3;
+          INSERT INTO t4 VALUES(1, 2);
+        }
+
+        sql1 {
+          UPDATE t2 SET b = randomblob(400) WHERE rowid <= $i;
+          UPDATE t3 SET b = randomblob(400) WHERE rowid = 1;
+        }
+
+        if {[csql3 COMMIT]!={1 {database is locked}}} {
+          error "Failed at i=$i"
+        }
+        csql3 ROLLBACK
+      }
+    } {}
+  }
+
+}
+
+finish_test

test/concurrent2.test

@@ -0,0 +1,623 @@
+# 2015 July 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Miscellaneous tests for transactions started with BEGIN CONCURRENT. 
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+set ::testprefix concurrent2
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+do_multiclient_test tn {
+
+  do_test 1.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      CREATE TABLE t2(y);
+    }
+  } {wal}
+  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+
+  # Test that an CONCURRENT transaction that allocates/frees no pages does
+  # not conflict with a transaction that does allocate pages.
+  do_test 1.$tn.2  {
+    sql1 { 
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(4);
+    }
+    sql2 {
+      INSERT INTO t2 VALUES(randomblob(1500));
+    }
+    sql1 {
+      COMMIT;
+    }
+  } {}
+  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+  
+  # But that an CONCURRENT transaction does conflict with a transaction
+  # that modifies the db schema.
+  do_test 1.$tn.3  {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(5);
+    }
+    sql2 {
+      CREATE TABLE t3(z);
+    }
+    list [catch { sql1 COMMIT } msg] $msg
+  } {1 {database is locked}}
+  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+  
+  # Test that an CONCURRENT transaction that allocates at least one page 
+  # does not conflict with a transaction that allocates no pages.
+  do_test 1.$tn.4  {
+    sql1 { 
+      ROLLBACK;
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(randomblob(1500));
+    }
+    sql2 {
+      INSERT INTO t2 VALUES(8);
+    }
+    sql1 {
+      COMMIT;
+    }
+  } {}
+
+  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+}
+
+do_multiclient_test tn {
+  do_test 2.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x UNIQUE);
+      CREATE TABLE t2(y UNIQUE);
+    }
+  } {wal}
+
+  do_test 2.$tn.2  {
+    sql1 { 
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(randomblob(1500));
+    }
+    sql2 {
+      INSERT INTO t2 VALUES(randomblob(1500));
+    }
+    sql1 COMMIT
+  } {}
+
+  do_test 2.$tn.3 { sql3 { PRAGMA integrity_check } } {ok}
+
+  do_test 2.$tn.4  {
+    sql1 { 
+      BEGIN CONCURRENT;
+        DELETE FROM t1;
+    }
+    sql2 {
+      DELETE FROM t2;
+    }
+    sql1 COMMIT
+  } {}
+
+  do_test 2.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+
+  do_test 2.$tn.6 {
+    sql1 {
+      INSERT INTO t1 VALUES(randomblob(1500));
+      INSERT INTO t1 VALUES(randomblob(1500));
+      INSERT INTO t2 VALUES(randomblob(1500));
+      DELETE FROM t1 WHERE rowid=1;
+    }
+
+    sql1 {
+      BEGIN CONCURRENT;
+        DELETE FROM t1 WHERE rowid=2;
+    }
+
+    sql2 {
+      DELETE FROM t2;
+    }
+
+    sql1 COMMIT
+  } {}
+
+  do_test 2.$tn.7 { sql3 { PRAGMA integrity_check } } {ok}
+}
+
+#-------------------------------------------------------------------------
+# When an CONCURRENT transaction is opened on a database, the nFree and 
+# iTrunk header fields of the cached version of page 1 are both set 
+# to 0. This allows an CONCURRENT transaction to use its own private 
+# free-page-list, which is merged with the main database free-list when
+# the transaction is committed.
+#
+# The following tests check that nFree/iTrunk are correctly restored if
+# an CONCURRENT transaction is rolled back, and that savepoint rollbacks
+# that occur within CONCURRENT transactions do not incorrectly restore
+# these fields to their on-disk values.
+#
+reset_db
+do_execsql_test 3.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x, y);
+  INSERT INTO t1 VALUES(randomblob(1500), randomblob(1500));
+  DELETE FROM t1;
+} {wal}
+
+do_execsql_test 3.1 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(1, 2);
+  ROLLBACK;
+}
+
+do_execsql_test 3.2 { PRAGMA integrity_check } {ok}
+do_execsql_test 3.3 { PRAGMA freelist_count } {2}
+
+do_execsql_test 3.4.1 {
+  BEGIN CONCURRENT;
+    PRAGMA freelist_count;
+} {2}
+do_execsql_test 3.4.2 {
+  SAVEPOINT xyz;
+    INSERT INTO t1 VALUES(randomblob(1500), NULL);
+    PRAGMA freelist_count;
+} {0}
+do_execsql_test 3.4.3 {
+  ROLLBACK TO xyz;
+} {}
+do_execsql_test 3.4.4 { PRAGMA freelist_count } {0}
+do_execsql_test 3.4.5 { COMMIT; PRAGMA freelist_count } {2}
+do_execsql_test 3.4.6 { PRAGMA integrity_check } {ok}
+
+do_execsql_test 3.5.1 {
+  BEGIN CONCURRENT;
+    UPDATE t1 SET x=randomblob(10) WHERE y=555;
+    PRAGMA freelist_count;
+} {0}
+do_execsql_test 3.5.2 {
+  ROLLBACK;
+  PRAGMA freelist_count;
+} {2}
+do_execsql_test 3.5.3 { PRAGMA integrity_check } {ok}
+
+#-------------------------------------------------------------------------
+# Test that nothing goes wrong if an CONCURRENT transaction allocates a
+# page at the end of the file, frees it within the same transaction, and
+# then has to move the same page to avoid a conflict on COMMIT.
+#
+do_multiclient_test tn {
+  do_test 4.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      CREATE TABLE t2(x);
+    }
+  } {wal}
+
+  do_test 4.$tn.2 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(randomblob(1500));
+        INSERT INTO t1 VALUES(randomblob(1500));
+        DELETE FROM t1 WHERE rowid = 1;
+    }
+
+    sql2 {
+      INSERT INTO t2 VALUES(randomblob(1500));
+      INSERT INTO t2 VALUES(randomblob(1500));
+      INSERT INTO t2 VALUES(randomblob(1500));
+      INSERT INTO t2 VALUES(randomblob(1500));
+      DELETE FROM t2 WHERE rowid IN (1, 2);
+    }
+
+    sql1 COMMIT
+  } {}
+}
+
+#-------------------------------------------------------------------------
+#
+do_multiclient_test tn {
+  do_test 5.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      CREATE TABLE t2(x);
+      INSERT INTO t1 VALUES(randomblob(1500));
+      PRAGMA page_count;
+    }
+  } {wal 4}
+
+  do_test 5.$tn.2 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t2 VALUES(randomblob(1500));
+        PRAGMA page_count;
+    }
+  } {5}
+
+  do_test 5.$tn.3 {
+    sql2 { 
+      DELETE FROM t1;
+      PRAGMA freelist_count;
+      PRAGMA page_count;
+    }
+  } {1 4}
+
+  do_test 5.$tn.4 { sql1 COMMIT } {}
+  do_test 5.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+}
+
+#-------------------------------------------------------------------------
+#
+do_multiclient_test tn {
+  do_test 6.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      INSERT INTO t1 VALUES(randomblob(1500));
+      PRAGMA wal_checkpoint;
+    }
+  } {wal 0 5 5}
+
+  do_test 6.$tn.2 {
+    sql1 { 
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(randomblob(1500));
+        INSERT INTO t1 VALUES(randomblob(1500));
+    }
+  } {}
+
+  do_test 6.$tn.3 {
+    sql2 {
+      BEGIN;
+        INSERT INTO t1 VALUES(randomblob(1500));
+        INSERT INTO t1 VALUES(randomblob(1500));
+      COMMIT;
+    }
+  } {}
+
+  do_test 6.$tn.4 { 
+    list [catch { sql1 COMMIT } msg] $msg
+  } {1 {database is locked}}
+  do_test 6.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+  do_test 6.$tn.5 { sql3 { SELECT count(*) from t1 } } {3}
+}
+
+#-------------------------------------------------------------------------
+# Test that if a corrupt wal-index-header is encountered when attempting
+# to commit a CONCURRENT transaction, the transaction is not committed
+# (or rolled back) and that SQLITE_BUSY_SNAPSHOT is returned to the user.
+#
+catch { db close }
+forcedelete test.db
+testvfs tvfs
+sqlite3 db test.db -vfs tvfs
+do_execsql_test 7.1 {
+  PRAGMA journal_mode = wal;
+  BEGIN;
+    CREATE TABLE t1(a, b, PRIMARY KEY(a));
+    INSERT INTO t1 VALUES(1, 2);
+    INSERT INTO t1 VALUES(3, 4);
+  COMMIT;
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(5, 6);
+    INSERT INTO t1 VALUES(7, 8);
+    SELECT * FROM t1;
+} {wal 1 2 3 4 5 6 7 8}
+
+# Corrupt the wal-index header
+incr_tvfs_hdr test.db 11 1
+
+do_catchsql_test 7.2.1 { COMMIT } {1 {database is locked}}
+do_test 7.2.2 { sqlite3_extended_errcode db } SQLITE_BUSY_SNAPSHOT
+
+do_execsql_test 7.3.1 {
+  SELECT * FROM t1;
+  ROLLBACK;
+} {1 2 3 4 5 6 7 8}
+do_execsql_test 7.3.2 {
+  SELECT * FROM t1;
+} {1 2 3 4}
+
+#-------------------------------------------------------------------------
+# Test that "PRAGMA integrity_check" works within a concurrent 
+# transaction. Within a concurrent transaction, "PRAGMA integrity_check"
+# is unable to detect unused database pages, but can detect other types
+# of corruption.
+#
+reset_db
+do_execsql_test 8.1 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE kv(k INTEGER PRIMARY KEY, v UNIQUE);
+  INSERT INTO kv VALUES(NULL, randomblob(750));
+  INSERT INTO kv SELECT NULL, randomblob(750) FROM kv;
+  INSERT INTO kv SELECT NULL, randomblob(750) FROM kv;
+  INSERT INTO kv SELECT NULL, randomblob(750) FROM kv;
+  INSERT INTO kv SELECT NULL, randomblob(750) FROM kv;
+  INSERT INTO kv SELECT NULL, randomblob(750) FROM kv;
+  DELETE FROM kv WHERE rowid%2;
+  PRAGMA freelist_count;
+} {wal 34}
+do_execsql_test 8.2 { PRAGMA integrity_check } ok
+do_execsql_test 8.3 { 
+  BEGIN CONCURRENT;
+    PRAGMA integrity_check;
+} {ok}
+do_execsql_test 8.4 { 
+    INSERT INTO kv VALUES(1100, 1100);
+    PRAGMA integrity_check;
+} {ok}
+do_execsql_test 8.5 { 
+  COMMIT;
+  PRAGMA integrity_check;
+} {ok}
+
+#-------------------------------------------------------------------------
+# Test that concurrent transactions do not allow foreign-key constraints
+# to be bypassed.
+#
+do_multiclient_test tn {
+  do_test 9.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE pp(i INTEGER PRIMARY KEY, j);
+      CREATE TABLE cc(a, b REFERENCES pp);
+
+      WITH seq(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM seq WHERE i<100)
+      INSERT INTO pp SELECT i, randomblob(1000) FROM seq;
+
+      PRAGMA foreign_keys = 1;
+    }
+  } {wal}
+
+
+  do_test 9.$tn.2.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO cc VALUES(42, 42);
+    }
+  } {}
+  do_test 9.$tn.2.2 {
+    sql2 { DELETE FROM pp WHERE i=42 }
+    list [catch { sql1 COMMIT } msg] $msg
+  } {1 {database is locked}}
+  do_test 9.$tn.2.3 {
+    sql1 ROLLBACK
+  } {}
+
+  do_test 9.$tn.3.1 {
+    sql1 {
+      PRAGMA foreign_keys = 0;
+      BEGIN CONCURRENT;
+        INSERT INTO cc VALUES(43, 43);
+    }
+  } {}
+  do_test 9.$tn.3.2 {
+    sql2 { DELETE FROM pp WHERE i=43 }
+    list [catch { sql1 COMMIT } msg] $msg
+  } {0 {}}
+
+  do_test 9.$tn.4.1 {
+    sql1 {
+      PRAGMA foreign_keys = on;
+      BEGIN CONCURRENT;
+        INSERT INTO cc VALUES(44, 44);
+    }
+  } {}
+  do_test 9.$tn.4.2 {
+    sql2 { DELETE FROM pp WHERE i=1 }
+    list [catch { sql1 COMMIT } msg] $msg
+  } {0 {}}
+}
+
+#-------------------------------------------------------------------------
+# Test that even if a SELECT statement appears before all writes within
+# a CONCURRENT transaction, the pages it reads are still considered when
+# considering whether or not the transaction may be committed.
+#
+do_multiclient_test tn {
+  do_test 10.$tn.1.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(a);
+      CREATE TABLE t2(b);
+      CREATE TABLE t3(c);
+      INSERT INTO t1 VALUES(1), (2), (3);
+      INSERT INTO t2 VALUES(1), (2), (3);
+      INSERT INTO t3 VALUES(1), (2), (3);
+    }
+  } {wal}
+
+  do_test 10.$tn.1.2 {
+    sql1 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t1;
+        INSERT INTO t2 VALUES(4);
+    }
+  } {1 2 3}
+
+  do_test 10.$tn.1.3 {
+    sql2 { INSERT INTO t1 VALUES(4) }
+    list [catch {sql1 COMMIT} msg] $msg
+  } {1 {database is locked}}
+  sql1 ROLLBACK
+
+  # In this case, because the "SELECT * FROM t1" is first stepped before
+  # the "BEGIN CONCURRENT", the pages it reads are not recorded by the
+  # pager object. And so the transaction can be committed. Technically
+  # this behaviour (the effect of an ongoing SELECT on a BEGIN CONCURRENT
+  # transacation) is undefined.
+  #
+  do_test 10.$tn.2.1 {
+    code1 {
+      set ::stmt [sqlite3_prepare db "SELECT * FROM t1" -1 dummy]
+      sqlite3_step $::stmt
+    }
+  } {SQLITE_ROW}
+  do_test 10.$tn.2.2 {
+    sql1 {
+      BEGIN CONCURRENT; 
+        INSERT INTO t2 VALUES(4);
+    }
+    code1 {
+      set res [list]
+      lappend res [sqlite3_column_int $::stmt 0]
+      while {[sqlite3_step $::stmt]=="SQLITE_ROW"} {
+        lappend res [sqlite3_column_int $::stmt 0]
+      }
+      sqlite3_finalize $::stmt
+      set res
+    }
+  } {1 2 3 4}
+  do_test 10.$tn.2.3 {
+    sql2 { INSERT INTO t1 VALUES(5) }
+    sql1 COMMIT
+  } {}
+
+  # More tests surrounding long-lived prepared statements and concurrent
+  # transactions.
+  do_test 10.$tn.3.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t1;
+      COMMIT;
+    }
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t2 VALUES(5);
+    }
+    sql2 {
+      INSERT INTO t1 VALUES(5);
+    }
+    sql1 COMMIT
+    sql3 {
+      SELECT * FROM t2;
+    }
+  } {1 2 3 4 5}
+  do_test 10.$tn.3.2 {
+    sql1 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t1;
+      ROLLBACK;
+    }
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t2 VALUES(6);
+    }
+    sql2 {
+      INSERT INTO t1 VALUES(6);
+    }
+    sql1 COMMIT
+    sql3 { SELECT * FROM t2 }
+  } {1 2 3 4 5 6}
+  do_test 10.$tn.3.3 {
+    sql1 { BEGIN CONCURRENT }
+    code1 {
+      set ::stmt [sqlite3_prepare db "SELECT * FROM t1" -1 dummy]
+      sqlite3_step $::stmt
+    }
+    sql1 {
+      INSERT INTO t2 VALUES(7);
+      SELECT * FROM t3;
+      ROLLBACK;
+      BEGIN CONCURRENT;
+    }
+    sql2 { INSERT INTO t3 VALUES(5) }
+    code1 { sqlite3_finalize $::stmt }
+    sql1 {
+      INSERT INTO t2 VALUES(8);
+      COMMIT;
+    }
+  } {}
+}
+
+do_multiclient_test tn {
+  do_test 11.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(a);
+    }
+  } {wal}
+
+  do_test 11.$tn.2 {
+    code1 { sqlite3_wal_info db main }
+  } {0 2}
+
+  do_test 11.$tn.3 {
+    sql1 { INSERT INTO t1 VALUES(1) }
+    code1 { sqlite3_wal_info db main }
+  } {2 3}
+
+  do_test 11.$tn.4 {
+    sql2 { INSERT INTO t1 VALUES(2) }
+    code2 { sqlite3_wal_info db2 main }
+  } {3 4}
+
+  do_test 11.$tn.5 {
+    sql1 { PRAGMA wal_checkpoint }
+    sql2 { INSERT INTO t1 VALUES(3) }
+    code2 { sqlite3_wal_info db2 main }
+  } {0 1}
+}
+
+reset_db
+do_execsql_test 12.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE tx(a INTEGER PRIMARY KEY, b);
+} {wal}
+do_test 12.1 {
+  for {set i 0} {$i < 50} {incr i} {
+    execsql {
+      BEGIN CONCURRENT;
+      INSERT INTO tx(b) VALUES( randomblob( 1200 ) );
+      COMMIT;
+    }
+  }
+  execsql { PRAGMA page_size }
+} {1024}
+do_execsql_test 12.2 {
+  DELETE FROM tx;
+}
+do_test 12.3 {
+  for {set i 0} {$i < 50} {incr i} {
+    execsql {
+      BEGIN CONCURRENT;
+      INSERT INTO tx(b) VALUES( randomblob( 1200 ) );
+      COMMIT;
+    }
+  }
+  execsql { PRAGMA page_size }
+} {1024}
+do_execsql_test 12.4 {
+  DELETE FROM tx;
+}
+do_test 12.5 {
+  execsql { BEGIN CONCURRENT }
+  for {set i 0} {$i < 5000} {incr i} {
+    execsql {
+      INSERT INTO tx(b) VALUES( randomblob( 1200 ) );
+    }
+  }
+  execsql { COMMIT }
+  execsql { PRAGMA page_size }
+} {1024}
+
+
+finish_test

test/concurrent3.test

@@ -0,0 +1,231 @@
+# 2015 July 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests for transactions started with BEGIN CONCURRENT. The tests in this
+# file focus on testing that deferred page allocation works properly.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set ::testprefix concurrent3
+
+if {$AUTOVACUUM} { finish_test ; return }
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+db close
+sqlite3_shutdown
+test_sqlite3_log xLog
+proc xLog {error_code msg} {
+  # puts "$error_code: $msg"
+  # Enable the previous for debugging
+}
+reset_db
+
+proc create_schema {} {
+  db eval {
+    PRAGMA journal_mode = wal;
+
+    CREATE TABLE t1(x, y);
+    CREATE TABLE t2(x, y);
+    CREATE TABLE t3(x, y);
+    CREATE TABLE t4(x, y);
+
+    CREATE INDEX i1 ON t1(y, x);
+    CREATE INDEX i2 ON t2(y, x);
+    CREATE INDEX i3 ON t3(y, x);
+    CREATE INDEX i4 ON t4(y, x);
+  }
+}
+
+proc do_sql_op {iTbl iOp} {
+  set db "db$iTbl"
+
+  switch $iOp {
+    "i" {
+      set sql "
+        WITH cnt(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM cnt WHERE i<10)
+        INSERT INTO t$iTbl SELECT randomblob(800), randomblob(800) FROM cnt;
+      "
+    }
+
+    "d" {
+      set sql "
+        DELETE FROM t$iTbl WHERE rowid IN (
+          SELECT rowid FROM t$iTbl ORDER BY 1 ASC LIMIT 10
+        )
+      "
+    }
+
+    "D" {
+      set sql "
+        DELETE FROM t$iTbl WHERE rowid IN (
+          SELECT rowid FROM t$iTbl o WHERE (
+            SELECT count(*) FROM t$iTbl i WHERE i.rowid<o.rowid
+          ) % 2
+        )
+      "
+    }
+
+    "I" {
+      set sql "
+        INSERT INTO t$iTbl SELECT randomblob(800), randomblob(800) FROM t$iTbl;
+      "
+    }
+
+    default {
+      error "bad iOp parameter: $iOp"
+    }
+  }
+
+  $db eval $sql
+}
+
+
+set DBLIST {db1 db2 db3 db4} 
+
+create_schema
+foreach {tn oplist} {
+  1  {1i   2i   3i   4i} 
+  2  {1iii 2iii 3iii 4iii}
+  3  {1d   2d   3d   4d} 
+  .  -----------------------
+  4  {1i}
+  5  {1d 2i}
+  .  -----------------------
+  6  {1iii 2iii 3iii 4iii}
+  7  {1di  2id  3iii 4ddd}
+  8  {1iii 2iii 3iii 4iii}
+  9  {1D  2II}
+  10 {1I  2D  3I  4D}
+  11 {1III 3dddddd 4III}
+} {
+  if {[string range $oplist 0 0]=="-"} {
+    reset_db
+    create_schema
+    continue
+  }
+  foreach db $DBLIST { sqlite3 $db test.db }
+
+  do_test 1.$tn {
+    foreach db $DBLIST { $db eval "BEGIN CONCURRENT" } 
+
+    foreach op $oplist {
+      set iTbl [string range $op 0 0]
+      foreach char [split [string range $op 1 end] {}] {
+        do_sql_op $iTbl $char
+      }
+    }
+
+    foreach db $DBLIST { $db eval "COMMIT" }
+    db eval {PRAGMA integrity_check}
+  } {ok}
+
+  foreach db $DBLIST { 
+    $db close 
+  }
+}
+
+#-------------------------------------------------------------------------
+#
+proc create_schema2 {} {
+  db eval {
+    PRAGMA journal_mode = wal;
+    CREATE TABLE t1(x INTEGER PRIMARY KEY, y);
+    CREATE INDEX i1 ON t1(y);
+  }
+}
+
+proc randint {nMax} {
+  db eval {SELECT abs(random() % $nMax)}
+}
+
+proc do_sql_op2 {db iOp} {
+  switch -- $iOp {
+    i {
+      # Insert 1 rows.
+      set r [randint 1000000000]
+      set ::rows($r) 1
+      #puts "insert row $r"
+      $db eval { INSERT OR IGNORE INTO t1 VALUES($r, randomblob(50)); }
+    }
+
+    d {
+      # Insert 1 row
+      set keys [array names ::rows]
+      set r [randint [llength $keys]]
+      set rowid [lindex $keys $r]
+      $db eval { DELETE FROM t1 WHERE x=$rowid }
+      unset ::rows($rowid)
+    }
+  }
+}
+
+foreach {tn nRepeat oplist} {
+  - - ----------------------------
+  1 100 { 1iiiiiiiiii }
+  2 100 { 1i 2d }
+  3 100 { 1d 2i }
+  4  50 { 1d 2i 3d }
+  5 500 { 1i 2i 3i 4i }
+  6 500 { 1i 2d 3d 4d }
+} {
+  if {[string range $oplist 0 0]=="-"} {
+    array unset rows
+    reset_db
+    create_schema2
+    continue
+  }
+
+  foreach db $DBLIST { 
+    sqlite3 $db test.db 
+    set stats($db,0) 0
+    set stats($db,1) 0
+  }
+  array unset used
+
+  do_test 2.$tn {
+
+    for {set i 0} {$i < $nRepeat} {incr i} {
+      foreach db $DBLIST { $db eval "BEGIN CONCURRENT" } 
+
+      foreach op $oplist {
+        set iDb [string range $op 0 0]
+        set used(db$iDb) 1
+        foreach char [split [string range $op 1 end] {}] {
+          do_sql_op2 "db$iDb" $char
+        }
+      }
+
+      foreach db $DBLIST { 
+        set rc [catch { $db eval COMMIT } msg]
+        if {$rc} { $db eval ROLLBACK }
+        incr stats($db,$rc)
+      }
+      set res [db eval {PRAGMA integrity_check}]
+      if {$res != "ok"} { puts "after $db $rc: $res" ; after 1000000 }
+    }
+  } {}
+
+  foreach db $DBLIST { 
+    $db close 
+  }
+  # foreach k [lsort [array names used]] {
+  #   puts "$k: $stats($k,0) committed, $stats($k,1) rolled back"
+  # }
+}
+
+
+
+finish_test

test/concurrent4.test

@@ -0,0 +1,194 @@
+# 2017 May 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Miscellaneous tests for transactions started with BEGIN CONCURRENT. 
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+set ::testprefix concurrent4
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x PRIMARY KEY, y UNIQUE);
+  WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s;
+  DELETE FROM t1 WHERE rowid<2;
+} {wal}
+
+do_execsql_test 1.1 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1(rowid, x, y) VALUES(1000, randomblob(3000), randomblob(3000));
+    SAVEPOINT abc;
+    DELETE FROM t1 WHERE rowid = 1000;
+}
+
+do_execsql_test 1.2 { ROLLBACK TO abc }
+do_execsql_test 1.3 { COMMIT }
+do_execsql_test 1.4 { PRAGMA integrity_check } {ok}
+
+do_multiclient_test tn {
+  do_test 2.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(a, b);
+      WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100)
+        INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s;
+
+      CREATE TABLE t2(a, b);
+      WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100)
+        INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM s;
+    }
+
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(randomblob(3000), randomblob(3000));
+    }
+  } {}
+
+  do_test 2.$tn.2 {
+    sql2 {
+      WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10)
+        INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM s;
+    }
+    sql2 {
+      DELETE FROM t2 WHERE rowid<10;
+    }
+  } {}
+
+  do_test 2.$tn.3 {
+    sql1 {
+      COMMIT;
+      PRAGMA integrity_check;
+    }
+  } {ok}
+  do_test 2.$tn.4 {
+    sql2 {
+      PRAGMA integrity_check;
+    }
+  } {ok}
+}
+
+reset_db
+do_execsql_test 3.1 {
+  PRAGMA page_size = 1024;
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t2(x);
+  INSERT INTO t2 VALUES(randomblob(5000));
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+  INSERT INTO t1 VALUES(25, randomblob(104));
+  DELETE FROM t2;
+} {wal}
+
+do_execsql_test 3.2 {
+  BEGIN CONCURRENT;
+    REPLACE INTO t1 VALUES(25, randomblob(1117));
+  COMMIT;
+} {}
+
+#-------------------------------------------------------------------------
+# Test the effect of BEGIN CONCURRENT transactions that consist entirely
+# of read-only statements.
+#
+reset_db
+do_execsql_test 4.0 {
+  PRAGMA page_size = 1024;
+  PRAGMA journal_mode = wal;
+  
+  CREATE TABLE t4(a, b);
+  INSERT INTO t4 VALUES(1, 2);
+  INSERT INTO t4 VALUES(3, 4);
+} {wal}
+ 
+sqlite3 db2 test.db
+do_test 4.1.1 {
+  db eval {
+    BEGIN CONCURRENT;
+      INSERT INTO t4 VALUES(5, 6);
+  }
+
+  db2 eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+    ROLLBACK;
+  }
+} {1 2 3 4}
+
+do_test 4.1.2 {
+  db eval { COMMIT }
+  db2 eval { SELECT * FROM t4 }
+} {1 2 3 4 5 6}
+
+do_test 4.2.1 {
+  db eval {
+    BEGIN CONCURRENT;
+      INSERT INTO t4 VALUES(7, 8);
+  }
+
+  db2 eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+    COMMIT;
+  }
+} {1 2 3 4 5 6}
+
+do_test 4.2.2 {
+  db eval { COMMIT }
+  db2 eval { SELECT * FROM t4 }
+} {1 2 3 4 5 6 7 8}
+
+do_test 4.3 {
+  db2 eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+  }
+
+  db eval {
+    BEGIN CONCURRENT;
+      INSERT INTO t4 VALUES(9, 10);
+    COMMIT;
+  }
+  db2 eval {
+    SELECT * FROM t4;
+    COMMIT;
+  }
+} {1 2 3 4 5 6 7 8}
+
+set sz [file size test.db-wal]
+do_test 4.4.1 {
+  db eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+      SELECT * FROM sqlite_master;
+  }
+
+  db eval COMMIT
+  file size test.db-wal
+} $sz
+do_test 4.4.2 {
+  db eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+      SELECT * FROM sqlite_master;
+    ROLLBACK;
+  }
+  file size test.db-wal
+} $sz
+
+finish_test
+

test/concurrent5.test

@@ -0,0 +1,133 @@
+# 2017 May 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+set ::testprefix concurrent5
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+db close
+sqlite3_shutdown
+test_sqlite3_log [list lappend ::log]
+set ::log [list]
+
+sqlite3 db test.db
+
+proc do_test_conflict_msg {tn msg} {
+  set msg "cannot commit CONCURRENT transaction - [string trim $msg]"
+  uplevel [list do_test $tn {lindex $::log end} $msg]
+}
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x, y);
+  CREATE TABLE t2(c);
+  WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s;
+} {wal}
+
+sqlite3 db2 test.db
+
+do_test 1.1.1 {
+  set ::log [list]
+  db2 eval {
+    BEGIN CONCURRENT;
+      SELECT count(*) FROM t1;
+      INSERT INTO t2 VALUES(10);
+  }
+
+  db eval {
+    INSERT INTO t1 VALUES(randomblob(200), randomblob(200));
+  }
+
+  catchsql COMMIT db2
+} {1 {database is locked}}
+do_test_conflict_msg 1.1.2 {
+  conflict at page 2 (read-only page; part of db table t1; content=0500000063021100...)
+}
+
+do_test 1.2.1 {
+  set ::log [list]
+  db2 eval {
+    ROLLBACK;
+    BEGIN CONCURRENT;
+      INSERT INTO t1 VALUES(11, 12);
+  }
+
+  db eval {
+    INSERT INTO t1 VALUES(12, 11);
+  }
+
+  catchsql COMMIT db2
+} {1 {database is locked}}
+
+do_test_conflict_msg 1.2.2 {
+  conflict at page 105 (read/write page; part of db table t1; content=0D00000002026100...)
+}
+
+do_test 1.3.1 {
+  set ::log [list]
+  db2 eval {
+    ROLLBACK;
+    BEGIN CONCURRENT;
+      INSERT INTO t2 VALUES('x');
+  }
+
+  db eval {
+    INSERT INTO t2 VALUES('y');
+  }
+
+  catchsql COMMIT db2
+} {1 {database is locked}}
+
+do_test_conflict_msg 1.3.2 {
+  conflict at page 3 (read/write page; part of db table t2; content=0D0000000103FB00...)
+}
+
+do_test 1.4.1 {
+  set ::log [list]
+
+  execsql {
+    ROLLBACK;
+    CREATE TABLE t3(a INTEGER PRIMARY KEY, b INTEGER);
+    CREATE INDEX i3 ON t3(b);
+
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<5000
+    ) INSERT INTO t3 SELECT i, i FROM s;
+
+    BEGIN CONCURRENT;
+      INSERT INTO t3 VALUES(0, 5001);
+  } db2
+
+  execsql { INSERT INTO t3 VALUES(NULL, 5002) } db
+  catchsql COMMIT db2
+} {1 {database is locked}}
+
+do_test_conflict_msg 1.3.2 {
+  conflict at page 211 (read/write page; part of db index t3.i3; content=0A0310006300D800...)
+}
+
+db close
+db2 close
+sqlite3_shutdown
+test_sqlite3_log 
+sqlite3_initialize
+finish_test
+

test/concurrent6.test

@@ -0,0 +1,60 @@
+# 2017 May 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+set ::testprefix concurrent6
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+sqlite3 db2 test.db
+
+do_execsql_test 1.0 {
+  PRAGMA page_size = 1024;
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(x);
+  CREATE TABLE t3(x);
+  CREATE TABLE t4(x);
+
+  INSERT INTO t1 VALUES(zeroblob(1500));
+} {wal}
+
+do_execsql_test -db db2 1.1 {
+  BEGIN CONCURRENT;
+    INSERT INTO t3 VALUES(zeroblob(4000));
+    DELETE FROM t1;
+}
+
+do_execsql_test 1.2 {
+  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) 
+  INSERT INTO t2 SELECT zeroblob(1000) FROM s;
+
+  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) 
+  INSERT INTO t4  SELECT zeroblob(1000) FROM s;
+
+  DELETE FROM t4;
+}
+
+do_execsql_test -db db2 1.3 {
+  COMMIT;
+}
+
+
+finish_test
+

test/concurrent7.test

@@ -0,0 +1,52 @@
+# 2018 Jan 5
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix concurrent7
+
+sqlite3 db2 test.db
+
+do_execsql_test 1 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(x);
+} {wal}
+
+do_execsql_test -db db2 2 {
+  SELECT * FROM t1;
+}
+
+do_execsql_test 3 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(randomblob(1500));
+    INSERT INTO t1 VALUES(randomblob(1500));
+    DELETE FROM t1 WHERE rowid = 1;
+}
+
+do_execsql_test -db db2 4 {
+  INSERT INTO t2 VALUES(randomblob(1500));
+  INSERT INTO t2 VALUES(randomblob(1500));
+  INSERT INTO t2 VALUES(randomblob(1500));
+  INSERT INTO t2 VALUES(randomblob(1500));
+  DELETE FROM t2 WHERE rowid IN (1, 2);
+}
+
+do_execsql_test 5 {
+  COMMIT;
+  PRAGMA integrity_check;
+} {ok}
+
+finish_test
+
+

test/lock_common.tcl

@@ -15,18 +15,20 @@
 
 proc do_multiclient_test {varname script} {
 
-  foreach code [list {
+  foreach {tn code} [list 1 {
     if {[info exists ::G(valgrind)]} { db close ; continue }
     set ::code2_chan [launch_testfixture]
     set ::code3_chan [launch_testfixture]
     proc code2 {tcl} { testfixture $::code2_chan $tcl }
     proc code3 {tcl} { testfixture $::code3_chan $tcl }
-    set tn 1
-  } {
+  } 2 {
     proc code2 {tcl} { uplevel #0 $tcl }
     proc code3 {tcl} { uplevel #0 $tcl }
-    set tn 2
   }] {
+    # Do not run multi-process tests with the unix-excl VFS.
+    #
+    if {$tn==1 && [permutation]=="unix-excl"} continue
+
     faultsim_delete_and_reopen
 
     proc code1 {tcl} { uplevel #0 $tcl }

test/parser1.test

@@ -1,103 +0,0 @@
-# 2014-08-24
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-# This file implements regression tests for SQLite library.
-# The focus of this script is testing details of the SQL language parser.
-# 
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-do_catchsql_test parser1-1.1 {
-  CREATE TABLE t1(
-    a TEXT PRIMARY KEY,
-    b TEXT,
-    FOREIGN KEY(b COLLATE nocase DESC) REFERENCES t1(a COLLATE binary ASC)
-  );
-} {1 {syntax error after column name "b"}}
-
-
-# Verify that a legacy schema in the sqlite_master file is allowed to have
-# COLLATE, ASC, and DESC keywords on the id list of a FK constraint, and that
-# those keywords are silently ignored.
-#
-sqlite3_db_config db DEFENSIVE 0
-do_execsql_test parser1-1.2 {
-  CREATE TABLE t1(
-    a TEXT PRIMARY KEY,
-    b TEXT,
-    FOREIGN KEY(b) REFERENCES t1(a)
-  );
-  INSERT INTO t1 VALUES('abc',NULL),('xyz','abc');
-  PRAGMA writable_schema=on;
-  UPDATE sqlite_master SET sql='CREATE TABLE t1(
-    a TEXT PRIMARY KEY,
-    b TEXT,
-    FOREIGN KEY(b COLLATE nocase) REFERENCES t1(a)
-  )' WHERE name='t1';
-  SELECT name FROM sqlite_master WHERE sql LIKE '%collate%';
-} {t1}
-sqlite3 db2 test.db
-do_test parser1-1.3 {
-  sqlite3 db2 test.db
-  db2 eval {SELECT * FROM t1 ORDER BY 1}
-} {abc {} xyz abc}
-db2 close
-
-do_execsql_test parser1-1.4 {
-  UPDATE sqlite_master SET sql='CREATE TABLE t1(
-    a TEXT PRIMARY KEY,
-    b TEXT,
-    FOREIGN KEY(b ASC) REFERENCES t1(a)
-  )' WHERE name='t1';
-  SELECT name FROM sqlite_master WHERE sql LIKE '%ASC%';
-} {t1}
-sqlite3 db2 test.db
-do_test parser1-1.5 {
-  sqlite3 db2 test.db
-  db2 eval {SELECT * FROM t1 ORDER BY 1}
-} {abc {} xyz abc}
-db2 close
-
-do_catchsql_test parser1-2.1 {
-  WITH RECURSIVE
-    c(x COLLATE binary) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<5)
-  SELECT x FROM c;
-} {1 {syntax error after column name "x"}}
-do_catchsql_test parser1-2.2 {
-  WITH RECURSIVE
-    c(x ASC) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<5)
-  SELECT x FROM c;
-} {1 {syntax error after column name "x"}}
-
-# Verify that the comma between multiple table constraints is
-# optional.
-#
-# The missing comma is technically a syntax error.  But we have to support
-# it because there might be legacy databases that omit the commas in their
-# sqlite_master tables.
-#
-do_execsql_test parser1-3.1 {
-  CREATE TABLE t300(id INTEGER PRIMARY KEY);
-  CREATE TABLE t301(
-    id INTEGER PRIMARY KEY,
-    c1 INTEGER NOT NULL,
-    c2 INTEGER NOT NULL,
-    c3 BOOLEAN NOT NULL DEFAULT 0,
-    FOREIGN KEY(c1) REFERENCES t300(id) ON DELETE CASCADE ON UPDATE RESTRICT
-        /* no comma */
-    FOREIGN KEY(c2) REFERENCES t300(id) ON DELETE CASCADE ON UPDATE RESTRICT
-        /* no comma */
-    UNIQUE(c1, c2)
-  );
-  PRAGMA foreign_key_list(t301);
-} {0 0 t300 c2 id RESTRICT CASCADE NONE 1 0 t300 c1 id RESTRICT CASCADE NONE}
-
-finish_test

test/permutations.test

@@ -89,7 +89,6 @@ foreach f [glob $testdir/*.test] { lappend alltests [file tail $f] }
 foreach f [glob -nocomplain            \
     $testdir/../ext/rtree/*.test       \
     $testdir/../ext/fts5/test/*.test   \
-    $testdir/../ext/expert/*.test      \
     $testdir/../ext/lsm1/test/*.test   \
 ] {
   lappend alltests $f 
@@ -607,6 +606,16 @@ test_suite "onefile" -description {
   rollback.test  select1.test  select2.test  select3.test
 }
 
+# Run some tests using the "unix-excl" VFS.
+#
+test_suite "unix-excl" -description {
+  Run some tests using the "unix-excl" VFS
+} -initialize {
+  set ::G(perm:sqlite3_args) [list -vfs unix-excl]
+} -files {
+  shmlock.test
+}
+
 # Run some tests using UTF-16 databases.
 #
 test_suite "utf16" -description {

test/shmlock.test

@@ -0,0 +1,173 @@
+# 2018 December 6
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+set testprefix shmlock
+
+ifcapable !wal {finish_test ; return }
+
+sqlite3 db2 test.db
+sqlite3 db3 test.db
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 2);
+} {wal}
+do_test 1.1 { execsql { SELECT * FROM t1 } db2 } {1 2}
+do_test 1.2 { execsql { SELECT * FROM t1 } db3 } {1 2}
+
+foreach {tn dbhandle cmd res} {
+  1    db  {shared    lock   7 1}    OK
+  2    db2 {exclusive lock   7 1}    BUSY
+  3    db  {shared    unlock 7 1}    OK
+  4    db2 {exclusive lock   7 1}    OK
+  5    db  {shared    lock   7 1}    BUSY
+  6    db  {exclusive lock   7 1}    BUSY
+  7    db2 {exclusive unlock 7 1}    OK
+
+  8    db  {exclusive lock   0 8}    OK
+  9    db  {exclusive unlock 0 8}    OK
+  10   db2 {exclusive lock   0 8}    OK
+  11   db2 {exclusive unlock 0 8}    OK
+
+  12   db  {shared    lock   0 1}    OK
+  13   db2 {shared    lock   0 1}    OK
+  14   db3 {shared    lock   0 1}    OK
+  15   db3 {shared    unlock 0 1}    OK
+  16   db3 {exclusive lock   0 1}    BUSY
+  17   db2 {shared    unlock 0 1}    OK
+  18   db3 {exclusive lock   0 1}    BUSY
+  19   db  {shared    unlock 0 1}    OK
+  20   db3 {exclusive lock   0 1}    OK
+  21   db3 {exclusive unlock 0 1}    OK
+
+  22   db  {shared    lock   3 1}    OK
+  23   db2 {exclusive lock   2 2}    BUSY
+  24   db  {shared    lock   2 1}    OK
+  25   db2 {exclusive lock   0 5}    BUSY
+  26   db2 {exclusive lock   0 4}    BUSY
+  27   db2 {exclusive lock   0 3}    BUSY
+  28   db  {shared    unlock 3 1}    OK
+  29   db2 {exclusive lock   2 2}    BUSY
+  28   db  {shared    unlock 2 1}    OK
+  29   db2 {exclusive lock   2 2}    OK
+  29   db2 {exclusive unlock 2 2}    OK
+} {
+  do_test 1.3.$tn [list vfs_shmlock $dbhandle main {*}$cmd] "SQLITE_$res"
+}
+
+db  close
+db2 close
+db3 close
+
+if {[permutation]=="unix-excl"} {
+  do_test 2.0 {
+    for {set i 0} {$i < 256} {incr i} { 
+      sqlite3 db$i test.db 
+      execsql { SELECT * FROM t1 } db$i
+    }
+    for {set i 0} {$i < 255} {incr i} { 
+      set rc [vfs_shmlock db$i main shared lock 4 1]
+      if {$rc != "SQLITE_OK"} { error $rc }
+    }
+
+    vfs_shmlock db255 main shared lock 4 1
+  } {SQLITE_BUSY}
+
+  do_test 2.1 { vfs_shmlock db255 main exclusive lock   4 1 } SQLITE_BUSY
+  do_test 2.2 { vfs_shmlock db0   main shared    unlock 4 1 } SQLITE_OK
+  do_test 2.3 { vfs_shmlock db255 main shared    lock   4 1 } SQLITE_OK
+  do_test 2.4 { vfs_shmlock db255 main shared    unlock 4 1 } SQLITE_OK
+  do_test 2.5 { vfs_shmlock db255 main exclusive lock   4 1 } SQLITE_BUSY
+
+  do_test 2.6 {
+    for {set i 1} {$i < 255} {incr i} { 
+      set rc [vfs_shmlock db255 main exclusive lock 4 1]
+      if {$rc != "SQLITE_BUSY"} { error $rc }
+      set rc [vfs_shmlock db$i main shared unlock 4 1]
+      if {$rc != "SQLITE_OK"} { error $rc }
+    }
+
+    vfs_shmlock db255 main exclusive lock 4 1
+  } {SQLITE_OK}
+
+  vfs_shmlock db255 main exclusive unlock 4 1
+
+  for {set i 0} {$i < 256} {incr i} {
+    db$i close
+  }
+}
+
+sqlite3 db0 test.db
+sqlite3 db1 test.db
+do_test 3.1 { execsql { SELECT * FROM t1 } db0 } {1 2}
+do_test 3.2 { execsql { SELECT * FROM t1 } db1 } {1 2}
+
+set L(0) {n n n n n n n n}
+set L(1) {n n n n n n n n}
+proc random_lock_test {idx} {
+  global L
+  set iSlot [expr int(rand()*8)]
+  if {[expr int(rand()*2)]} {
+    # Unlock operation
+    if {[lindex $L($idx) $iSlot]!="n"} {
+      vfs_shmlock db$idx main [lindex $L($idx) $iSlot] unlock $iSlot 1
+      lset L($idx) $iSlot n
+    }
+  } else {
+    # Lock operation
+    if {[lindex $L($idx) $iSlot]=="n"} {
+      set locktype [lindex {e s} [expr int(rand()*2)]]
+      set n 1
+      if {$locktype=="e"} {
+        for {set l $iSlot} {$l<8 && [lindex $L($idx) $l]=="n"} {incr l} {}
+        set n [expr int(rand()*($l-$iSlot))+1]
+        # puts "iSlot=$iSlot l=$l L=$L($idx)"
+        # puts "$iSlot $n"
+      }
+      set res [vfs_shmlock db$idx main $locktype lock $iSlot $n]
+
+      set bBusy 0
+      for {set i $iSlot} {$i<($iSlot+$n)} {incr i} {
+        set other [lindex $L([expr ($idx+1)%2]) $i]
+        if {($other!="n" && $locktype=="e")||($other=="e" && $locktype=="s")} {
+          if {$res != "SQLITE_BUSY"} { error "BUSY not detected" }
+          set bBusy 1
+          break
+        } 
+      }
+
+      if {$bBusy==0} {
+        if {$res != "SQLITE_OK"} { error "BUSY false-positive" }
+        for {set i $iSlot} {$i<($iSlot+$n)} {incr i} {
+          lset L($idx) $i $locktype
+        }
+      }
+    }
+  }
+}
+
+set nStep 100000
+for {set i 0} {$i < $nStep} {incr i} {
+  random_lock_test 0
+  random_lock_test 1
+}
+
+db0 close
+db1 close
+
+finish_test
+
+

test/threadtest3.c

@@ -32,885 +32,20 @@
 ** The exit status is non-zero if any test fails.
 */
 
-/* 
-** The "Set Error Line" macro.
-*/
-#define SEL(e) ((e)->iLine = ((e)->rc ? (e)->iLine : __LINE__))
-
-/* Database functions */
-#define opendb(w,x,y,z)         (SEL(w), opendb_x(w,x,y,z))
-#define closedb(y,z)            (SEL(y), closedb_x(y,z))
-
-/* Functions to execute SQL */
-#define sql_script(x,y,z)       (SEL(x), sql_script_x(x,y,z))
-#define integrity_check(x,y)    (SEL(x), integrity_check_x(x,y))
-#define execsql_i64(x,y,...)    (SEL(x), execsql_i64_x(x,y,__VA_ARGS__))
-#define execsql_text(x,y,z,...) (SEL(x), execsql_text_x(x,y,z,__VA_ARGS__))
-#define execsql(x,y,...)        (SEL(x), (void)execsql_i64_x(x,y,__VA_ARGS__))
-#define sql_script_printf(x,y,z,...) (                \
-    SEL(x), sql_script_printf_x(x,y,z,__VA_ARGS__)    \
-) 
-
-/* Thread functions */
-#define launch_thread(w,x,y,z)     (SEL(w), launch_thread_x(w,x,y,z))
-#define join_all_threads(y,z)      (SEL(y), join_all_threads_x(y,z))
-
-/* Timer functions */
-#define setstoptime(y,z)        (SEL(y), setstoptime_x(y,z))
-#define timetostop(z)           (SEL(z), timetostop_x(z))
-
-/* Report/clear errors. */
-#define test_error(z, ...)      test_error_x(z, sqlite3_mprintf(__VA_ARGS__))
-#define clear_error(y,z)        clear_error_x(y, z)
-
-/* File-system operations */
-#define filesize(y,z)           (SEL(y), filesize_x(y,z))
-#define filecopy(x,y,z)         (SEL(x), filecopy_x(x,y,z))
-
-#define PTR2INT(x) ((int)((intptr_t)x))
-#define INT2PTR(x) ((void*)((intptr_t)x))
-
-/*
-** End of test code/infrastructure interface macros.
-*************************************************************************/
 
 
 
 
 #include <sqlite3.h>
-#include <unistd.h>
-#include <stdio.h>
-#include <pthread.h>
-#include <assert.h>
-#include <sys/types.h> 
-#include <sys/stat.h> 
-#include <string.h>
-#include <fcntl.h>
-#include <errno.h>
 
 #include "test_multiplex.h"
+#include "tt3_core.c"
 
 /* Required to link test_multiplex.c */
 #ifndef SQLITE_OMIT_WSD
 int sqlite3PendingByte = 0x40000000;
 #endif
 
-/*
- * This code implements the MD5 message-digest algorithm.
- * The algorithm is due to Ron Rivest.  This code was
- * written by Colin Plumb in 1993, no copyright is claimed.
- * This code is in the public domain; do with it what you wish.
- *
- * Equivalent code is available from RSA Data Security, Inc.
- * This code has been tested against that, and is equivalent,
- * except that you don't need to include two pages of legalese
- * with every copy.
- *
- * To compute the message digest of a chunk of bytes, declare an
- * MD5Context structure, pass it to MD5Init, call MD5Update as
- * needed on buffers full of bytes, and then call MD5Final, which
- * will fill a supplied 16-byte array with the digest.
- */
-
-/*
- * If compiled on a machine that doesn't have a 32-bit integer,
- * you just set "uint32" to the appropriate datatype for an
- * unsigned 32-bit integer.  For example:
- *
- *       cc -Duint32='unsigned long' md5.c
- *
- */
-#ifndef uint32
-#  define uint32 unsigned int
-#endif
-
-struct MD5Context {
-  int isInit;
-  uint32 buf[4];
-  uint32 bits[2];
-  union {
-    unsigned char in[64];
-    uint32 in32[16];
-  } u;
-};
-typedef struct MD5Context MD5Context;
-
-/*
- * Note: this code is harmless on little-endian machines.
- */
-static void byteReverse (unsigned char *buf, unsigned longs){
-  uint32 t;
-  do {
-    t = (uint32)((unsigned)buf[3]<<8 | buf[2]) << 16 |
-          ((unsigned)buf[1]<<8 | buf[0]);
-    *(uint32 *)buf = t;
-    buf += 4;
-  } while (--longs);
-}
-/* The four core functions - F1 is optimized somewhat */
-
-/* #define F1(x, y, z) (x & y | ~x & z) */
-#define F1(x, y, z) (z ^ (x & (y ^ z)))
-#define F2(x, y, z) F1(z, x, y)
-#define F3(x, y, z) (x ^ y ^ z)
-#define F4(x, y, z) (y ^ (x | ~z))
-
-/* This is the central step in the MD5 algorithm. */
-#define MD5STEP(f, w, x, y, z, data, s) \
-  ( w += f(x, y, z) + data,  w = w<<s | w>>(32-s),  w += x )
-
-/*
- * The core of the MD5 algorithm, this alters an existing MD5 hash to
- * reflect the addition of 16 longwords of new data.  MD5Update blocks
- * the data and converts bytes into longwords for this routine.
- */
-static void MD5Transform(uint32 buf[4], const uint32 in[16]){
-  register uint32 a, b, c, d;
-
-  a = buf[0];
-  b = buf[1];
-  c = buf[2];
-  d = buf[3];
-
-  MD5STEP(F1, a, b, c, d, in[ 0]+0xd76aa478,  7);
-  MD5STEP(F1, d, a, b, c, in[ 1]+0xe8c7b756, 12);
-  MD5STEP(F1, c, d, a, b, in[ 2]+0x242070db, 17);
-  MD5STEP(F1, b, c, d, a, in[ 3]+0xc1bdceee, 22);
-  MD5STEP(F1, a, b, c, d, in[ 4]+0xf57c0faf,  7);
-  MD5STEP(F1, d, a, b, c, in[ 5]+0x4787c62a, 12);
-  MD5STEP(F1, c, d, a, b, in[ 6]+0xa8304613, 17);
-  MD5STEP(F1, b, c, d, a, in[ 7]+0xfd469501, 22);
-  MD5STEP(F1, a, b, c, d, in[ 8]+0x698098d8,  7);
-  MD5STEP(F1, d, a, b, c, in[ 9]+0x8b44f7af, 12);
-  MD5STEP(F1, c, d, a, b, in[10]+0xffff5bb1, 17);
-  MD5STEP(F1, b, c, d, a, in[11]+0x895cd7be, 22);
-  MD5STEP(F1, a, b, c, d, in[12]+0x6b901122,  7);
-  MD5STEP(F1, d, a, b, c, in[13]+0xfd987193, 12);
-  MD5STEP(F1, c, d, a, b, in[14]+0xa679438e, 17);
-  MD5STEP(F1, b, c, d, a, in[15]+0x49b40821, 22);
-
-  MD5STEP(F2, a, b, c, d, in[ 1]+0xf61e2562,  5);
-  MD5STEP(F2, d, a, b, c, in[ 6]+0xc040b340,  9);
-  MD5STEP(F2, c, d, a, b, in[11]+0x265e5a51, 14);
-  MD5STEP(F2, b, c, d, a, in[ 0]+0xe9b6c7aa, 20);
-  MD5STEP(F2, a, b, c, d, in[ 5]+0xd62f105d,  5);
-  MD5STEP(F2, d, a, b, c, in[10]+0x02441453,  9);
-  MD5STEP(F2, c, d, a, b, in[15]+0xd8a1e681, 14);
-  MD5STEP(F2, b, c, d, a, in[ 4]+0xe7d3fbc8, 20);
-  MD5STEP(F2, a, b, c, d, in[ 9]+0x21e1cde6,  5);
-  MD5STEP(F2, d, a, b, c, in[14]+0xc33707d6,  9);
-  MD5STEP(F2, c, d, a, b, in[ 3]+0xf4d50d87, 14);
-  MD5STEP(F2, b, c, d, a, in[ 8]+0x455a14ed, 20);
-  MD5STEP(F2, a, b, c, d, in[13]+0xa9e3e905,  5);
-  MD5STEP(F2, d, a, b, c, in[ 2]+0xfcefa3f8,  9);
-  MD5STEP(F2, c, d, a, b, in[ 7]+0x676f02d9, 14);
-  MD5STEP(F2, b, c, d, a, in[12]+0x8d2a4c8a, 20);
-
-  MD5STEP(F3, a, b, c, d, in[ 5]+0xfffa3942,  4);
-  MD5STEP(F3, d, a, b, c, in[ 8]+0x8771f681, 11);
-  MD5STEP(F3, c, d, a, b, in[11]+0x6d9d6122, 16);
-  MD5STEP(F3, b, c, d, a, in[14]+0xfde5380c, 23);
-  MD5STEP(F3, a, b, c, d, in[ 1]+0xa4beea44,  4);
-  MD5STEP(F3, d, a, b, c, in[ 4]+0x4bdecfa9, 11);
-  MD5STEP(F3, c, d, a, b, in[ 7]+0xf6bb4b60, 16);
-  MD5STEP(F3, b, c, d, a, in[10]+0xbebfbc70, 23);
-  MD5STEP(F3, a, b, c, d, in[13]+0x289b7ec6,  4);
-  MD5STEP(F3, d, a, b, c, in[ 0]+0xeaa127fa, 11);
-  MD5STEP(F3, c, d, a, b, in[ 3]+0xd4ef3085, 16);
-  MD5STEP(F3, b, c, d, a, in[ 6]+0x04881d05, 23);
-  MD5STEP(F3, a, b, c, d, in[ 9]+0xd9d4d039,  4);
-  MD5STEP(F3, d, a, b, c, in[12]+0xe6db99e5, 11);
-  MD5STEP(F3, c, d, a, b, in[15]+0x1fa27cf8, 16);
-  MD5STEP(F3, b, c, d, a, in[ 2]+0xc4ac5665, 23);
-
-  MD5STEP(F4, a, b, c, d, in[ 0]+0xf4292244,  6);
-  MD5STEP(F4, d, a, b, c, in[ 7]+0x432aff97, 10);
-  MD5STEP(F4, c, d, a, b, in[14]+0xab9423a7, 15);
-  MD5STEP(F4, b, c, d, a, in[ 5]+0xfc93a039, 21);
-  MD5STEP(F4, a, b, c, d, in[12]+0x655b59c3,  6);
-  MD5STEP(F4, d, a, b, c, in[ 3]+0x8f0ccc92, 10);
-  MD5STEP(F4, c, d, a, b, in[10]+0xffeff47d, 15);
-  MD5STEP(F4, b, c, d, a, in[ 1]+0x85845dd1, 21);
-  MD5STEP(F4, a, b, c, d, in[ 8]+0x6fa87e4f,  6);
-  MD5STEP(F4, d, a, b, c, in[15]+0xfe2ce6e0, 10);
-  MD5STEP(F4, c, d, a, b, in[ 6]+0xa3014314, 15);
-  MD5STEP(F4, b, c, d, a, in[13]+0x4e0811a1, 21);
-  MD5STEP(F4, a, b, c, d, in[ 4]+0xf7537e82,  6);
-  MD5STEP(F4, d, a, b, c, in[11]+0xbd3af235, 10);
-  MD5STEP(F4, c, d, a, b, in[ 2]+0x2ad7d2bb, 15);
-  MD5STEP(F4, b, c, d, a, in[ 9]+0xeb86d391, 21);
-
-  buf[0] += a;
-  buf[1] += b;
-  buf[2] += c;
-  buf[3] += d;
-}
-
-/*
- * Start MD5 accumulation.  Set bit count to 0 and buffer to mysterious
- * initialization constants.
- */
-static void MD5Init(MD5Context *ctx){
-  ctx->isInit = 1;
-  ctx->buf[0] = 0x67452301;
-  ctx->buf[1] = 0xefcdab89;
-  ctx->buf[2] = 0x98badcfe;
-  ctx->buf[3] = 0x10325476;
-  ctx->bits[0] = 0;
-  ctx->bits[1] = 0;
-}
-
-/*
- * Update context to reflect the concatenation of another buffer full
- * of bytes.
- */
-static 
-void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){
-  uint32 t;
-
-  /* Update bitcount */
-
-  t = ctx->bits[0];
-  if ((ctx->bits[0] = t + ((uint32)len << 3)) < t)
-    ctx->bits[1]++; /* Carry from low to high */
-  ctx->bits[1] += len >> 29;
-
-  t = (t >> 3) & 0x3f;    /* Bytes already in shsInfo->data */
-
-  /* Handle any leading odd-sized chunks */
-
-  if ( t ) {
-    unsigned char *p = (unsigned char *)ctx->u.in + t;
-
-    t = 64-t;
-    if (len < t) {
-      memcpy(p, buf, len);
-      return;
-    }
-    memcpy(p, buf, t);
-    byteReverse(ctx->u.in, 16);
-    MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
-    buf += t;
-    len -= t;
-  }
-
-  /* Process data in 64-byte chunks */
-
-  while (len >= 64) {
-    memcpy(ctx->u.in, buf, 64);
-    byteReverse(ctx->u.in, 16);
-    MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
-    buf += 64;
-    len -= 64;
-  }
-
-  /* Handle any remaining bytes of data. */
-
-  memcpy(ctx->u.in, buf, len);
-}
-
-/*
- * Final wrapup - pad to 64-byte boundary with the bit pattern 
- * 1 0* (64-bit count of bits processed, MSB-first)
- */
-static void MD5Final(unsigned char digest[16], MD5Context *ctx){
-  unsigned count;
-  unsigned char *p;
-
-  /* Compute number of bytes mod 64 */
-  count = (ctx->bits[0] >> 3) & 0x3F;
-
-  /* Set the first char of padding to 0x80.  This is safe since there is
-     always at least one byte free */
-  p = ctx->u.in + count;
-  *p++ = 0x80;
-
-  /* Bytes of padding needed to make 64 bytes */
-  count = 64 - 1 - count;
-
-  /* Pad out to 56 mod 64 */
-  if (count < 8) {
-    /* Two lots of padding:  Pad the first block to 64 bytes */
-    memset(p, 0, count);
-    byteReverse(ctx->u.in, 16);
-    MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
-
-    /* Now fill the next block with 56 bytes */
-    memset(ctx->u.in, 0, 56);
-  } else {
-    /* Pad block to 56 bytes */
-    memset(p, 0, count-8);
-  }
-  byteReverse(ctx->u.in, 14);
-
-  /* Append length in bits and transform */
-  ctx->u.in32[14] = ctx->bits[0];
-  ctx->u.in32[15] = ctx->bits[1];
-
-  MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
-  byteReverse((unsigned char *)ctx->buf, 4);
-  memcpy(digest, ctx->buf, 16);
-  memset(ctx, 0, sizeof(*ctx));    /* In case it is sensitive */
-}
-
-/*
-** Convert a 128-bit MD5 digest into a 32-digit base-16 number.
-*/
-static void MD5DigestToBase16(unsigned char *digest, char *zBuf){
-  static char const zEncode[] = "0123456789abcdef";
-  int i, j;
-
-  for(j=i=0; i<16; i++){
-    int a = digest[i];
-    zBuf[j++] = zEncode[(a>>4)&0xf];
-    zBuf[j++] = zEncode[a & 0xf];
-  }
-  zBuf[j] = 0;
-}
-
-/*
-** During testing, the special md5sum() aggregate function is available.
-** inside SQLite.  The following routines implement that function.
-*/
-static void md5step(sqlite3_context *context, int argc, sqlite3_value **argv){
-  MD5Context *p;
-  int i;
-  if( argc<1 ) return;
-  p = sqlite3_aggregate_context(context, sizeof(*p));
-  if( p==0 ) return;
-  if( !p->isInit ){
-    MD5Init(p);
-  }
-  for(i=0; i<argc; i++){
-    const char *zData = (char*)sqlite3_value_text(argv[i]);
-    if( zData ){
-      MD5Update(p, (unsigned char*)zData, strlen(zData));
-    }
-  }
-}
-static void md5finalize(sqlite3_context *context){
-  MD5Context *p;
-  unsigned char digest[16];
-  char zBuf[33];
-  p = sqlite3_aggregate_context(context, sizeof(*p));
-  MD5Final(digest,p);
-  MD5DigestToBase16(digest, zBuf);
-  sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
-}
-
-/*
-** End of copied md5sum() code.
-**************************************************************************/
-
-typedef sqlite3_int64 i64;
-
-typedef struct Error Error;
-typedef struct Sqlite Sqlite;
-typedef struct Statement Statement;
-
-typedef struct Threadset Threadset;
-typedef struct Thread Thread;
-
-/* Total number of errors in this process so far. */
-static int nGlobalErr = 0;
-
-struct Error {
-  int rc;
-  int iLine;
-  char *zErr;
-};
-
-struct Sqlite {
-  sqlite3 *db;                    /* Database handle */
-  Statement *pCache;              /* Linked list of cached statements */
-  int nText;                      /* Size of array at aText[] */
-  char **aText;                   /* Stored text results */
-};
-
-struct Statement {
-  sqlite3_stmt *pStmt;            /* Pre-compiled statement handle */
-  Statement *pNext;               /* Next statement in linked-list */
-};
-
-struct Thread {
-  int iTid;                       /* Thread number within test */
-  void* pArg;                     /* Pointer argument passed by caller */
-
-  pthread_t tid;                  /* Thread id */
-  char *(*xProc)(int, void*);     /* Thread main proc */
-  Thread *pNext;                  /* Next in this list of threads */
-};
-
-struct Threadset {
-  int iMaxTid;                    /* Largest iTid value allocated so far */
-  Thread *pThread;                /* Linked list of threads */
-};
-
-static void free_err(Error *p){
-  sqlite3_free(p->zErr);
-  p->zErr = 0;
-  p->rc = 0;
-}
-
-static void print_err(Error *p){
-  if( p->rc!=SQLITE_OK ){
-    int isWarn = 0;
-    if( p->rc==SQLITE_SCHEMA ) isWarn = 1;
-    if( sqlite3_strglob("* - no such table: *",p->zErr)==0 ) isWarn = 1;
-    printf("%s: (%d) \"%s\" at line %d\n", isWarn ? "Warning" : "Error",
-            p->rc, p->zErr, p->iLine);
-    if( !isWarn ) nGlobalErr++;
-    fflush(stdout);
-  }
-}
-
-static void print_and_free_err(Error *p){
-  print_err(p);
-  free_err(p);
-}
-
-static void system_error(Error *pErr, int iSys){
-  pErr->rc = iSys;
-  pErr->zErr = (char *)sqlite3_malloc(512);
-  strerror_r(iSys, pErr->zErr, 512);
-  pErr->zErr[511] = '\0';
-}
-
-static void sqlite_error(
-  Error *pErr, 
-  Sqlite *pDb, 
-  const char *zFunc
-){
-  pErr->rc = sqlite3_errcode(pDb->db);
-  pErr->zErr = sqlite3_mprintf(
-      "sqlite3_%s() - %s (%d)", zFunc, sqlite3_errmsg(pDb->db),
-      sqlite3_extended_errcode(pDb->db)
-  );
-}
-
-static void test_error_x(
-  Error *pErr,
-  char *zErr
-){
-  if( pErr->rc==SQLITE_OK ){
-    pErr->rc = 1;
-    pErr->zErr = zErr;
-  }else{
-    sqlite3_free(zErr);
-  }
-}
-
-static void clear_error_x(
-  Error *pErr,
-  int rc
-){
-  if( pErr->rc==rc ){
-    pErr->rc = SQLITE_OK;
-    sqlite3_free(pErr->zErr);
-    pErr->zErr = 0;
-  }
-}
-
-static int busyhandler(void *pArg, int n){
-  usleep(10*1000);
-  return 1;
-}
-
-static void opendb_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb,                    /* OUT: Database handle */
-  const char *zFile,              /* Database file name */
-  int bDelete                     /* True to delete db file before opening */
-){
-  if( pErr->rc==SQLITE_OK ){
-    int rc;
-    int flags = SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI;
-    if( bDelete ) unlink(zFile);
-    rc = sqlite3_open_v2(zFile, &pDb->db, flags, 0);
-    if( rc ){
-      sqlite_error(pErr, pDb, "open");
-      sqlite3_close(pDb->db);
-      pDb->db = 0;
-    }else{
-      sqlite3_create_function(
-          pDb->db, "md5sum", -1, SQLITE_UTF8, 0, 0, md5step, md5finalize
-      );
-      sqlite3_busy_handler(pDb->db, busyhandler, 0);
-      sqlite3_exec(pDb->db, "PRAGMA synchronous=OFF", 0, 0, 0);
-    }
-  }
-}
-
-static void closedb_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb                     /* OUT: Database handle */
-){
-  int rc;
-  int i;
-  Statement *pIter;
-  Statement *pNext;
-  for(pIter=pDb->pCache; pIter; pIter=pNext){
-    pNext = pIter->pNext;
-    sqlite3_finalize(pIter->pStmt);
-    sqlite3_free(pIter);
-  }
-  for(i=0; i<pDb->nText; i++){
-    sqlite3_free(pDb->aText[i]);
-  }
-  sqlite3_free(pDb->aText);
-  rc = sqlite3_close(pDb->db);
-  if( rc && pErr->rc==SQLITE_OK ){
-    pErr->zErr = sqlite3_mprintf("%s", sqlite3_errmsg(pDb->db));
-  }
-  memset(pDb, 0, sizeof(Sqlite));
-}
-
-static void sql_script_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb,                    /* Database handle */
-  const char *zSql                /* SQL script to execute */
-){
-  if( pErr->rc==SQLITE_OK ){
-    pErr->rc = sqlite3_exec(pDb->db, zSql, 0, 0, &pErr->zErr);
-  }
-}
-
-static void sql_script_printf_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb,                    /* Database handle */
-  const char *zFormat,            /* SQL printf format string */
-  ...                             /* Printf args */
-){
-  va_list ap;                     /* ... printf arguments */
-  va_start(ap, zFormat);
-  if( pErr->rc==SQLITE_OK ){
-    char *zSql = sqlite3_vmprintf(zFormat, ap);
-    pErr->rc = sqlite3_exec(pDb->db, zSql, 0, 0, &pErr->zErr);
-    sqlite3_free(zSql);
-  }
-  va_end(ap);
-}
-
-static Statement *getSqlStatement(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb,                    /* Database handle */
-  const char *zSql                /* SQL statement */
-){
-  Statement *pRet;
-  int rc;
-
-  for(pRet=pDb->pCache; pRet; pRet=pRet->pNext){
-    if( 0==strcmp(sqlite3_sql(pRet->pStmt), zSql) ){
-      return pRet;
-    }
-  }
-
-  pRet = sqlite3_malloc(sizeof(Statement));
-  rc = sqlite3_prepare_v2(pDb->db, zSql, -1, &pRet->pStmt, 0);
-  if( rc!=SQLITE_OK ){
-    sqlite_error(pErr, pDb, "prepare_v2");
-    return 0;
-  }
-  assert( 0==strcmp(sqlite3_sql(pRet->pStmt), zSql) );
-
-  pRet->pNext = pDb->pCache;
-  pDb->pCache = pRet;
-  return pRet;
-}
-
-static sqlite3_stmt *getAndBindSqlStatement(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb,                    /* Database handle */
-  va_list ap                      /* SQL followed by parameters */
-){
-  Statement *pStatement;          /* The SQLite statement wrapper */
-  sqlite3_stmt *pStmt;            /* The SQLite statement to return */
-  int i;                          /* Used to iterate through parameters */
-
-  pStatement = getSqlStatement(pErr, pDb, va_arg(ap, const char *));
-  if( !pStatement ) return 0;
-  pStmt = pStatement->pStmt;
-  for(i=1; i<=sqlite3_bind_parameter_count(pStmt); i++){
-    const char *zName = sqlite3_bind_parameter_name(pStmt, i);
-    void * pArg = va_arg(ap, void*);
-
-    switch( zName[1] ){
-      case 'i':
-        sqlite3_bind_int64(pStmt, i, *(i64 *)pArg);
-        break;
-
-      default:
-        pErr->rc = 1;
-        pErr->zErr = sqlite3_mprintf("Cannot discern type: \"%s\"", zName);
-        pStmt = 0;
-        break;
-    }
-  }
-
-  return pStmt;
-}
-
-static i64 execsql_i64_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb,                    /* Database handle */
-  ...                             /* SQL and pointers to parameter values */
-){
-  i64 iRet = 0;
-  if( pErr->rc==SQLITE_OK ){
-    sqlite3_stmt *pStmt;          /* SQL statement to execute */
-    va_list ap;                   /* ... arguments */
-    va_start(ap, pDb);
-    pStmt = getAndBindSqlStatement(pErr, pDb, ap);
-    if( pStmt ){
-      int first = 1;
-      while( SQLITE_ROW==sqlite3_step(pStmt) ){
-        if( first && sqlite3_column_count(pStmt)>0 ){
-          iRet = sqlite3_column_int64(pStmt, 0);
-        }
-        first = 0;
-      }
-      if( SQLITE_OK!=sqlite3_reset(pStmt) ){
-        sqlite_error(pErr, pDb, "reset");
-      }
-    }
-    va_end(ap);
-  }
-  return iRet;
-}
-
-static char * execsql_text_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb,                    /* Database handle */
-  int iSlot,                      /* Db handle slot to store text in */
-  ...                             /* SQL and pointers to parameter values */
-){
-  char *zRet = 0;
-
-  if( iSlot>=pDb->nText ){
-    int nByte = sizeof(char *)*(iSlot+1);
-    pDb->aText = (char **)sqlite3_realloc(pDb->aText, nByte);
-    memset(&pDb->aText[pDb->nText], 0, sizeof(char*)*(iSlot+1-pDb->nText));
-    pDb->nText = iSlot+1;
-  }
-
-  if( pErr->rc==SQLITE_OK ){
-    sqlite3_stmt *pStmt;          /* SQL statement to execute */
-    va_list ap;                   /* ... arguments */
-    va_start(ap, iSlot);
-    pStmt = getAndBindSqlStatement(pErr, pDb, ap);
-    if( pStmt ){
-      int first = 1;
-      while( SQLITE_ROW==sqlite3_step(pStmt) ){
-        if( first && sqlite3_column_count(pStmt)>0 ){
-          zRet = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));
-          sqlite3_free(pDb->aText[iSlot]);
-          pDb->aText[iSlot] = zRet;
-        }
-        first = 0;
-      }
-      if( SQLITE_OK!=sqlite3_reset(pStmt) ){
-        sqlite_error(pErr, pDb, "reset");
-      }
-    }
-    va_end(ap);
-  }
-
-  return zRet;
-}
-
-static void integrity_check_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb                     /* Database handle */
-){
-  if( pErr->rc==SQLITE_OK ){
-    Statement *pStatement;        /* Statement to execute */
-    char *zErr = 0;               /* Integrity check error */
-
-    pStatement = getSqlStatement(pErr, pDb, "PRAGMA integrity_check");
-    if( pStatement ){
-      sqlite3_stmt *pStmt = pStatement->pStmt;
-      while( SQLITE_ROW==sqlite3_step(pStmt) ){
-        const char *z = (const char*)sqlite3_column_text(pStmt, 0);
-        if( strcmp(z, "ok") ){
-          if( zErr==0 ){
-            zErr = sqlite3_mprintf("%s", z);
-          }else{
-            zErr = sqlite3_mprintf("%z\n%s", zErr, z);
-          }
-        }
-      }
-      sqlite3_reset(pStmt);
-
-      if( zErr ){
-        pErr->zErr = zErr;
-        pErr->rc = 1;
-      }
-    }
-  }
-}
-
-static void *launch_thread_main(void *pArg){
-  Thread *p = (Thread *)pArg;
-  return (void *)p->xProc(p->iTid, p->pArg);
-}
-
-static void launch_thread_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Threadset *pThreads,            /* Thread set */
-  char *(*xProc)(int, void*),     /* Proc to run */
-  void *pArg                      /* Argument passed to thread proc */
-){
-  if( pErr->rc==SQLITE_OK ){
-    int iTid = ++pThreads->iMaxTid;
-    Thread *p;
-    int rc;
-
-    p = (Thread *)sqlite3_malloc(sizeof(Thread));
-    memset(p, 0, sizeof(Thread));
-    p->iTid = iTid;
-    p->pArg = pArg;
-    p->xProc = xProc;
-
-    rc = pthread_create(&p->tid, NULL, launch_thread_main, (void *)p);
-    if( rc!=0 ){
-      system_error(pErr, rc);
-      sqlite3_free(p);
-    }else{
-      p->pNext = pThreads->pThread;
-      pThreads->pThread = p;
-    }
-  }
-}
-
-static void join_all_threads_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Threadset *pThreads             /* Thread set */
-){
-  Thread *p;
-  Thread *pNext;
-  for(p=pThreads->pThread; p; p=pNext){
-    void *ret;
-    pNext = p->pNext;
-    int rc;
-    rc = pthread_join(p->tid, &ret);
-    if( rc!=0 ){
-      if( pErr->rc==SQLITE_OK ) system_error(pErr, rc);
-    }else{
-      printf("Thread %d says: %s\n", p->iTid, (ret==0 ? "..." : (char *)ret));
-      fflush(stdout);
-    }
-    sqlite3_free(p);
-  }
-  pThreads->pThread = 0;
-}
-
-static i64 filesize_x(
-  Error *pErr,
-  const char *zFile
-){
-  i64 iRet = 0;
-  if( pErr->rc==SQLITE_OK ){
-    struct stat sStat;
-    if( stat(zFile, &sStat) ){
-      iRet = -1;
-    }else{
-      iRet = sStat.st_size;
-    }
-  }
-  return iRet;
-}
-
-static void filecopy_x(
-  Error *pErr,
-  const char *zFrom,
-  const char *zTo
-){
-  if( pErr->rc==SQLITE_OK ){
-    i64 nByte = filesize_x(pErr, zFrom);
-    if( nByte<0 ){
-      test_error_x(pErr, sqlite3_mprintf("no such file: %s", zFrom));
-    }else{
-      i64 iOff;
-      char aBuf[1024];
-      int fd1;
-      int fd2;
-      unlink(zTo);
-
-      fd1 = open(zFrom, O_RDONLY);
-      if( fd1<0 ){
-        system_error(pErr, errno);
-        return;
-      }
-      fd2 = open(zTo, O_RDWR|O_CREAT|O_EXCL, 0644);
-      if( fd2<0 ){
-        system_error(pErr, errno);
-        close(fd1);
-        return;
-      }
-
-      iOff = 0;
-      while( iOff<nByte ){
-        int nCopy = sizeof(aBuf);
-        if( nCopy+iOff>nByte ){
-          nCopy = nByte - iOff;
-        }
-        if( nCopy!=read(fd1, aBuf, nCopy) ){
-          system_error(pErr, errno);
-          break;
-        }
-        if( nCopy!=write(fd2, aBuf, nCopy) ){
-          system_error(pErr, errno);
-          break;
-        }
-        iOff += nCopy;
-      }
-
-      close(fd1);
-      close(fd2);
-    }
-  }
-}
-
-/* 
-** Used by setstoptime() and timetostop().
-*/
-static double timelimit = 0.0;
-
-static double currentTime(void){
-  double t;
-  static sqlite3_vfs *pTimelimitVfs = 0;
-  if( pTimelimitVfs==0 ) pTimelimitVfs = sqlite3_vfs_find(0);
-  if( pTimelimitVfs->iVersion>=2 && pTimelimitVfs->xCurrentTimeInt64!=0 ){
-    sqlite3_int64 tm;
-    pTimelimitVfs->xCurrentTimeInt64(pTimelimitVfs, &tm);
-    t = tm/86400000.0;
-  }else{
-    pTimelimitVfs->xCurrentTime(pTimelimitVfs, &t);
-  }
-  return t;
-}
-
-static void setstoptime_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  int nMs                         /* Milliseconds until "stop time" */
-){
-  if( pErr->rc==SQLITE_OK ){
-    double t = currentTime();
-    timelimit = t + ((double)nMs)/(1000.0*60.0*60.0*24.0);
-  }
-}
-
-static int timetostop_x(
-  Error *pErr                     /* IN/OUT: Error code */
-){
-  int ret = 1;
-  if( pErr->rc==SQLITE_OK ){
-    double t = currentTime();
-    ret = (t >= timelimit);
-  }
-  return ret;
-}
-
 
 /*************************************************************************
 **************************************************************************

test/wal.test

@@ -1297,51 +1297,53 @@ do_test wal-19.4 {
 # At one point, SQLite was failing to grow the mapping of the wal-index
 # file in step 3 and the checkpoint was corrupting the database file.
 #
-do_test wal-20.1 {
-  catch {db close}
-  forcedelete test.db test.db-wal test.db-journal
-  sqlite3 db test.db
-  execsql {
-    PRAGMA journal_mode = WAL;
-    CREATE TABLE t1(x);
-    INSERT INTO t1 VALUES(randomblob(900));
-    SELECT count(*) FROM t1;
-  }
-} {wal 1}
-do_test wal-20.2 {
-  set ::buddy [launch_testfixture]
-  testfixture $::buddy {
+if {[permutation]!="unix-excl"} {
+  do_test wal-20.1 {
+    catch {db close}
+    forcedelete test.db test.db-wal test.db-journal
     sqlite3 db test.db
-    db transaction { db eval {
-      PRAGMA wal_autocheckpoint = 0;
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 2 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 4 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 8 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 16 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 32 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 64 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 128 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 256 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 512 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 1024 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 2048 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 4096 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 8192 */
-      INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 16384 */
-    } }
-  }
-} {0}
-do_test wal-20.3 {
-  close $::buddy
-  execsql { PRAGMA wal_checkpoint }
-  execsql { SELECT count(*) FROM t1 }
-} {16384}
-do_test wal-20.4 {
-  db close
-  sqlite3 db test.db
-  execsql { SELECT count(*) FROM t1 }
-} {16384}
-integrity_check wal-20.5
+    execsql {
+      PRAGMA journal_mode = WAL;
+      CREATE TABLE t1(x);
+      INSERT INTO t1 VALUES(randomblob(900));
+      SELECT count(*) FROM t1;
+    }
+  } {wal 1}
+  do_test wal-20.2 {
+    set ::buddy [launch_testfixture]
+    testfixture $::buddy {
+      sqlite3 db test.db
+      db transaction { db eval {
+        PRAGMA wal_autocheckpoint = 0;
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 2 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 4 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 8 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 16 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 32 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 64 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 128 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 256 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 512 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 1024 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 2048 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 4096 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 8192 */
+        INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 16384 */
+      } }
+    }
+  } {0}
+  do_test wal-20.3 {
+    close $::buddy
+    execsql { PRAGMA wal_checkpoint }
+    execsql { SELECT count(*) FROM t1 }
+  } {16384}
+  do_test wal-20.4 {
+    db close
+    sqlite3 db test.db
+    execsql { SELECT count(*) FROM t1 }
+  } {16384}
+  integrity_check wal-20.5
+}
 
 catch { db2 close }
 catch { db close }

test/wal2.test

@@ -35,43 +35,6 @@ proc cond_incr_sync_count {adj} {
   }
 }
 
-proc set_tvfs_hdr {file args} {
-
-  # Set $nHdr to the number of bytes in the wal-index header:
-  set nHdr 48
-  set nInt [expr {$nHdr/4}]
-
-  if {[llength $args]>2} {
-    error {wrong # args: should be "set_tvfs_hdr fileName ?val1? ?val2?"}
-  }
-
-  set blob [tvfs shm $file]
-  if {$::tcl_platform(byteOrder)=="bigEndian"} {set fmt I} {set fmt i}
-
-  if {[llength $args]} {
-    set ia [lindex $args 0]
-    set ib $ia
-    if {[llength $args]==2} {
-      set ib [lindex $args 1]
-    }
-    binary scan $blob a[expr $nHdr*2]a* dummy tail
-    set blob [binary format ${fmt}${nInt}${fmt}${nInt}a* $ia $ib $tail]
-    tvfs shm $file $blob
-  }
-
-  binary scan $blob ${fmt}${nInt} ints
-  return $ints
-}
-
-proc incr_tvfs_hdr {file idx incrval} {
-  set ints [set_tvfs_hdr $file]
-  set v [lindex $ints $idx]
-  incr v $incrval
-  lset ints $idx $v
-  set_tvfs_hdr $file $ints
-}
-
-
 #-------------------------------------------------------------------------
 # Test case wal2-1.*:
 #

test/wal_common.tcl

@@ -90,4 +90,44 @@ proc wal_fix_walindex_cksum {hdrvar} {
   lset hdr 11 $c2
 }
 
+# This command assumes that $file is the name of a database file opened
+# in wal mode using a [testvfs] VFS. It returns a list of the 12 32-bit
+# integers that make up the wal-index-header for the named file.
+#
+proc set_tvfs_hdr {file args} {
+
+  # Set $nHdr to the number of bytes in the wal-index header:
+  set nHdr 48
+  set nInt [expr {$nHdr/4}]
+
+  if {[llength $args]>2} {
+    error {wrong # args: should be "set_tvfs_hdr fileName ?val1? ?val2?"}
+  }
+
+  set blob [tvfs shm $file]
+  if {$::tcl_platform(byteOrder)=="bigEndian"} {set fmt I} {set fmt i}
+
+  if {[llength $args]} {
+    set ia [lindex $args 0]
+    set ib $ia
+    if {[llength $args]==2} {
+      set ib [lindex $args 1]
+    }
+    binary scan $blob a[expr $nHdr*2]a* dummy tail
+    set blob [binary format ${fmt}${nInt}${fmt}${nInt}a* $ia $ib $tail]
+    tvfs shm $file $blob
+  }
+
+  binary scan $blob ${fmt}${nInt} ints
+  return $ints
+}
+
+proc incr_tvfs_hdr {file idx incrval} {
+  set ints [set_tvfs_hdr $file]
+  set v [lindex $ints $idx]
+  incr v $incrval
+  lset ints $idx $v
+  set_tvfs_hdr $file $ints
+}
+
 

tool/addopcodes.tcl

@@ -32,6 +32,7 @@ set extras {
   UPLUS
   TRUTH
   REGISTER
+  CONCURRENT
   VECTOR
   SELECT_COLUMN
   IF_NULL_ROW

tool/mkkeywordhash.c

@@ -204,7 +204,6 @@ static Keyword aKeywordTable[] = {
   { "ELSE",             "TK_ELSE",         ALWAYS                 },
   { "ESCAPE",           "TK_ESCAPE",       ALWAYS                 },
   { "EXCEPT",           "TK_EXCEPT",       COMPOUND               },
-  { "EXCLUSIVE",        "TK_EXCLUSIVE",    ALWAYS                 },
   { "EXISTS",           "TK_EXISTS",       ALWAYS                 },
   { "EXPLAIN",          "TK_EXPLAIN",      EXPLAIN                },
   { "FAIL",             "TK_FAIL",         CONFLICT|TRIGGER       },

tool/mkpragmatab.tcl

@@ -112,6 +112,12 @@ set pragma_def {
   IF:   !defined(SQLITE_OMIT_FLAG_PRAGMAS)
   IF:   defined(SQLITE_DEBUG)
 
+  NAME: noop_update
+  TYPE: FLAG
+  ARG:  SQLITE_NoopUpdate
+  IF:   !defined(SQLITE_OMIT_FLAG_PRAGMAS)
+  IF:   defined(SQLITE_ENABLE_NOOP_UPDATE)
+
   NAME: ignore_check_constraints
   TYPE: FLAG
   ARG:  SQLITE_IgnoreChecks