Disable the OR optimization based on expressions in the ON clause of a

LEFT JOIN.  This is a temporary measure until we can devise a better fix.

FossilOrigin-Name: 4df5ea935efe2ed8056c1fc12f446c91b47fa72a3e79fbaf669b5e6b4e295304

ext/session/changebatch1.test

@@ -0,0 +1,222 @@
+# 2016 August 23
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source $testdir/tester.tcl
+ifcapable !session {finish_test; return}
+
+set testprefix changebatch1
+
+
+proc sql_to_changeset {method sql} {
+  sqlite3session S db main
+  S attach *
+  execsql $sql
+  set ret [S $method]
+  S delete
+  return $ret
+}
+
+proc do_changebatch_test {tn method args} {
+  set C [list]
+  foreach a $args {
+    lappend C [sql_to_changeset $method $a]
+  }
+
+  sqlite3changebatch cb db
+  set i 1
+  foreach ::cs [lrange $C 0 end-1] {
+    set rc [cb add $::cs]
+    if {$rc!="SQLITE_OK"} { error "expected SQLITE_OK, got $rc (i=$i)" }
+    incr i
+  }
+
+  set ::cs [lindex $C end]
+  do_test $tn { cb add [set ::cs] } SQLITE_CONSTRAINT
+  cb delete
+}
+
+proc do_changebatch_test1 {tn args} {
+  uplevel do_changebatch_test $tn changeset $args
+}
+proc do_changebatch_test2 {tn args} {
+  uplevel do_changebatch_test $tn fullchangeset $args
+}
+
+#-------------------------------------------------------------------------
+# The body of the following loop contains tests for database schemas
+# that do not feature multi-column UNIQUE constraints. In this case
+# it doesn't matter if the changesets are generated using
+# sqlite3session_changeset() or sqlite3session_fullchangeset().
+#
+foreach {tn testfunction} {
+  1 do_changebatch_test1
+  2 do_changebatch_test2
+} {
+  reset_db
+
+  #-------------------------------------------------------------------------
+  #
+  do_execsql_test $tn.1.0 {
+    CREATE TABLE t1(a PRIMARY KEY, b);
+  }
+  
+  $testfunction $tn.1.1 {
+    INSERT INTO t1 VALUES(1, 1);
+  } {
+    DELETE FROM t1 WHERE a=1;
+  }
+  
+  do_execsql_test $tn.1.2.0 {
+    INSERT INTO t1 VALUES(1, 1);
+    INSERT INTO t1 VALUES(2, 2);
+    INSERT INTO t1 VALUES(3, 3);
+  }
+  $testfunction $tn.1.2.1 {
+    DELETE FROM t1 WHERE a=2;
+  } {
+    INSERT INTO t1 VALUES(2, 2);
+  }
+  
+  #-------------------------------------------------------------------------
+  #
+  do_execsql_test $tn.2.0 {
+    CREATE TABLE x1(a, b PRIMARY KEY, c UNIQUE);
+    CREATE TABLE x2(a PRIMARY KEY, b UNIQUE, c UNIQUE);
+    CREATE INDEX x1a ON x1(a);
+  
+    INSERT INTO x1 VALUES(1, 1, 'a');
+    INSERT INTO x1 VALUES(1, 2, 'b');
+    INSERT INTO x1 VALUES(1, 3, 'c');
+  }
+  
+  $testfunction $tn.2.1 {
+    DELETE FROM x1 WHERE b=2;
+  } {
+    UPDATE x1 SET c='b' WHERE b=3;
+  }
+  
+  $testfunction $tn.2.2 {
+    DELETE FROM x1 WHERE b=1;
+  } {
+    INSERT INTO x1 VALUES(1, 5, 'a');
+  }
+
+  set L [list]
+  for {set i 1000} {$i < 10000} {incr i} {
+    lappend L "INSERT INTO x2 VALUES($i, $i, 'x' || $i)"
+  }
+  lappend L "DELETE FROM x2 WHERE b=1005"
+  $testfunction $tn.2.3 {*}$L
+
+  execsql { INSERT INTO x1 VALUES('f', 'f', 'f') }
+  $testfunction $tn.2.4 {
+    INSERT INTO x2 VALUES('f', 'f', 'f');
+  } {
+    INSERT INTO x1 VALUES('g', 'g', 'g');
+  } {
+    DELETE FROM x1 WHERE b='f';
+  } {
+    INSERT INTO x2 VALUES('g', 'g', 'g');
+  } {
+    INSERT INTO x1 VALUES('f', 'f', 'f');
+  }
+
+  execsql {
+    DELETE FROM x1;
+    INSERT INTO x1 VALUES(1.5, 1.5, 1.5);
+  }
+  $testfunction $tn.2.5 {
+    DELETE FROM x1 WHERE b BETWEEN 1 AND 2;
+  } {
+    INSERT INTO x1 VALUES(2.5, 2.5, 2.5);
+  } {
+    INSERT INTO x1 VALUES(1.5, 1.5, 1.5);
+  }
+
+  execsql {
+    DELETE FROM x2;
+    INSERT INTO x2 VALUES(X'abcd', X'1234', X'7890');
+    INSERT INTO x2 VALUES(X'0000', X'0000', X'0000');
+  }
+  breakpoint
+  $testfunction $tn.2.6 {
+    UPDATE x2 SET c = X'1234' WHERE a=X'abcd';
+    INSERT INTO x2 VALUES(X'1234', X'abcd', X'7890');
+  } {
+    DELETE FROM x2 WHERE b=X'0000';
+  } {
+    INSERT INTO x2 VALUES(1, X'0000', NULL);
+  }
+}
+
+#-------------------------------------------------------------------------
+# Test some multi-column UNIQUE constraints. First Using _changeset() to
+# demonstrate the problem, then using _fullchangeset() to show that it has
+# been fixed.
+#
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE y1(a PRIMARY KEY, b, c, UNIQUE(b, c));
+  INSERT INTO y1 VALUES(1, 1, 1);
+  INSERT INTO y1 VALUES(2, 2, 2);
+  INSERT INTO y1 VALUES(3, 3, 3);
+  INSERT INTO y1 VALUES(4, 3, 4);
+  BEGIN;
+}
+
+do_test 3.1.1 {
+  set c1 [sql_to_changeset changeset { DELETE FROM y1 WHERE a=4    }]
+  set c2 [sql_to_changeset changeset { UPDATE y1 SET c=4 WHERE a=3 }]
+  sqlite3changebatch cb db
+  cb add $c1
+  cb add $c2
+} {SQLITE_OK}
+do_test 3.1.2 {
+  cb delete
+  execsql ROLLBACK
+} {}
+
+do_test 3.1.1 {
+  set c1 [sql_to_changeset fullchangeset { DELETE FROM y1 WHERE a=4    }]
+  set c2 [sql_to_changeset fullchangeset { UPDATE y1 SET c=4 WHERE a=3 }]
+  sqlite3changebatch cb db
+  cb add $c1
+  cb add $c2
+} {SQLITE_CONSTRAINT}
+do_test 3.1.2 {
+  cb delete
+} {}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 4.0 {
+  CREATE TABLE t1(x, y, z, PRIMARY KEY(x, y), UNIQUE(z));
+}
+
+do_test 4.1 {
+  set c1 [sql_to_changeset fullchangeset { INSERT INTO t1 VALUES(1, 2, 3) }]
+  execsql {
+    DROP TABLE t1;
+    CREATE TABLE t1(w, x, y, z, PRIMARY KEY(x, y), UNIQUE(z));
+  }
+  sqlite3changebatch cb db
+  list [catch { cb add $c1 } msg] $msg
+} {1 SQLITE_RANGE}
+
+
+
+finish_test

ext/session/changebatchfault.test

@@ -0,0 +1,42 @@
+# 2011 Mar 21
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the session module.
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source [file join [file dirname [info script]] session_common.tcl]
+source $testdir/tester.tcl
+ifcapable !session {finish_test; return}
+set testprefix changebatchfault
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c PRIMARY KEY, UNIQUE(a, b));
+  INSERT INTO t1 VALUES('a', 'a', 'a');
+  INSERT INTO t1 VALUES('b', 'b', 'b');
+}
+
+set ::c1 [changeset_from_sql { delete from t1 where c='a' }]
+set ::c2 [changeset_from_sql { insert into t1 values('c', 'c', 'c') }]
+
+do_faultsim_test 1 -faults oom-* -body {
+  sqlite3changebatch cb db
+  cb add $::c1
+  cb add $::c2
+} -test {
+  faultsim_test_result {0 SQLITE_OK} {1 SQLITE_NOMEM}
+  catch { cb delete }
+}
+
+
+finish_test

ext/session/sqlite3changebatch.c

@@ -0,0 +1,485 @@
+
+#if !defined(SQLITE_TEST) || (defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK))
+
+#include "sqlite3session.h"
+#include "sqlite3changebatch.h"
+
+#include <assert.h>
+#include <string.h>
+
+typedef struct BatchTable BatchTable;
+typedef struct BatchIndex BatchIndex;
+typedef struct BatchIndexEntry BatchIndexEntry;
+typedef struct BatchHash BatchHash;
+
+struct sqlite3_changebatch {
+  sqlite3 *db;                    /* Database handle used to read schema */
+  BatchTable *pTab;               /* First in linked list of tables */
+  int iChangesetId;               /* Current changeset id */
+  int iNextIdxId;                 /* Next available index id */
+  int nEntry;                     /* Number of entries in hash table */
+  int nHash;                      /* Number of hash buckets */
+  BatchIndexEntry **apHash;       /* Array of hash buckets */
+};
+
+struct BatchTable {
+  BatchIndex *pIdx;               /* First in linked list of UNIQUE indexes */
+  BatchTable *pNext;              /* Next table */
+  char zTab[1];                   /* Table name */
+};
+
+struct BatchIndex {
+  BatchIndex *pNext;              /* Next index on same table */
+  int iId;                        /* Index id (assigned internally) */
+  int bPk;                        /* True for PK index */
+  int nCol;                       /* Size of aiCol[] array */
+  int *aiCol;                     /* Array of columns that make up index */
+};
+
+struct BatchIndexEntry {
+  BatchIndexEntry *pNext;         /* Next colliding hash table entry */
+  int iChangesetId;               /* Id of associated changeset */
+  int iIdxId;                     /* Id of index this key is from */
+  int szRecord;
+  char aRecord[1];
+};
+
+/*
+** Allocate and zero a block of nByte bytes. Must be freed using cbFree().
+*/
+static void *cbMalloc(int *pRc, int nByte){
+  void *pRet;
+
+  if( *pRc ){
+    pRet = 0;
+  }else{
+    pRet = sqlite3_malloc(nByte);
+    if( pRet ){
+      memset(pRet, 0, nByte);
+    }else{
+      *pRc = SQLITE_NOMEM;
+    }
+  }
+
+  return pRet;
+}
+
+/*
+** Free an allocation made by cbMalloc().
+*/
+static void cbFree(void *p){
+  sqlite3_free(p);
+}
+
+/*
+** Return the hash bucket that pEntry belongs in.
+*/
+static int cbHash(sqlite3_changebatch *p, BatchIndexEntry *pEntry){
+  unsigned int iHash = (unsigned int)pEntry->iIdxId;
+  unsigned char *pEnd = (unsigned char*)&pEntry->aRecord[pEntry->szRecord];
+  unsigned char *pIter;
+
+  for(pIter=(unsigned char*)pEntry->aRecord; pIter<pEnd; pIter++){
+    iHash += (iHash << 7) + *pIter;
+  }
+
+  return (int)(iHash % p->nHash);
+}
+
+/*
+** Resize the hash table.
+*/
+static int cbHashResize(sqlite3_changebatch *p){
+  int rc = SQLITE_OK;
+  BatchIndexEntry **apNew;
+  int nNew = (p->nHash ? p->nHash*2 : 512);
+  int i;
+
+  apNew = cbMalloc(&rc, sizeof(BatchIndexEntry*) * nNew);
+  if( rc==SQLITE_OK ){
+    int nHash = p->nHash;
+    p->nHash = nNew;
+    for(i=0; i<nHash; i++){
+      BatchIndexEntry *pEntry;
+      while( (pEntry=p->apHash[i])!=0 ){
+        int iHash = cbHash(p, pEntry);
+        p->apHash[i] = pEntry->pNext;
+        pEntry->pNext = apNew[iHash];
+        apNew[iHash] = pEntry;
+      }
+    }
+
+    cbFree(p->apHash);
+    p->apHash = apNew;
+  }
+
+  return rc;
+}
+
+
+/*
+** Allocate a new sqlite3_changebatch object.
+*/
+int sqlite3changebatch_new(sqlite3 *db, sqlite3_changebatch **pp){
+  sqlite3_changebatch *pRet;
+  int rc = SQLITE_OK;
+  *pp = pRet = (sqlite3_changebatch*)cbMalloc(&rc, sizeof(sqlite3_changebatch));
+  if( pRet ){
+    pRet->db = db;
+  }
+  return rc;
+}
+
+/*
+** Add a BatchIndex entry for index zIdx to table pTab.
+*/
+static int cbAddIndex(
+  sqlite3_changebatch *p, 
+  BatchTable *pTab, 
+  const char *zIdx, 
+  int bPk
+){
+  int nCol = 0;
+  sqlite3_stmt *pIndexInfo = 0;
+  BatchIndex *pNew = 0;
+  int rc;
+  char *zIndexInfo;
+
+  zIndexInfo = (char*)sqlite3_mprintf("PRAGMA main.index_info = %Q", zIdx);
+  if( zIndexInfo ){
+    rc = sqlite3_prepare_v2(p->db, zIndexInfo, -1, &pIndexInfo, 0);
+    sqlite3_free(zIndexInfo);
+  }else{
+    rc = SQLITE_NOMEM;
+  }
+
+  if( rc==SQLITE_OK ){
+    while( SQLITE_ROW==sqlite3_step(pIndexInfo) ){ nCol++; }
+    rc = sqlite3_reset(pIndexInfo);
+  }
+
+  pNew = (BatchIndex*)cbMalloc(&rc, sizeof(BatchIndex) + sizeof(int) * nCol);
+  if( rc==SQLITE_OK ){
+    pNew->nCol = nCol;
+    pNew->bPk = bPk;
+    pNew->aiCol = (int*)&pNew[1];
+    pNew->iId = p->iNextIdxId++;
+    while( SQLITE_ROW==sqlite3_step(pIndexInfo) ){ 
+      int i = sqlite3_column_int(pIndexInfo, 0);
+      int j = sqlite3_column_int(pIndexInfo, 1);
+      pNew->aiCol[i] = j;
+    }
+    rc = sqlite3_reset(pIndexInfo);
+  }
+
+  if( rc==SQLITE_OK ){
+    pNew->pNext = pTab->pIdx;
+    pTab->pIdx = pNew;
+  }else{
+    cbFree(pNew);
+  }
+  sqlite3_finalize(pIndexInfo);
+
+  return rc;
+}
+
+/*
+** Free the object passed as the first argument.
+*/
+static void cbFreeTable(BatchTable *pTab){
+  BatchIndex *pIdx;
+  BatchIndex *pIdxNext;
+  for(pIdx=pTab->pIdx; pIdx; pIdx=pIdxNext){
+    pIdxNext = pIdx->pNext;
+    cbFree(pIdx);
+  }
+  cbFree(pTab);
+}
+
+/*
+** Find or create the BatchTable object named zTab.
+*/
+static int cbFindTable(
+  sqlite3_changebatch *p, 
+  const char *zTab, 
+  BatchTable **ppTab
+){
+  BatchTable *pRet = 0;
+  int rc = SQLITE_OK;
+
+  for(pRet=p->pTab; pRet; pRet=pRet->pNext){
+    if( 0==sqlite3_stricmp(zTab, pRet->zTab) ) break;
+  }
+
+  if( pRet==0 ){
+    int nTab = strlen(zTab);
+    pRet = (BatchTable*)cbMalloc(&rc, nTab + sizeof(BatchTable));
+    if( pRet ){
+      sqlite3_stmt *pIndexList = 0;
+      char *zIndexList = 0;
+      int rc2;
+      memcpy(pRet->zTab, zTab, nTab);
+
+      zIndexList = sqlite3_mprintf("PRAGMA main.index_list = %Q", zTab);
+      if( zIndexList==0 ){
+        rc = SQLITE_NOMEM;
+      }else{
+        rc = sqlite3_prepare_v2(p->db, zIndexList, -1, &pIndexList, 0);
+        sqlite3_free(zIndexList);
+      }
+
+      while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pIndexList) ){
+        if( sqlite3_column_int(pIndexList, 2) ){
+          const char *zIdx = (const char*)sqlite3_column_text(pIndexList, 1);
+          const char *zTyp = (const char*)sqlite3_column_text(pIndexList, 3);
+          rc = cbAddIndex(p, pRet, zIdx, (zTyp[0]=='p'));
+        }
+      }
+      rc2 = sqlite3_finalize(pIndexList);
+      if( rc==SQLITE_OK ) rc = rc2;
+
+      if( rc==SQLITE_OK ){
+        pRet->pNext = p->pTab;
+        p->pTab = pRet;
+      }else{
+        cbFreeTable(pRet);
+        pRet = 0;
+      }
+    }
+  }
+
+  *ppTab = pRet;
+  return rc;
+}
+
+/*
+** Extract value iVal from the changeset iterator passed as the first
+** argument. Set *ppVal to point to the value before returning.
+**
+** This function attempts to extract the value using function xVal
+** (which is always either sqlite3changeset_new or sqlite3changeset_old).
+** If the call returns SQLITE_OK but does not supply an sqlite3_value*
+** pointer, an attempt to extract the value is made using the xFallback 
+** function.
+*/
+static int cbGetChangesetValue(
+  sqlite3_changeset_iter *pIter, 
+  int (*xVal)(sqlite3_changeset_iter*,int,sqlite3_value**),
+  int (*xFallback)(sqlite3_changeset_iter*,int,sqlite3_value**),
+  int iVal,
+  sqlite3_value **ppVal
+){
+  int rc = xVal(pIter, iVal, ppVal);
+  if( rc==SQLITE_OK && *ppVal==0 && xFallback ){
+    rc = xFallback(pIter, iVal, ppVal);
+  }
+  return rc;
+}
+
+static int cbAddToHash(
+  sqlite3_changebatch *p, 
+  sqlite3_changeset_iter *pIter, 
+  BatchIndex *pIdx, 
+  int (*xVal)(sqlite3_changeset_iter*,int,sqlite3_value**),
+  int (*xFallback)(sqlite3_changeset_iter*,int,sqlite3_value**),
+  int *pbConf
+){
+  BatchIndexEntry *pNew;
+  int sz = pIdx->nCol;
+  int i;
+  int iOut = 0;
+  int rc = SQLITE_OK;
+
+  for(i=0; rc==SQLITE_OK && i<pIdx->nCol; i++){
+    sqlite3_value *pVal;
+    rc = cbGetChangesetValue(pIter, xVal, xFallback, pIdx->aiCol[i], &pVal);
+    if( rc==SQLITE_OK ){
+      int eType = 0;
+      if( pVal ) eType = sqlite3_value_type(pVal);
+      switch( eType ){
+        case 0:
+        case SQLITE_NULL:
+          return SQLITE_OK;
+
+        case SQLITE_INTEGER:
+          sz += 8;
+          break;
+        case SQLITE_FLOAT:
+          sz += 8;
+          break;
+
+        default:
+          assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB );
+          sz += sqlite3_value_bytes(pVal);
+          break;
+      }
+    }
+  }
+
+  pNew = cbMalloc(&rc, sizeof(BatchIndexEntry) + sz);
+  if( pNew ){
+    pNew->iChangesetId = p->iChangesetId;
+    pNew->iIdxId = pIdx->iId;
+    pNew->szRecord = sz;
+
+    for(i=0; i<pIdx->nCol; i++){
+      int eType;
+      sqlite3_value *pVal;
+      rc = cbGetChangesetValue(pIter, xVal, xFallback, pIdx->aiCol[i], &pVal);
+      if( rc!=SQLITE_OK ) break;  /* coverage: condition is never true */
+      eType = sqlite3_value_type(pVal);
+      pNew->aRecord[iOut++] = eType;
+      switch( eType ){
+        case SQLITE_INTEGER: {
+          sqlite3_int64 i64 = sqlite3_value_int64(pVal);
+          memcpy(&pNew->aRecord[iOut], &i64, 8);
+          iOut += 8;
+          break;
+        }
+        case SQLITE_FLOAT: {
+          double d64 = sqlite3_value_double(pVal);
+          memcpy(&pNew->aRecord[iOut], &d64, sizeof(double));
+          iOut += sizeof(double);
+          break;
+        }
+
+        default: {
+          int nByte = sqlite3_value_bytes(pVal);
+          const char *z = (const char*)sqlite3_value_blob(pVal);
+          memcpy(&pNew->aRecord[iOut], z, nByte);
+          iOut += nByte;
+          break;
+        }
+      }
+    }
+  }
+
+  if( rc==SQLITE_OK && p->nEntry>=(p->nHash/2) ){
+    rc = cbHashResize(p);
+  }
+
+  if( rc==SQLITE_OK ){
+    BatchIndexEntry *pIter;
+    int iHash = cbHash(p, pNew);
+
+    assert( iHash>=0 && iHash<p->nHash );
+    for(pIter=p->apHash[iHash]; pIter; pIter=pIter->pNext){
+      if( pNew->szRecord==pIter->szRecord 
+       && 0==memcmp(pNew->aRecord, pIter->aRecord, pNew->szRecord)
+      ){
+        if( pNew->iChangesetId!=pIter->iChangesetId ){
+          *pbConf = 1;
+        }
+        cbFree(pNew);
+        pNew = 0;
+        break;
+      }
+    }
+
+    if( pNew ){
+      pNew->pNext = p->apHash[iHash];
+      p->apHash[iHash] = pNew;
+      p->nEntry++;
+    }
+  }else{
+    cbFree(pNew);
+  }
+
+  return rc;
+}
+
+
+/*
+** Add a changeset to the current batch.
+*/
+int sqlite3changebatch_add(sqlite3_changebatch *p, void *pBuf, int nBuf){
+  sqlite3_changeset_iter *pIter;  /* Iterator opened on pBuf/nBuf */
+  int rc;                         /* Return code */
+  int bConf = 0;                  /* Conflict was detected */
+
+  rc = sqlite3changeset_start(&pIter, nBuf, pBuf);
+  if( rc==SQLITE_OK ){
+    int rc2;
+    for(rc2 = sqlite3changeset_next(pIter);
+        rc2==SQLITE_ROW;
+        rc2 = sqlite3changeset_next(pIter)
+    ){
+      BatchTable *pTab;
+      BatchIndex *pIdx;
+      const char *zTab;           /* Table this change applies to */
+      int nCol;                   /* Number of columns in table */
+      int op;                     /* UPDATE, INSERT or DELETE */
+
+      sqlite3changeset_op(pIter, &zTab, &nCol, &op, 0);
+      assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE );
+
+      rc = cbFindTable(p, zTab, &pTab);
+      assert( pTab || rc!=SQLITE_OK );
+      if( pTab ){
+        for(pIdx=pTab->pIdx; pIdx && rc==SQLITE_OK; pIdx=pIdx->pNext){
+          if( op==SQLITE_UPDATE && pIdx->bPk ) continue;
+          if( op==SQLITE_UPDATE || op==SQLITE_DELETE ){
+            rc = cbAddToHash(p, pIter, pIdx, sqlite3changeset_old, 0, &bConf);
+          }
+          if( op==SQLITE_UPDATE || op==SQLITE_INSERT ){
+            rc = cbAddToHash(p, pIter, pIdx, 
+                sqlite3changeset_new, sqlite3changeset_old, &bConf
+            );
+          }
+        }
+      }
+      if( rc!=SQLITE_OK ) break;
+    }
+
+    rc2 = sqlite3changeset_finalize(pIter);
+    if( rc==SQLITE_OK ) rc = rc2;
+  }
+
+  if( rc==SQLITE_OK && bConf ){
+    rc = SQLITE_CONSTRAINT;
+  }
+  p->iChangesetId++;
+  return rc;
+}
+
+/*
+** Zero an existing changebatch object.
+*/
+void sqlite3changebatch_zero(sqlite3_changebatch *p){
+  int i;
+  for(i=0; i<p->nHash; i++){
+    BatchIndexEntry *pEntry;
+    BatchIndexEntry *pNext;
+    for(pEntry=p->apHash[i]; pEntry; pEntry=pNext){
+      pNext = pEntry->pNext;
+      cbFree(pEntry);
+    }
+  }
+  cbFree(p->apHash);
+  p->nHash = 0;
+  p->apHash = 0;
+}
+
+/*
+** Delete a changebatch object.
+*/
+void sqlite3changebatch_delete(sqlite3_changebatch *p){
+  BatchTable *pTab;
+  BatchTable *pTabNext;
+
+  sqlite3changebatch_zero(p);
+  for(pTab=p->pTab; pTab; pTab=pTabNext){
+    pTabNext = pTab->pNext;
+    cbFreeTable(pTab);
+  }
+  cbFree(p);
+}
+
+/*
+** Return the db handle.
+*/
+sqlite3 *sqlite3changebatch_db(sqlite3_changebatch *p){
+  return p->db;
+}
+
+#endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */

ext/session/sqlite3changebatch.h

@@ -0,0 +1,82 @@
+
+#if !defined(SQLITECHANGEBATCH_H_) 
+#define SQLITECHANGEBATCH_H_ 1
+
+typedef struct sqlite3_changebatch sqlite3_changebatch;
+
+/*
+** Create a new changebatch object for detecting conflicts between
+** changesets associated with a schema equivalent to that of the "main"
+** database of the open database handle db passed as the first
+** parameter. It is the responsibility of the caller to ensure that
+** the database handle is not closed until after the changebatch
+** object has been deleted.
+**
+** A changebatch object is used to detect batches of non-conflicting
+** changesets. Changesets that do not conflict may be applied to the 
+** target database in any order without affecting the final state of 
+** the database.
+**
+** The changebatch object only works reliably if PRIMARY KEY and UNIQUE
+** constraints on tables affected by the changesets use collation
+** sequences that are equivalent to built-in collation sequence 
+** BINARY for the == operation.
+**
+** If successful, SQLITE_OK is returned and (*pp) set to point to
+** the new changebatch object. If an error occurs, an SQLite error
+** code is returned and the final value of (*pp) is undefined.
+*/
+int sqlite3changebatch_new(sqlite3 *db, sqlite3_changebatch **pp);
+
+/*
+** Argument p points to a buffer containing a changeset n bytes in
+** size. Assuming no error occurs, this function returns SQLITE_OK
+** if the changeset does not conflict with any changeset passed 
+** to an sqlite3changebatch_add() call made on the same 
+** sqlite3_changebatch* handle since the most recent call to
+** sqlite3changebatch_zero(). If the changeset does conflict with 
+** an earlier such changeset, SQLITE_CONSTRAINT is returned. Or, 
+** if an error occurs, some other SQLite error code may be returned.
+**
+** One changeset is said to conflict with another if
+** either:
+**
+**   * the two changesets contain operations (INSERT, UPDATE or 
+**     DELETE) on the same row, identified by primary key, or
+**
+**   * the two changesets contain operations (INSERT, UPDATE or 
+**     DELETE) on rows with identical values in any combination 
+**     of fields constrained by a UNIQUE constraint.
+**
+** Even if this function returns SQLITE_CONFLICT, the current
+** changeset is added to the internal data structures - so future
+** calls to this function may conflict with it. If this function
+** returns any result code other than SQLITE_OK or SQLITE_CONFLICT,
+** the result of any future call to sqlite3changebatch_add() is
+** undefined.
+**
+** Only changesets may be passed to this function. Passing a 
+** patchset to this function results in an SQLITE_MISUSE error.
+*/
+int sqlite3changebatch_add(sqlite3_changebatch*, void *p, int n);
+
+/*
+** Zero a changebatch object. This causes the records of all earlier 
+** calls to sqlite3changebatch_add() to be discarded.
+*/
+void sqlite3changebatch_zero(sqlite3_changebatch*);
+
+/*
+** Return a copy of the first argument passed to the sqlite3changebatch_new()
+** call used to create the changebatch object passed as the only argument
+** to this function.
+*/
+sqlite3 *sqlite3changebatch_db(sqlite3_changebatch*);
+
+/*
+** Delete a changebatch object.
+*/
+void sqlite3changebatch_delete(sqlite3_changebatch*);
+
+#endif /* !defined(SQLITECHANGEBATCH_H_) */
+

ext/session/sqlite3session.c

@@ -25,6 +25,13 @@ typedef struct SessionInput SessionInput;
 # endif
 #endif
 
+/*
+** The three different types of changesets generated.
+*/
+#define SESSIONS_PATCHSET      0
+#define SESSIONS_CHANGESET     1
+#define SESSIONS_FULLCHANGESET 2
+
 typedef struct SessionHook SessionHook;
 struct SessionHook {
   void *pCtx;
@@ -2013,7 +2020,7 @@ static void sessionAppendCol(
 */ 
 static int sessionAppendUpdate(
   SessionBuffer *pBuf,            /* Buffer to append to */
-  int bPatchset,                  /* True for "patchset", 0 for "changeset" */
+  int ePatchset,                  /* True for "patchset", 0 for "changeset" */
   sqlite3_stmt *pStmt,            /* Statement handle pointing at new row */
   SessionChange *p,               /* Object containing old values */
   u8 *abPK                        /* Boolean array - true for PK columns */
@@ -2076,8 +2083,8 @@ static int sessionAppendUpdate(
 
     /* Add a field to the old.* record. This is omitted if this modules is
     ** currently generating a patchset. */
-    if( bPatchset==0 ){
-      if( bChanged || abPK[i] ){
+    if( ePatchset!=SESSIONS_PATCHSET ){
+      if( ePatchset==SESSIONS_FULLCHANGESET || bChanged || abPK[i] ){
         sessionAppendBlob(pBuf, pCsr, nAdvance, &rc);
       }else{
         sessionAppendByte(pBuf, 0, &rc);
@@ -2086,7 +2093,7 @@ static int sessionAppendUpdate(
 
     /* Add a field to the new.* record. Or the only record if currently
     ** generating a patchset.  */
-    if( bChanged || (bPatchset && abPK[i]) ){
+    if( bChanged || (ePatchset==SESSIONS_PATCHSET && abPK[i]) ){
       sessionAppendCol(&buf2, pStmt, i, &rc);
     }else{
       sessionAppendByte(&buf2, 0, &rc);
@@ -2112,7 +2119,7 @@ static int sessionAppendUpdate(
 */
 static int sessionAppendDelete(
   SessionBuffer *pBuf,            /* Buffer to append to */
-  int bPatchset,                  /* True for "patchset", 0 for "changeset" */
+  int eChangeset,                 /* One of SESSIONS_CHANGESET etc. */
   SessionChange *p,               /* Object containing old values */
   int nCol,                       /* Number of columns in table */
   u8 *abPK                        /* Boolean array - true for PK columns */
@@ -2122,7 +2129,7 @@ static int sessionAppendDelete(
   sessionAppendByte(pBuf, SQLITE_DELETE, &rc);
   sessionAppendByte(pBuf, p->bIndirect, &rc);
 
-  if( bPatchset==0 ){
+  if( eChangeset!=SESSIONS_PATCHSET ){
     sessionAppendBlob(pBuf, p->aRecord, p->nRecord, &rc);
   }else{
     int i;
@@ -2294,12 +2301,12 @@ static int sessionSelectBind(
 */
 static void sessionAppendTableHdr(
   SessionBuffer *pBuf,            /* Append header to this buffer */
-  int bPatchset,                  /* Use the patchset format if true */
+  int ePatchset,                  /* Use the patchset format if true */
   SessionTable *pTab,             /* Table object to append header for */
   int *pRc                        /* IN/OUT: Error code */
 ){
   /* Write a table header */
-  sessionAppendByte(pBuf, (bPatchset ? 'P' : 'T'), pRc);
+  sessionAppendByte(pBuf, (ePatchset==SESSIONS_PATCHSET) ? 'P' : 'T', pRc);
   sessionAppendVarint(pBuf, pTab->nCol, pRc);
   sessionAppendBlob(pBuf, pTab->abPK, pTab->nCol, pRc);
   sessionAppendBlob(pBuf, (u8 *)pTab->zName, (int)strlen(pTab->zName)+1, pRc);
@@ -2317,7 +2324,7 @@ static void sessionAppendTableHdr(
 */
 static int sessionGenerateChangeset(
   sqlite3_session *pSession,      /* Session object */
-  int bPatchset,                  /* True for patchset, false for changeset */
+  int ePatchset,                  /* One of SESSIONS_CHANGESET etc. */
   int (*xOutput)(void *pOut, const void *pData, int nData),
   void *pOut,                     /* First argument for xOutput */
   int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
@@ -2362,7 +2369,7 @@ static int sessionGenerateChangeset(
       }
 
       /* Write a table header */
-      sessionAppendTableHdr(&buf, bPatchset, pTab, &rc);
+      sessionAppendTableHdr(&buf, ePatchset, pTab, &rc);
 
       /* Build and compile a statement to execute: */
       if( rc==SQLITE_OK ){
@@ -2386,10 +2393,10 @@ static int sessionGenerateChangeset(
                 sessionAppendCol(&buf, pSel, iCol, &rc);
               }
             }else{
-              rc = sessionAppendUpdate(&buf, bPatchset, pSel, p, abPK);
+              rc = sessionAppendUpdate(&buf, ePatchset, pSel, p, abPK);
             }
           }else if( p->op!=SQLITE_INSERT ){
-            rc = sessionAppendDelete(&buf, bPatchset, p, nCol, abPK);
+            rc = sessionAppendDelete(&buf, ePatchset, p, nCol, abPK);
           }
           if( rc==SQLITE_OK ){
             rc = sqlite3_reset(pSel);
@@ -2446,7 +2453,8 @@ int sqlite3session_changeset(
   int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
   void **ppChangeset              /* OUT: Buffer containing changeset */
 ){
-  return sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset, ppChangeset);
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_CHANGESET, 0, 0, pnChangeset, ppChangeset);
 }
 
 /*
@@ -2457,7 +2465,8 @@ int sqlite3session_changeset_strm(
   int (*xOutput)(void *pOut, const void *pData, int nData),
   void *pOut
 ){
-  return sessionGenerateChangeset(pSession, 0, xOutput, pOut, 0, 0);
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_CHANGESET, xOutput, pOut, 0, 0);
 }
 
 /*
@@ -2468,7 +2477,8 @@ int sqlite3session_patchset_strm(
   int (*xOutput)(void *pOut, const void *pData, int nData),
   void *pOut
 ){
-  return sessionGenerateChangeset(pSession, 1, xOutput, pOut, 0, 0);
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_PATCHSET, xOutput, pOut, 0, 0);
 }
 
 /*
@@ -2483,9 +2493,20 @@ int sqlite3session_patchset(
   int *pnPatchset,                /* OUT: Size of buffer at *ppChangeset */
   void **ppPatchset               /* OUT: Buffer containing changeset */
 ){
-  return sessionGenerateChangeset(pSession, 1, 0, 0, pnPatchset, ppPatchset);
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_PATCHSET, 0, 0, pnPatchset, ppPatchset);
 }
 
+int sqlite3session_fullchangeset(
+  sqlite3_session *pSession,      /* Session object */
+  int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
+  void **ppChangeset              /* OUT: Buffer containing changeset */
+){
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_FULLCHANGESET, 0, 0, pnChangeset, ppChangeset);
+}
+
+
 /*
 ** Enable or disable the session object passed as the first argument.
 */
@@ -4670,10 +4691,11 @@ static int sessionChangegroupOutput(
   ** hash tables attached to the SessionTable objects in list p->pList. 
   */
   for(pTab=pGrp->pList; rc==SQLITE_OK && pTab; pTab=pTab->pNext){
+    int eChangeset = pGrp->bPatch ? SESSIONS_PATCHSET : SESSIONS_CHANGESET;
     int i;
     if( pTab->nEntry==0 ) continue;
 
-    sessionAppendTableHdr(&buf, pGrp->bPatch, pTab, &rc);
+    sessionAppendTableHdr(&buf, eChangeset, pTab, &rc);
     for(i=0; i<pTab->nChange; i++){
       SessionChange *p;
       for(p=pTab->apChange[i]; p; p=p->pNext){

ext/session/sqlite3session.h

@@ -323,6 +323,19 @@ int sqlite3session_changeset(
   void **ppChangeset              /* OUT: Buffer containing changeset */
 );
 
+/*
+** CAPI3REF: Generate A Full Changeset From A Session Object
+**
+** This function is similar to sqlite3session_changeset(), except that for
+** each row affected by an UPDATE statement, all old.* values are recorded
+** as part of the changeset, not just those modified.
+*/
+int sqlite3session_fullchangeset(
+  sqlite3_session *pSession,      /* Session object */
+  int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
+  void **ppChangeset              /* OUT: Buffer containing changeset */
+);
+
 /*
 ** CAPI3REF: Load The Difference Between Tables Into A Session
 ** METHOD: sqlite3_session

ext/session/test_session.c

@@ -214,6 +214,7 @@ static int testStreamOutput(
 **          $session indirect INTEGER
 **          $session patchset
 **          $session table_filter SCRIPT
+**          $session fullchangeset
 */
 static int SQLITE_TCLAPI test_session_cmd(
   void *clientData,
@@ -227,17 +228,17 @@ static int SQLITE_TCLAPI test_session_cmd(
     const char *zSub;
     int nArg;
     const char *zMsg;
-    int iSub;
   } aSub[] = {
-    { "attach",       1, "TABLE",      }, /* 0 */
-    { "changeset",    0, "",           }, /* 1 */
-    { "delete",       0, "",           }, /* 2 */
-    { "enable",       1, "BOOL",       }, /* 3 */
-    { "indirect",     1, "BOOL",       }, /* 4 */
-    { "isempty",      0, "",           }, /* 5 */
-    { "table_filter", 1, "SCRIPT",     }, /* 6 */
+    { "attach",       1, "TABLE"       }, /* 0 */
+    { "changeset",    0, ""            }, /* 1 */
+    { "delete",       0, ""            }, /* 2 */
+    { "enable",       1, "BOOL"        }, /* 3 */
+    { "indirect",     1, "BOOL"        }, /* 4 */
+    { "isempty",      0, ""            }, /* 5 */
+    { "table_filter", 1, "SCRIPT"      }, /* 6 */
     { "patchset",     0, "",           }, /* 7 */
-    { "diff",         2, "FROMDB TBL", }, /* 8 */
+    { "diff",         2, "FROMDB TBL"  }, /* 8 */
+    { "fullchangeset",0, ""            }, /* 9 */
     { 0 }
   };
   int iSub;
@@ -267,10 +268,11 @@ static int SQLITE_TCLAPI test_session_cmd(
       break;
     }
 
+    case 9:        /* fullchangeset */
     case 7:        /* patchset */
     case 1: {      /* changeset */
       TestSessionsBlob o = {0, 0};
-      if( test_tcl_integer(interp, SESSION_STREAM_TCL_VAR) ){
+      if( iSub!=9 && test_tcl_integer(interp, SESSION_STREAM_TCL_VAR) ){
         void *pCtx = (void*)&o;
         if( iSub==7 ){
           rc = sqlite3session_patchset_strm(pSession, testStreamOutput, pCtx);
@@ -280,6 +282,8 @@ static int SQLITE_TCLAPI test_session_cmd(
       }else{
         if( iSub==7 ){
           rc = sqlite3session_patchset(pSession, &o.n, &o.p);
+        }else if( iSub==9 ){
+          rc = sqlite3session_fullchangeset(pSession, &o.n, &o.p);
         }else{
           rc = sqlite3session_changeset(pSession, &o.n, &o.p);
         }
@@ -294,6 +298,7 @@ static int SQLITE_TCLAPI test_session_cmd(
       break;
     }
 
+
     case 2:        /* delete */
       Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
       break;
@@ -474,7 +479,7 @@ static int test_filter_handler(
 
   Tcl_DecrRefCount(pEval);
   return res;
-}  
+}
 
 static int test_conflict_handler(
   void *pCtx,                     /* Pointer to TestConflictHandler structure */
@@ -1019,6 +1024,127 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach(
   return TCL_OK;
 }
 
+#include "sqlite3changebatch.h"
+
+typedef struct TestChangebatch TestChangebatch;
+struct TestChangebatch {
+  sqlite3_changebatch *pChangebatch;
+};
+
+/*
+** Tclcmd:  $changebatch add BLOB
+**          $changebatch zero
+**          $changebatch delete
+*/
+static int SQLITE_TCLAPI test_changebatch_cmd(
+  void *clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  TestChangebatch *p = (TestChangebatch*)clientData;
+  sqlite3_changebatch *pChangebatch = p->pChangebatch;
+  struct SessionSubcmd {
+    const char *zSub;
+    int nArg;
+    const char *zMsg;
+    int iSub;
+  } aSub[] = {
+    { "add",          1, "CHANGESET",  }, /* 0 */
+    { "zero",         0, "",           }, /* 1 */
+    { "delete",       0, "",           }, /* 2 */
+    { 0 }
+  };
+  int iSub;
+  int rc;
+
+  if( objc<2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ...");
+    return TCL_ERROR;
+  }
+  rc = Tcl_GetIndexFromObjStruct(interp, 
+      objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub
+  );
+  if( rc!=TCL_OK ) return rc;
+  if( objc!=2+aSub[iSub].nArg ){
+    Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg);
+    return TCL_ERROR;
+  }
+
+  switch( iSub ){
+    case 0: {      /* add */
+      int nArg;
+      unsigned char *pArg = Tcl_GetByteArrayFromObj(objv[2], &nArg);
+      rc = sqlite3changebatch_add(pChangebatch, pArg, nArg);
+      if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){
+        return test_session_error(interp, rc, 0);
+      }else{
+        extern const char *sqlite3ErrName(int);
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+      }
+      break;
+    }
+
+    case 1: {      /* zero */
+      sqlite3changebatch_zero(pChangebatch);
+      break;
+    }
+
+    case 2:        /* delete */
+      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
+      break;
+  }
+
+  return TCL_OK;
+}
+
+static void SQLITE_TCLAPI test_changebatch_del(void *clientData){
+  TestChangebatch *p = (TestChangebatch*)clientData;
+  sqlite3changebatch_delete(p->pChangebatch);
+  ckfree((char*)p);
+}
+
+/*
+** Tclcmd:  sqlite3changebatch CMD DB-HANDLE
+*/
+static int SQLITE_TCLAPI test_sqlite3changebatch(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3 *db;
+  Tcl_CmdInfo info;
+  int rc;                         /* sqlite3session_create() return code */
+  TestChangebatch *p;             /* New wrapper object */
+
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "CMD DB-HANDLE");
+    return TCL_ERROR;
+  }
+
+  if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[2]), &info) ){
+    Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[2]), 0);
+    return TCL_ERROR;
+  }
+  db = *(sqlite3 **)info.objClientData;
+
+  p = (TestChangebatch*)ckalloc(sizeof(TestChangebatch));
+  memset(p, 0, sizeof(TestChangebatch));
+  rc = sqlite3changebatch_new(db, &p->pChangebatch);
+  if( rc!=SQLITE_OK ){
+    ckfree((char*)p);
+    return test_session_error(interp, rc, 0);
+  }
+
+  Tcl_CreateObjCommand(
+      interp, Tcl_GetString(objv[1]), test_changebatch_cmd, (ClientData)p,
+      test_changebatch_del
+  );
+  Tcl_SetObjResult(interp, objv[1]);
+  return TCL_OK;
+}
+
 int TestSession_Init(Tcl_Interp *interp){
   struct Cmd {
     const char *zCmd;
@@ -1040,6 +1166,10 @@ int TestSession_Init(Tcl_Interp *interp){
     Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, 0, 0);
   }
 
+
+  Tcl_CreateObjCommand(
+      interp, "sqlite3changebatch", test_sqlite3changebatch, 0, 0
+  );
   return TCL_OK;
 }
 

main.mk

@@ -426,6 +426,7 @@ TESTSRC2 = \
   $(TOP)/ext/async/sqlite3async.c \
   $(TOP)/ext/misc/stmt.c \
   $(TOP)/ext/session/sqlite3session.c \
+  $(TOP)/ext/session/sqlite3changebatch.c \
   $(TOP)/ext/session/test_session.c 
 
 # Header files used by all library source files.
@@ -953,6 +954,9 @@ THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
 threadtest3$(EXE): sqlite3.o $(THREADTEST3_SRC) $(TOP)/src/test_multiplex.c
 	$(TCCX) $(TOP)/test/threadtest3.c $(TOP)/src/test_multiplex.c sqlite3.o -o $@ $(THREADLIB)
 
+bc_test1$(EXE): sqlite3.o $(TOP)/test/bc_test1.c $(TOP)/test/tt3_core.c
+	$(TCCX) $(TOP)/test/bc_test1.c sqlite3.o -o $@ $(THREADLIB)
+
 threadtest: threadtest3$(EXE)
 	./threadtest3$(EXE)
 

manifest

@@ -1,5 +1,5 @@
-C Avoid\sunnecessary\swrite\sto\sthe\ssqlite_sequence\stable\swhen\san\sinsert\nis\sdone\sinto\san\sautoincrement\stable\swith\san\sapplication-specified\srowid\nthat\sis\sless\sthan\sthe\smaximum.
-D 2018-03-20T13:44:10.070
+C Disable\sthe\sOR\soptimization\sbased\son\sexpressions\sin\sthe\sON\sclause\sof\sa\nLEFT\sJOIN.\s\sThis\sis\sa\stemporary\smeasure\suntil\swe\scan\sdevise\sa\sbetter\sfix.
+D 2018-04-23T19:30:04.280
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7016fc56c6b9bfe5daac4f34be8be38d8c0b5fab79ccbfb764d3b23bf1c6fff3
@@ -377,6 +377,8 @@ F ext/rtree/rtreeconnect.test 225ad3fcb483d36cbee423a25052a6bbae762c9576ae926833
 F ext/rtree/sqlite3rtree.h 9c5777af3d2921c7b4ae4954e8e5697502289d28
 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de
 F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
+F ext/session/changebatch1.test 9ceaa8f7b2a505933e250fbe6cbc550e4ce1e59d
+F ext/session/changebatchfault.test be49c793219bf387ad692a60856b921f0854ad6d
 F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a
 F ext/session/session1.test 736d7ff178662f0b717c37f46531b84a5ce0210ccb0c4edf629c55dbcbbc3ea1
 F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0
@@ -402,16 +404,18 @@ F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7
 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sessionstat1.test 41cd97c2e48619a41cdf8ae749e1b25f34719de638689221aa43971be693bf4e
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
-F ext/session/sqlite3session.c 9edfaaa74977ddecd7bbd94e8f844d9b0f6eec22d1d547e806361670db814c1e
-F ext/session/sqlite3session.h 2e1584b030fbd841cefdce15ba984871978d305f586da2d1972f6e1958fa10b1
-F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
+F ext/session/sqlite3changebatch.c d5553b79e012ee2cb06c0a96bdf9dfe19e66354390ea0036cc46c4953142d517
+F ext/session/sqlite3changebatch.h e72016998c9a22d439ddfd547b69e1ebac810c24
+F ext/session/sqlite3session.c 1c1164008ad756d4dd7a2cb651647925cd0a88378853b4f225bbc5aa18c042e2
+F ext/session/sqlite3session.h 1eb6bc132e1e23eba7290b8cb85141fa4ef6cb5c8104a6851a4c4e88512996b6
+F ext/session/test_session.c badd5da3cb561564b093745f7d843430d1d76347
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 63668484c95454af7fc04a384da27ac556f27368d6d0c345e405e1677c66768f
+F main.mk 451db1a4c9a37a1a3c725a100d4169a0c797451c7aeb512fb07f1d32b78782c9
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -428,12 +432,12 @@ F src/analyze.c 6b42e36a5dcc2703a771f2411bd5e99524bd62c7ecde209bb88dfb04c72f046e
 F src/attach.c f6f212c43dddba79dfcb723fb9470785f3ff55bde8953cd9d2546f3022070a41
 F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
-F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
+F src/bitvec.c 8433d9e98dd6f2ea3286e0d2fe5d65de1bfc18a706486eb2026b01be066b5806
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 18a53540aa35dbdf77f715ea928422a4ed9011dc16ea7b50f803fd1617fcc4f5
-F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84
-F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96
-F src/build.c 8b53aacc26944bb7fd9ab5ddeedecb4cc7c4b84df3a420cf6d2b8f772ad421df
+F src/btree.c 828485509abc548bfd5f0a5d9b7b619a5c5d94350fb9194d130f54e7b09612b3
+F src/btree.h 0cd745755efd1f3df4c70544c54253920ea32fe6b179b97e9daeb786ba0de4ba
+F src/btreeInt.h 6c65e6c96f561596f6870c79a64d4706af81613881d7947e3f063e923f14115f
+F src/build.c adf56922a67ae5cbfe4dd993ae3643b6cc319e3cb6768a8a7052c69e60054527
 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c bd9da3f1ff21b432564a16ef0b154cff03585dc43742842e99c58907c6cb4bef
@@ -453,7 +457,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c b9ff71cc2913d1d57698a1e22bf853261a9a642baf62bdf40ddeb3809adb85b5
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c f6e4e416a736369f9e80eba609f0acda97148a8b0453784d670c78d3eed2f302
-F src/main.c c1e97e4c6fffff1fb1a6f1c48807386819af78657b4ae4acfd6d6d17353b8277
+F src/main.c e4e95e03b9bfc5ecf14f1b5f547f822cc85e39b95890683e5a148283035a569a
 F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -473,17 +477,17 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 928586cbde31890538854f1657463392d63b2d7794f577774573ae53ec283d7b
+F src/os_unix.c da5704d0c54eec4b8e02095782c0360af4edfbf7832191f5ff697fddfbb4f9ef
 F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c 3e3b659b6b3f40b5a5efef6b8890ced490c397a81d96feb5266fde1f7362e608
-F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
-F src/parse.y 140bbc53b5f67f731239f7fc8704a4f1e60cbbc10fb84bf9577322f974725f19
+F src/pager.c e9b4176a83e86d4113757d2ad4ca82fd3961402e226e08274e3569ce39ae25a2
+F src/pager.h 72e1a29470f2c2ad482ce944f61c7818098a842740d94c2bd167f348da4fdbac
+F src/parse.y 5e4afc0ca16cf5a0b464f49554f49bba03c455b9b973ec57665286f3f5abfbdb
 F src/pcache.c 135ef0bc6fb2e3b7178d49ab5c9176254c8a691832c1bceb1156b2fbdd0869bd
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c bea56df3ae0637768c0da4fbbb8f2492f780980d95000034a105ff291bf7ca69
-F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
+F src/pragma.h ce41efb7d4cdafca499839f29014d9b1d9534c8f503eeceb88310920c62d6097
 F src/prepare.c b086fea6a1952db88beca31fdd621201ee5e4ce3f02905248cc3035a8174aa89
 F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
@@ -491,15 +495,15 @@ F src/resolve.c 66c73fcb7719b8ff0e841b58338f13604ff3e2b50a723f9b8f383595735262f6
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 97a2131f02c605c52a6d5deac5b2ae6604db8c346829eb6b185bc38512a9b49a
 F src/shell.c.in 911b9e3bce40413c78fdba28efa28363e98183819bd4b300780bf57bacfc4b84
-F src/sqlite.h.in 19762b57baa1ade67531f254de94374428fb9c82452ef305017847945f9c2911
+F src/sqlite.h.in a95e2ac3a11c2d44faf18700c5ecc8beffc1409a0d21d8b855f1821469dd32c1
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
-F src/sqliteInt.h 5c07bbc55d9eb3389a10536bd008e2d6f20f586871972610431d943595b40ca4
+F src/sqliteInt.h 61d9f90fc7f9f8748a9cc845de4360a466cdc8e913126dbf6e08665a2519a6ef
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c 916a92de77ec5cbe27818ca194d8cf0c58aa7ad5b87527098f6aa5a6068800ce
-F src/test1.c 1ab7cbbb6693e08364c1a9241e2aee17f8c4925e4cc52396be77ae6845a05828
+F src/test1.c c2b5de91195cf2911760cfe543eb1bf315a01e1db32a95d2ba5d8fe6be06f41c
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b
 F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6
@@ -514,7 +518,7 @@ F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0
 F src/test_bestindex.c 78809f11026f18a93fcfd798d9479cba37e1201c830260bf1edc674b2fa9b857
 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce
 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274
-F src/test_config.c 097c6189803886a1fb26ec37d8bc62b90512cb53ab79a1fb6d35196c1ec42ded
+F src/test_config.c 7f049288c6174ba7038f37b68c809897426790170b25c178bf18d7358e6eed69
 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f
 F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e
 F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2
@@ -554,25 +558,25 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 5b0c661a85f783d35b9883830736eeb63be4aefc4f6b7d9cd081d48782c041e2
 F src/treeview.c 14d5d1254702ec96876aa52642cb31548612384134970409fae333b25b39d6bb
 F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1
-F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
+F src/update.c 503fcd664830492ef0abc0d8c683411c3b124079ee9e51232ffdbe15e54d9c32
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157
-F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5
-F src/vdbe.c 066a4e1de2ed83e253adfd2e97a684cf562eaa41d31ee7f3d3e4c8aea4485a55
+F src/vacuum.c f0b8302219b00461ac0edbb790b5ef52d3d454a7ef9b78030a44b32bbdc39e8a
+F src/vdbe.c 6f4b0624d493d3abe189658593716854e531de1b043a20e23f4b960c32792e1e
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h 95f7adfdc5c8f1353321f55a6c5ec00a90877e3b85af5159e393afb41ff54110
 F src/vdbeapi.c 29d2baf9c1233131ec467d7bed1b7c8a03c27579048d768c4b04acf427838858
-F src/vdbeaux.c 2756ac68ac259c416554100598fc291870063288cd7e1af22847f57b3e130e56
+F src/vdbeaux.c db9d38ca41466d5cc845bedbf88f470eefb5c4edaff2070249fa08a40273d1a1
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
 F src/vdbemem.c 414e28d3a7e2a8bee2bb247de115dcbc68e3cbac284d5862d077002f7a93bce1
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c aa9cffc7a2bad6b826a86c8562dd4978398720ed41cb8ee7aa9d054eb8b456a0
-F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
+F src/wal.c 05d9364fefacdff639b7fb63b1113317e045e2ae5c7908fa50157f9f24883257
+F src/wal.h 1713fefe4587678c295bbeb91c4e7442998ad74f19249869ce49dda9e8ce5d53
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c 7cae47e813393d70c6d327fdf000fcb30f76b1b0b5a5b52ff6402e0c658de32c
+F src/where.c 2e8d9899e73a453da055fab91538a88ce8ec79166baf41db85b8c9df5a9b3809
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c e1aaadd8fec650037cfbf27d1b3470338fb3b58fec34d11082df16fe9a08fbd7
 F src/whereexpr.c 53532be687e12f3cd314f1e204cd4fbdac7ad250e918a182b048121e16e828ae
@@ -639,6 +643,7 @@ F test/backup_malloc.test 0c9abdf74c51e7bedb66d504cd684f28d4bd4027
 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f
 F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f
 F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c
+F test/bc_test1.c e0a092579552e066ed4ce7bcdaecfa69c4aacc8d
 F test/bestindex1.test 0cf1bd2d7b97d3a3a8c10736125274f64765c4ee
 F test/bestindex2.test 4a06b8922ab2fd09434870da8d1cdf525aaf7060
 F test/bestindex3.test 578b6a52dab819e63f28e3640e04b32c85aed320
@@ -694,6 +699,14 @@ F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6
 F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95
 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1
 F test/colname.test fb28b3687e03625425bc216edf8b186ce974aa71008e2aa1f426a7dcb75a601d
+F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43
+F test/concurrent.test 86661967a680670127a62a819e60dc93c2d3d49043ac95b26dfa70d3e60dbde5
+F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903873111f8
+F test/concurrent3.test 530671ac706f6a1d0f4992dbdd33a86408330d03cd90fb9e82ecb1b27f5fd081
+F test/concurrent4.test e0b12cd467137e50259df3b4f837507e82aaa07c35941c88664dc8ed1d089c44
+F test/concurrent5.test d5d7d9d404a9b4502464fc097c1fc5c3012bb4f1b063fae7ad707ca983fc86c5
+F test/concurrent6.test a7860e9ca13bb5fb76bcf41c5524fbfa9c37e6e258ecf84ffb5748a272488c67
+F test/concurrent7.test b96fa5c4cfdf8d5c0bc66b6934214500bad0260884a736f054ccc76e81aae85d
 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db
 F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c
 F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9
@@ -1132,11 +1145,10 @@ F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f
 F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8
 F test/pageropt.test 84e4cc5cbca285357f7906e99b21be4f2bf5abc0
 F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305
-F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test 8ada8c1dee071e0fc275bc8bc2db7de537d625cad949d2200664b99a0a89eac5
+F test/permutations.test 6307a0f7d2821f3b4efd9888efee9e54b47b742dcfd5e20740de6e3df1e2afff
 F test/pragma.test 7c8cfc328a1717a95663cf8edb06c52ddfeaf97bb0aee69ae7457132e8d39e7d
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
@@ -1317,7 +1329,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46
 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd
 F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b
 F test/threadtest2.c a70a8e94bef23339d34226eb9521015ef99f4df8
-F test/threadtest3.c 38a612ea62854349ed66372f330a40d73c5cf956
+F test/threadtest3.c 4b413718ab3ef5e4f9f1ab0502b89707c5a9ccdc
 F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925
 F test/time-wordcount.sh 8e0b0f8109367827ad5d58f5cc849705731e4b90
 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c
@@ -1486,6 +1498,7 @@ F test/triggerE.test 15fa63f1097db1f83dd62d121616006978063d1f
 F test/triggerF.test 6a8c22bd058cf467f0c7d112afe87f7a8c579c0c4681b914b8f19020f48528a4
 F test/triggerG.test d5caeef6144ede2426dd13211fd72248241ff2ebc68e12a4c0bf30f5faa21499
 F test/tt3_checkpoint.c 9e75cf7c1c364f52e1c47fd0f14c4340a9db0fe1
+F test/tt3_core.c 8cd89ead95410f70e7fb02c79f1e040f9c5ad5cf
 F test/tt3_index.c 39eec10a35f57672225be4d182862152896dee4a
 F test/tt3_lookaside1.c 0377e202c3c2a50d688cb65ba203afeda6fafeb9
 F test/tt3_stress.c c57d804716165811d979d4a719e05baccd79277f
@@ -1536,7 +1549,7 @@ F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c37
 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad
 F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477
-F test/wal2.test 155b9efa999bdb38ce1cd729b9a4fcdbffd6b88be27f039bad1d2929d287d918
+F test/wal2.test a225bafac35a47765b890bacdeb57e5e81039f21cc18a1e8ce88eb76e56b843c
 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
 F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9
@@ -1545,7 +1558,7 @@ F test/wal64k.test 163655ecd2cb8afef4737cac2a40fdd2eeaf20b8
 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd
 F test/wal8.test d9df3fba4caad5854ed69ed673c68482514203c8
 F test/wal9.test 378e76a9ad09cd9bee06c172ad3547b0129a6750
-F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe
+F test/wal_common.tcl 37902864b63794e9bf59393b8887e21a9cbd4db8
 F test/walbak.test 018d4e5a3d45c6298d11b99f09a8ef6876527946
 F test/walbig.test f437473a16cfb314867c6b5d1dbcd519e73e3434
 F test/walblock.test be48f3a75eff0b4456209f26b3ce186c2015497d
@@ -1617,7 +1630,7 @@ F test/zipfilefault.test 44d4d7a7f7cca7521d569d7f71026b241d65a6b1757aa409c1a1688
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
-F tool/addopcodes.tcl 0288d5b26b9b35f4cb5affb76eec63f1dfce117bbc2020066708069ef60b86ff
+F tool/addopcodes.tcl 251331be53533f11d9ba36c730b9baaa911df89a08da3043344d0d634dc4a48e
 F tool/build-all-msvc.bat c12328d06c45fec8baada5949e3d5af54bf8c887 x
 F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367
 F tool/cg_anno.tcl f95b0006c52cf7f0496b506343415b6ee3cdcdd3 x
@@ -1640,12 +1653,12 @@ F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439
 F tool/mkautoconfamal.sh 422fc365358a2e92876ffc62971a0ff28ed472fc8bcf9de0df921c736fdeca5e
 F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x
 F tool/mkctimec.tcl dd183b73ae1c28249669741c250525f0407e579a70482371668fd5f130d9feb3
-F tool/mkkeywordhash.c 2e852ac0dfdc5af18886dc1ce7e9676d11714ae3df0a282dc7d90b3a0fe2033c
+F tool/mkkeywordhash.c 09ee7db43e6143fc6feb7490b9b56027efab5ae7257ee1f2014b07cd9eed1ec7
 F tool/mkmsvcmin.tcl 8baf26690b80d861d0ac341b29880eec6ade39e4f11fe690271ded9cb90563a3
 F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c
 F tool/mkopcodeh.tcl 4ee2a30ccbd900dc4d5cdb61bdab87cd2166cd2affcc78c9cc0b8d22a65b2eee
 F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa
-F tool/mkpragmatab.tcl 2144bc8550a6471a029db262a132d2df4b9e0db61b90398bf64f5b7b3f8d92cd
+F tool/mkpragmatab.tcl 0d67312ad1a6db29e85215c87e3f5c0cd3a9bf9e32784bf3a02e5ebaf1faadb0
 F tool/mkshellc.tcl 1f45770aea226ac093a9c72f718efbb88a2a2833409ec2e1c4cecae4202626f5
 F tool/mksourceid.c d458f9004c837bee87a6382228ac20d3eae3c49ea3b0a5aace936f8b60748d3b
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
@@ -1712,8 +1725,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ae34edb7f4fa7b37a66cdf3a1d851980cb013e9abe72bdeca23e5323051e0646 3e3849a9d1a06673e6c713a42194f5da339fbf6533fa418c38f63d09bc045867
-R b14ad23178f33348049b37fe5dfca3b5
-T +closed 3e3849a9d1a06673e6c713a42194f5da339fbf6533fa418c38f63d09bc045867
+P 257900494f9aac7e4181b65d0eddc9cf2e9a52dc794ad68284f085d1c402addb
+R 74cc9c776194c09a1da05e4dd3f5040c
+T *branch * begin-concurrent-pnu-tempfix
+T *sym-begin-concurrent-pnu-tempfix *
+T -sym-begin-concurrent-pnu *
 U drh
-Z 2e0c8bc153c99ae88009245613a69afa
+Z e083d3ae2d9b37daaf471638ab99a71c

manifest.uuid

@@ -1 +1 @@
-ec7addc87f97bcff3c3694b14a680453b52de3f8c106436f0708a1cc04b90faa
+4df5ea935efe2ed8056c1fc12f446c91b47fa72a3e79fbaf669b5e6b4e295304

src/bitvec.c

@@ -171,6 +171,12 @@ int sqlite3BitvecSet(Bitvec *p, u32 i){
   if( p==0 ) return SQLITE_OK;
   assert( i>0 );
   assert( i<=p->iSize );
+  if( i>p->iSize || i==0 ){
+    sqlite3_log(SQLITE_ERROR, 
+        "Bitvec: setting bit %d of bitvec size %d\n", (int)i, (int)p->iSize
+    );
+    abort();
+  }
   i--;
   while((p->iSize > BITVEC_NBIT) && p->iDivisor) {
     u32 bin = i/p->iDivisor;

src/btree.c

@@ -467,7 +467,240 @@ static void downgradeAllSharedCacheTableLocks(Btree *p){
 
 #endif /* SQLITE_OMIT_SHARED_CACHE */
 
-static void releasePage(MemPage *pPage);         /* Forward reference */
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** The following structure - BtreePtrmap - stores the in-memory pointer map
+** used for newly allocated pages in CONCURRENT transactions. Such pages are
+** always allocated in a contiguous block (from the end of the file) starting
+** with page BtreePtrmap.iFirst.
+*/
+typedef struct RollbackEntry RollbackEntry;
+typedef struct PtrmapEntry PtrmapEntry;
+struct PtrmapEntry {
+  Pgno parent;
+  u8 eType;
+};
+struct RollbackEntry {
+  Pgno pgno;
+  Pgno parent;
+  u8 eType;
+};
+struct BtreePtrmap {
+  Pgno iFirst;                    /* First new page number aPtr[0] */
+
+  int nPtrAlloc;                  /* Allocated size of aPtr[] array */
+  PtrmapEntry *aPtr;              /* Array of parent page numbers */
+
+  int nSvpt;                      /* Used size of aSvpt[] array */
+  int nSvptAlloc;                 /* Allocated size of aSvpt[] */
+  int *aSvpt;                     /* First aRollback[] entry for savepoint i */
+
+  int nRollback;                  /* Used size of aRollback[] array */
+  int nRollbackAlloc;             /* Allocated size of aRollback[] array */
+  RollbackEntry *aRollback;       /* Array of rollback entries */
+};
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** If page number pgno is greater than or equal to BtreePtrmap.iFirst, 
+** store an entry for it in the pointer-map structure.
+*/
+static int btreePtrmapStore(
+  BtShared *pBt,
+  Pgno pgno,
+  u8 eType, 
+  Pgno parent
+){
+  BtreePtrmap *pMap = pBt->pMap;
+  if( pgno>=pMap->iFirst ){
+    int iEntry = pgno - pMap->iFirst;
+
+    /* Grow the aPtr[] array as required */
+    while( iEntry>=pMap->nPtrAlloc ){
+      int nNew = pMap->nPtrAlloc ? pMap->nPtrAlloc*2 : 16;
+      PtrmapEntry *aNew = (PtrmapEntry*)sqlite3_realloc(
+          pMap->aPtr, nNew*sizeof(PtrmapEntry)
+      );
+      if( aNew==0 ){
+        return SQLITE_NOMEM;
+      }else{
+        int nByte = (nNew-pMap->nPtrAlloc)*sizeof(PtrmapEntry);
+        memset(&aNew[pMap->nPtrAlloc], 0, nByte);
+        pMap->aPtr = aNew;
+        pMap->nPtrAlloc = nNew;
+      }
+    }
+
+    /* Add an entry to the rollback log if required */
+    if( pMap->nSvpt>0 && pMap->aPtr[iEntry].parent ){
+      if( pMap->nRollback>=pMap->nRollbackAlloc ){
+        int nNew = pMap->nRollback ? pMap->nRollback*2 : 16;
+        RollbackEntry *aNew = (RollbackEntry*)sqlite3_realloc(
+            pMap->aRollback, nNew*sizeof(RollbackEntry)
+        );
+        if( aNew==0 ){
+          return SQLITE_NOMEM;
+        }else{
+          pMap->aRollback = aNew;
+          pMap->nRollbackAlloc = nNew;
+        }
+      }
+
+      pMap->aRollback[pMap->nRollback].pgno = pgno;
+      pMap->aRollback[pMap->nRollback].parent = pMap->aPtr[iEntry].parent;
+      pMap->aRollback[pMap->nRollback].eType = pMap->aPtr[iEntry].eType;
+      pMap->nRollback++;
+    }
+
+    /* Update the aPtr[] array */
+    pMap->aPtr[iEntry].parent = parent;
+    pMap->aPtr[iEntry].eType = eType;
+  }
+
+  return SQLITE_OK;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Open savepoint iSavepoint, if it is not already open.
+*/
+static int btreePtrmapBegin(BtShared *pBt, int nSvpt){
+  BtreePtrmap *pMap = pBt->pMap;
+  if( pMap && nSvpt>pMap->nSvpt ){
+    int i;
+    if( nSvpt>=pMap->nSvptAlloc ){
+      int nNew = pMap->nSvptAlloc ? pMap->nSvptAlloc*2 : 16;
+      int *aNew = sqlite3_realloc(pMap->aSvpt, sizeof(int) * nNew);
+      if( aNew==0 ){
+        return SQLITE_NOMEM;
+      }else{
+        pMap->aSvpt = aNew;
+        pMap->nSvptAlloc = nNew;
+      }
+    }
+
+    for(i=pMap->nSvpt; i<nSvpt; i++){
+      pMap->aSvpt[i] = pMap->nRollback;
+    }
+    pMap->nSvpt = nSvpt;
+  }
+
+  return SQLITE_OK;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Rollback (if op==SAVEPOINT_ROLLBACK) or release (if op==SAVEPOINT_RELEASE)
+** savepoint iSvpt.
+*/
+static void btreePtrmapEnd(BtShared *pBt, int op, int iSvpt){
+  BtreePtrmap *pMap = pBt->pMap;
+  if( pMap ){
+    assert( op==SAVEPOINT_ROLLBACK || op==SAVEPOINT_RELEASE );
+    assert( iSvpt>=0 || (iSvpt==-1 && op==SAVEPOINT_ROLLBACK) );
+    if( iSvpt<0 ){
+      pMap->nSvpt = 0;
+      pMap->nRollback = 0;
+      memset(pMap->aPtr, 0, sizeof(Pgno) * pMap->nPtrAlloc);
+    }else if( iSvpt<pMap->nSvpt ){
+      if( op==SAVEPOINT_ROLLBACK ){
+        int ii;
+        for(ii=pMap->nRollback-1; ii>=pMap->aSvpt[iSvpt]; ii--){
+          RollbackEntry *p = &pMap->aRollback[ii];
+          PtrmapEntry *pEntry = &pMap->aPtr[p->pgno - pMap->iFirst];
+          pEntry->parent = p->parent;
+          pEntry->eType = p->eType;
+        }
+      }
+      pMap->nSvpt = iSvpt + (op==SAVEPOINT_ROLLBACK);
+      pMap->nRollback = pMap->aSvpt[iSvpt];
+    }
+  }
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** This function is called after an CONCURRENT transaction is opened on the
+** database. It allocates the BtreePtrmap structure used to track pointers
+** to allocated pages and zeroes the nFree/iTrunk fields in the database 
+** header on page 1.
+*/
+static int btreePtrmapAllocate(BtShared *pBt){
+  int rc = SQLITE_OK;
+  if( pBt->pMap==0 ){
+    BtreePtrmap *pMap = sqlite3_malloc(sizeof(BtreePtrmap));
+    if( pMap==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      memset(&pBt->pPage1->aData[32], 0, sizeof(u32)*2);
+      memset(pMap, 0, sizeof(BtreePtrmap));
+      pMap->iFirst = pBt->nPage + 1;
+      pBt->pMap = pMap;
+    }
+  }
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Free any BtreePtrmap structure allocated by an earlier call to
+** btreePtrmapAllocate().
+*/
+static void btreePtrmapDelete(BtShared *pBt){
+  BtreePtrmap *pMap = pBt->pMap;
+  if( pMap ){
+    sqlite3_free(pMap->aRollback);
+    sqlite3_free(pMap->aPtr);
+    sqlite3_free(pMap->aSvpt);
+    sqlite3_free(pMap);
+    pBt->pMap = 0;
+  }
+}
+
+/*
+** Check that the pointer-map does not contain any entries with a parent
+** page of 0. Call sqlite3_log() multiple times to output the entire
+** data structure if it does.
+*/
+static void btreePtrmapCheck(BtShared *pBt, Pgno nPage){
+  Pgno i;
+  int bProblem = 0;
+  BtreePtrmap *p = pBt->pMap;
+
+  for(i=p->iFirst; i<=nPage; i++){
+    PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst];
+    if( pEntry->eType==PTRMAP_OVERFLOW1
+     || pEntry->eType==PTRMAP_OVERFLOW2
+     || pEntry->eType==PTRMAP_BTREE
+    ){
+      if( pEntry->parent==0 ){
+        bProblem = 1;
+        break;
+      }
+    }
+  }
+
+  if( bProblem ){
+    for(i=p->iFirst; i<=nPage; i++){
+      PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst];
+      sqlite3_log(SQLITE_CORRUPT, 
+          "btreePtrmapCheck: pgno=%d eType=%d parent=%d", 
+          (int)i, (int)pEntry->eType, (int)pEntry->parent
+      );
+    }
+    abort();
+  }
+}
+
+#else  /* SQLITE_OMIT_CONCURRENT */
+# define btreePtrmapAllocate(x) SQLITE_OK
+# define btreePtrmapDelete(x) 
+# define btreePtrmapBegin(x,y)  SQLITE_OK
+# define btreePtrmapEnd(x,y,z) 
+# define btreePtrmapCheck(y,z) 
+#endif /* SQLITE_OMIT_CONCURRENT */
+
+static void releasePage(MemPage *pPage);  /* Forward reference */
 static void releasePageOne(MemPage *pPage);      /* Forward reference */
 static void releasePageNotNull(MemPage *pPage);  /* Forward reference */
 
@@ -974,9 +1207,16 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){
   if( *pRC ) return;
 
   assert( sqlite3_mutex_held(pBt->mutex) );
-  /* The master-journal page number must never be used as a pointer map page */
+  /* The master-journal page number is never added to a pointer-map page */
   assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) );
 
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( pBt->pMap ){
+    *pRC = btreePtrmapStore(pBt, key, eType, parent);
+    return;
+  }
+#endif
+
   assert( pBt->autoVacuum );
   if( key==0 ){
     *pRC = SQLITE_CORRUPT_BKPT;
@@ -2138,6 +2378,17 @@ getAndInitPage_error:
   return rc;
 }
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/* 
+** Set the value of the MemPage.pgnoRoot variable, if it exists.
+*/
+static void setMempageRoot(MemPage *pPg, u32 pgnoRoot){
+  pPg->pgnoRoot = pgnoRoot;
+}
+#else
+# define setMempageRoot(x,y)
+#endif
+
 /*
 ** Release a MemPage.  This should be called once for each prior
 ** call to btreeGetPage.
@@ -3291,6 +3542,7 @@ int sqlite3BtreeNewDb(Btree *p){
 int sqlite3BtreeBeginTrans(Btree *p, int wrflag){
   BtShared *pBt = p->pBt;
   int rc = SQLITE_OK;
+  int bConcurrent = (p->db->bConcurrent && !ISAUTOVACUUM);
 
   sqlite3BtreeEnter(p);
   btreeIntegrity(p);
@@ -3360,7 +3612,8 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){
       if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){
         rc = SQLITE_READONLY;
       }else{
-        rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db));
+        int exFlag = bConcurrent ? -1 : (wrflag>1);
+        rc = sqlite3PagerBegin(pBt->pPager, exFlag, sqlite3TempInMemory(p->db));
         if( rc==SQLITE_OK ){
           rc = newDatabase(pBt);
         }
@@ -3415,12 +3668,25 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){
 
 
 trans_begun:
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( bConcurrent && rc==SQLITE_OK && sqlite3PagerIsWal(pBt->pPager) ){
+    rc = sqlite3PagerBeginConcurrent(pBt->pPager);
+    if( rc==SQLITE_OK && wrflag ){
+      rc = btreePtrmapAllocate(pBt);
+    }
+  }
+#endif
+
   if( rc==SQLITE_OK && wrflag ){
     /* This call makes sure that the pager has the correct number of
     ** open savepoints. If the second parameter is greater than 0 and
     ** the sub-journal is not already open, then it will be opened here.
     */
-    rc = sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint);
+    int nSavepoint = p->db->nSavepoint;
+    rc = sqlite3PagerOpenSavepoint(pBt->pPager, nSavepoint);
+    if( rc==SQLITE_OK && nSavepoint ){
+      rc = btreePtrmapBegin(pBt, nSavepoint);
+    }
   }
 
   btreeIntegrity(p);
@@ -3851,6 +4117,175 @@ static int autoVacuumCommit(BtShared *pBt){
 # define setChildPtrmaps(x) SQLITE_OK
 #endif
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** This function is called as part of merging an CONCURRENT transaction with
+** the snapshot at the head of the wal file. It relocates all pages in the
+** range iFirst..iLast, inclusive. It is assumed that the BtreePtrmap 
+** structure at BtShared.pMap contains the location of the pointers to each
+** page in the range.
+**
+** If pnCurrent is NULL, then all pages in the range are moved to currently
+** free locations (i.e. free-list entries) within the database file before page
+** iFirst.
+**
+** Or, if pnCurrent is not NULL, then it points to a value containing the
+** current size of the database file in pages. In this case, all pages are
+** relocated to the end of the database file - page iFirst is relocated to
+** page (*pnCurrent+1), page iFirst+1 to page (*pnCurrent+2), and so on.
+** Value *pnCurrent is set to the new size of the database before this 
+** function returns.
+**
+** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error code.
+*/
+static int btreeRelocateRange(
+  BtShared *pBt,                  /* B-tree handle */
+  Pgno iFirst,                    /* First page to relocate */
+  Pgno iLast,                     /* Last page to relocate */
+  Pgno *pnCurrent                 /* If not NULL, IN/OUT: Database size */
+){
+  int rc = SQLITE_OK;
+  BtreePtrmap *pMap = pBt->pMap;
+  Pgno iPg;
+
+  for(iPg=iFirst; iPg<=iLast && rc==SQLITE_OK; iPg++){
+    MemPage *pFree = 0;     /* Page allocated from free-list */
+    MemPage *pPg = 0;
+    Pgno iNew;              /* New page number for pPg */
+    PtrmapEntry *pEntry;    /* Pointer map entry for page iPg */
+
+    if( iPg==PENDING_BYTE_PAGE(pBt) ) continue;
+    pEntry = &pMap->aPtr[iPg - pMap->iFirst];
+
+    if( pEntry->eType==PTRMAP_FREEPAGE ){
+      Pgno dummy;
+      rc = allocateBtreePage(pBt, &pFree, &dummy, iPg, BTALLOC_EXACT);
+      if( pFree ){
+        assert( sqlite3PagerPageRefcount(pFree->pDbPage)==1 );
+        sqlite3PcacheDrop(pFree->pDbPage);
+      }
+      assert( rc!=SQLITE_OK || dummy==iPg );
+    }else if( pnCurrent ){
+      btreeGetPage(pBt, iPg, &pPg, 0);
+      assert( sqlite3PagerIswriteable(pPg->pDbPage) );
+      assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 );
+      iNew = ++(*pnCurrent);
+      if( iNew==PENDING_BYTE_PAGE(pBt) ) iNew = ++(*pnCurrent);
+      rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent, iNew, 1);
+      releasePageNotNull(pPg);
+    }else{
+      rc = allocateBtreePage(pBt, &pFree, &iNew, iFirst-1, BTALLOC_LE);
+      assert( rc!=SQLITE_OK || iNew<iFirst );
+      if( rc==SQLITE_OK ){
+        releasePage(pFree);
+        btreeGetPage(pBt, iPg, &pPg, 0);
+        rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent,iNew,1);
+        releasePage(pPg);
+      }
+    }
+  }
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** The b-tree handle passed as the only argument is about to commit an
+** CONCURRENT transaction. At this point it is guaranteed that this is 
+** possible - the wal WRITER lock is held and it is known that there are 
+** no conflicts with committed transactions.
+*/
+static int btreeFixUnlocked(Btree *p){
+  BtShared *pBt = p->pBt;
+  MemPage *pPage1 = pBt->pPage1;
+  u8 *p1 = pPage1->aData;
+  Pager *pPager = pBt->pPager;
+  int rc = SQLITE_OK;
+
+  /* If page 1 of the database is not writable, then no pages were allocated
+  ** or freed by this transaction. In this case no special handling is 
+  ** required. Otherwise, if page 1 is dirty, proceed.  */
+  BtreePtrmap *pMap = pBt->pMap;
+  Pgno iTrunk = get4byte(&p1[32]);
+  Pgno nPage = btreePagecount(pBt);
+  u32 nFree = get4byte(&p1[36]);
+
+  assert( pBt->pMap );
+  rc = sqlite3PagerUpgradeSnapshot(pPager, pPage1->pDbPage);
+  assert( p1==pPage1->aData );
+
+  if( rc==SQLITE_OK ){
+    Pgno nHPage = get4byte(&p1[28]);
+    Pgno nFin = nHPage;         /* Size of db after transaction merge */
+
+    if( sqlite3PagerIswriteable(pPage1->pDbPage) ){
+      Pgno iHTrunk = get4byte(&p1[32]);
+      u32 nHFree = get4byte(&p1[36]);
+
+      btreePtrmapCheck(pBt, nPage);
+
+      /* Attach the head database free list to the end of the current
+      ** transactions free-list (if any).  */
+      if( iTrunk!=0 ){
+        put4byte(&p1[36], nHFree + nFree);
+        put4byte(&p1[32], iTrunk);
+        while( iTrunk ){
+          DbPage *pTrunk = sqlite3PagerLookup(pPager, iTrunk);
+          iTrunk = get4byte((u8*)pTrunk->pData);
+          if( iTrunk==0 ){
+            put4byte((u8*)pTrunk->pData, iHTrunk);
+          }
+          sqlite3PagerUnref(pTrunk);
+        };
+      }
+
+      if( nHPage<(pMap->iFirst-1) ){
+        /* The database consisted of (pMap->iFirst-1) pages when the current
+        ** concurrent transaction was opened. And an concurrent transaction may
+        ** not be executed on an auto-vacuum database - so the db should 
+        ** not have shrunk since the transaction was opened. Therefore nHPage
+        ** should be set to (pMap->iFirst-1) or greater. */
+        rc = SQLITE_CORRUPT_BKPT;
+      }else{
+        /* The current transaction allocated pages pMap->iFirst through
+        ** nPage (inclusive) at the end of the database file. Meanwhile,
+        ** other transactions have allocated (iFirst..nHPage). So move
+        ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1). */
+        Pgno iLast = MIN(nPage, nHPage);    /* Last page to move */
+        Pgno nCurrent;                      /* Current size of db */
+
+        nCurrent = MAX(nPage, nHPage);
+        pBt->nPage = nCurrent;
+        rc = btreeRelocateRange(pBt, pMap->iFirst, iLast, &nCurrent);
+
+        /* There are now no collisions with the snapshot at the head of the
+        ** database file. So at this point it would be possible to write
+        ** the transaction out to disk. Before doing so though, attempt to
+        ** relocate some of the new pages to free locations within the body
+        ** of the database file (i.e. free-list entries). */
+        if( rc==SQLITE_OK ){
+          assert( nCurrent!=PENDING_BYTE_PAGE(pBt) );
+          sqlite3PagerSetDbsize(pBt->pPager, nCurrent);
+          nFree = get4byte(&p1[36]);
+          nFin = nCurrent-nFree;
+          if( nCurrent>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){
+            nFin--;
+          }
+          nFin = MAX(nFin, nHPage);
+          rc = btreeRelocateRange(pBt, nFin+1, nCurrent, 0);
+        }
+
+        put4byte(&p1[28], nFin);
+      }
+    }
+    sqlite3PagerSetDbsize(pPager, nFin);
+  }
+
+  return rc;
+}
+#else
+# define btreeFixUnlocked(X)  SQLITE_OK
+#endif /* SQLITE_OMIT_CONCURRENT */
+
 /*
 ** This routine does the first phase of a two-phase commit.  This routine
 ** causes a rollback journal to be created (if it does not already exist)
@@ -3882,8 +4317,10 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){
   if( p->inTrans==TRANS_WRITE ){
     BtShared *pBt = p->pBt;
     sqlite3BtreeEnter(p);
+
 #ifndef SQLITE_OMIT_AUTOVACUUM
     if( pBt->autoVacuum ){
+      assert( ISCONCURRENT==0 );
       rc = autoVacuumCommit(pBt);
       if( rc!=SQLITE_OK ){
         sqlite3BtreeLeave(p);
@@ -3894,7 +4331,12 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){
       sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage);
     }
 #endif
-    rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0);
+    if( rc==SQLITE_OK && ISCONCURRENT ){
+      rc = btreeFixUnlocked(p);
+    }
+    if( rc==SQLITE_OK ){
+      rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0);
+    }
     sqlite3BtreeLeave(p);
   }
   return rc;
@@ -3937,6 +4379,11 @@ static void btreeEndTransaction(Btree *p){
     unlockBtreeIfUnused(pBt);
   }
 
+  /* If this was an CONCURRENT transaction, delete the pBt->pMap object.
+  ** Also call PagerEndConcurrent() to ensure that the pager has discarded
+  ** the record of all pages read within the transaction.  */
+  btreePtrmapDelete(pBt);
+  sqlite3PagerEndConcurrent(pBt->pPager);
   btreeIntegrity(p);
 }
 
@@ -4158,6 +4605,9 @@ int sqlite3BtreeBeginStmt(Btree *p, int iStatement){
   ** such savepoints while the statement transaction savepoint is active.
   */
   rc = sqlite3PagerOpenSavepoint(pBt->pPager, iStatement);
+  if( rc==SQLITE_OK ){
+    rc = btreePtrmapBegin(pBt, iStatement);
+  }
   sqlite3BtreeLeave(p);
   return rc;
 }
@@ -4181,6 +4631,7 @@ int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){
     assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK );
     assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) );
     sqlite3BtreeEnter(p);
+    btreePtrmapEnd(pBt, op, iSavepoint);
     if( op==SAVEPOINT_ROLLBACK ){
       rc = saveAllCursors(pBt, 0, 0);
     }
@@ -4926,6 +5377,7 @@ const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){
 */
 static int moveToChild(BtCursor *pCur, u32 newPgno){
   BtShared *pBt = pCur->pBt;
+  int rc;
 
   assert( cursorOwnsBtShared(pCur) );
   assert( pCur->eState==CURSOR_VALID );
@@ -4940,7 +5392,12 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){
   pCur->apPage[pCur->iPage] = pCur->pPage;
   pCur->ix = 0;
   pCur->iPage++;
-  return getAndInitPage(pBt, newPgno, &pCur->pPage, pCur, pCur->curPagerFlags);
+  rc = getAndInitPage(pBt, newPgno, &pCur->pPage,
+                        pCur, pCur->curPagerFlags);
+  if( rc==SQLITE_OK ){
+    setMempageRoot(pCur->pPage, pCur->pgnoRoot);
+  }
+  return rc;
 }
 
 #ifdef SQLITE_DEBUG
@@ -5052,6 +5509,7 @@ static int moveToRoot(BtCursor *pCur){
       pCur->eState = CURSOR_INVALID;
       return rc;
     }
+    setMempageRoot(pCur->pPage, pCur->pgnoRoot);
     pCur->iPage = 0;
     pCur->curIntKey = pCur->pPage->intKey;
   }
@@ -5757,7 +6215,7 @@ static int allocateBtreePage(
   Pgno mxPage;     /* Total size of the database file */
 
   assert( sqlite3_mutex_held(pBt->mutex) );
-  assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) );
+  assert( eMode==BTALLOC_ANY || (nearby>0 && REQUIRE_PTRMAP ) );
   pPage1 = pBt->pPage1;
   mxPage = btreePagecount(pBt);
   /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36
@@ -5767,6 +6225,15 @@ static int allocateBtreePage(
   if( n>=mxPage ){
     return SQLITE_CORRUPT_BKPT;
   }
+
+  /* Ensure page 1 is writable. This function will either change the number
+  ** of pages in the free-list or the size of the database file. Since both
+  ** of these operations involve modifying page 1 header fields, page 1
+  ** will definitely be written by this transaction. If this is an CONCURRENT
+  ** transaction, ensure the BtreePtrmap structure has been allocated.  */
+  rc = sqlite3PagerWrite(pPage1->pDbPage);
+  if( rc ) return rc;
+
   if( n>0 ){
     /* There are pages on the freelist.  Reuse one of those pages. */
     Pgno iTrunk;
@@ -5777,28 +6244,29 @@ static int allocateBtreePage(
     ** shows that the page 'nearby' is somewhere on the free-list, then
     ** the entire-list will be searched for that page.
     */
-#ifndef SQLITE_OMIT_AUTOVACUUM
     if( eMode==BTALLOC_EXACT ){
-      if( nearby<=mxPage ){
-        u8 eType;
-        assert( nearby>0 );
-        assert( pBt->autoVacuum );
-        rc = ptrmapGet(pBt, nearby, &eType, 0);
-        if( rc ) return rc;
-        if( eType==PTRMAP_FREEPAGE ){
-          searchList = 1;
+      assert( ISAUTOVACUUM!=ISCONCURRENT );
+      if( ISAUTOVACUUM ){
+        if( nearby<=mxPage ){
+          u8 eType;
+          assert( nearby>0 );
+          assert( pBt->autoVacuum );
+          rc = ptrmapGet(pBt, nearby, &eType, 0);
+          if( rc ) return rc;
+          if( eType==PTRMAP_FREEPAGE ){
+            searchList = 1;
+          }
         }
+      }else{
+        searchList = 1;
       }
     }else if( eMode==BTALLOC_LE ){
       searchList = 1;
     }
-#endif
 
     /* Decrement the free-list count by 1. Set iTrunk to the index of the
     ** first free-list trunk page. iPrevTrunk is initially 1.
     */
-    rc = sqlite3PagerWrite(pPage1->pDbPage);
-    if( rc ) return rc;
     put4byte(&pPage1->aData[36], n-1);
 
     /* The code within this loop is run only once if the 'searchList' variable
@@ -6104,7 +6572,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){
   /* If the database supports auto-vacuum, write an entry in the pointer-map
   ** to indicate that the page is free.
   */
-  if( ISAUTOVACUUM ){
+  if( REQUIRE_PTRMAP ){
     ptrmapPut(pBt, iPage, PTRMAP_FREEPAGE, 0, &rc);
     if( rc ) goto freepage_out;
   }
@@ -6418,7 +6886,7 @@ static int fillInCell(
       }
 #endif
       rc = allocateBtreePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl, 0);
-#ifndef SQLITE_OMIT_AUTOVACUUM
+
       /* If the database supports auto-vacuum, and the second or subsequent
       ** overflow page is being allocated, add an entry to the pointer-map
       ** for that page now. 
@@ -6429,14 +6897,13 @@ static int fillInCell(
       ** may misinterpret the uninitialized values and delete the
       ** wrong pages from the database.
       */
-      if( pBt->autoVacuum && rc==SQLITE_OK ){
+      if( REQUIRE_PTRMAP && rc==SQLITE_OK ){
         u8 eType = (pgnoPtrmap?PTRMAP_OVERFLOW2:PTRMAP_OVERFLOW1);
         ptrmapPut(pBt, pgnoOvfl, eType, pgnoPtrmap, &rc);
         if( rc ){
           releasePage(pOvfl);
         }
       }
-#endif
       if( rc ){
         releasePage(pToRelease);
         return rc;
@@ -6578,6 +7045,7 @@ static void insertCell(
     assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */
     assert( j==0 || i==pPage->aiOvfl[j-1]+1 );   /* Overflows are sequential */
   }else{
+    BtShared *pBt = pPage->pBt;
     int rc = sqlite3PagerWrite(pPage->pDbPage);
     if( rc!=SQLITE_OK ){
       *pRC = rc;
@@ -6592,7 +7060,7 @@ static void insertCell(
     ** if it returns successfully */
     assert( idx >= 0 );
     assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB );
-    assert( idx+sz <= (int)pPage->pBt->usableSize );
+    assert( idx+sz <= (int)pBt->usableSize );
     pPage->nFree -= (u16)(2 + sz);
     memcpy(&data[idx], pCell, sz);
     if( iChild ){
@@ -6605,14 +7073,12 @@ static void insertCell(
     /* increment the cell count */
     if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++;
     assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell );
-#ifndef SQLITE_OMIT_AUTOVACUUM
-    if( pPage->pBt->autoVacuum ){
+    if( REQUIRE_PTRMAP ){
       /* The cell may contain a pointer to an overflow page. If so, write
       ** the entry for the overflow page into the pointer map.
       */
       ptrmapPutOvflPtr(pPage, pCell, pRC);
     }
-#endif
   }
 }
 
@@ -7023,7 +7489,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){
     ** be marked as dirty. Returning an error code will cause a
     ** rollback, undoing any changes made to the parent page.
     */
-    if( ISAUTOVACUUM ){
+    if( REQUIRE_PTRMAP ){
       ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc);
       if( szCell>pNew->minLocal ){
         ptrmapPutOvflPtr(pNew, pCell, &rc);
@@ -7160,7 +7626,7 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){
     /* If this is an auto-vacuum database, update the pointer-map entries
     ** for any b-tree or overflow pages that pTo now contains the pointers to.
     */
-    if( ISAUTOVACUUM ){
+    if( REQUIRE_PTRMAP ){
       *pRC = setChildPtrmaps(pTo);
     }
   }
@@ -7211,7 +7677,8 @@ static int balance_nonroot(
   int iParentIdx,                 /* Index of "the page" in pParent */
   u8 *aOvflSpace,                 /* page-size bytes of space for parent ovfl */
   int isRoot,                     /* True if pParent is a root-page */
-  int bBulk                       /* True if this call is part of a bulk load */
+  int bBulk,                      /* True if this call is part of a bulk load */
+  Pgno pgnoRoot                   /* Root page of b-tree being balanced */
 ){
   BtShared *pBt;               /* The whole database */
   int nMaxCells = 0;           /* Allocated size of apCell, szCell, aFrom. */
@@ -7303,6 +7770,8 @@ static int balance_nonroot(
       memset(apOld, 0, (i+1)*sizeof(MemPage*));
       goto balance_cleanup;
     }
+    setMempageRoot(apOld[i], pgnoRoot);
+
     nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow;
     if( (i--)==0 ) break;
 
@@ -7623,7 +8092,7 @@ static int balance_nonroot(
       cntOld[i] = b.nCell;
 
       /* Set the pointer-map entry for the new sibling page. */
-      if( ISAUTOVACUUM ){
+      if( REQUIRE_PTRMAP ){
         ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc);
         if( rc!=SQLITE_OK ){
           goto balance_cleanup;
@@ -7717,7 +8186,7 @@ static int balance_nonroot(
   ** updated. This happens below, after the sibling pages have been 
   ** populated, not here.
   */
-  if( ISAUTOVACUUM ){
+  if( REQUIRE_PTRMAP ){
     MemPage *pNew = apNew[0];
     u8 *aOld = pNew->aData;
     int cntOldNext = pNew->nCell + pNew->nOverflow;
@@ -7901,7 +8370,7 @@ static int balance_nonroot(
     );
     copyNodeContent(apNew[0], pParent, &rc);
     freePage(apNew[0], &rc);
-  }else if( ISAUTOVACUUM && !leafCorrection ){
+  }else if( REQUIRE_PTRMAP && !leafCorrection ){
     /* Fix the pointer map entries associated with the right-child of each
     ** sibling page. All other pointer map entries have already been taken
     ** care of.  */
@@ -7984,7 +8453,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){
   if( rc==SQLITE_OK ){
     rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0);
     copyNodeContent(pRoot, pChild, &rc);
-    if( ISAUTOVACUUM ){
+    if( REQUIRE_PTRMAP ){
       ptrmapPut(pBt, pgnoChild, PTRMAP_BTREE, pRoot->pgno, &rc);
     }
   }
@@ -8111,7 +8580,7 @@ static int balance(BtCursor *pCur){
           */
           u8 *pSpace = sqlite3PageMalloc(pCur->pBt->pageSize);
           rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1,
-                               pCur->hints&BTREE_BULKLOAD);
+                               pCur->hints&BTREE_BULKLOAD, pCur->pgnoRoot);
           if( pFree ){
             /* If pFree is not NULL, it points to the pSpace buffer used 
             ** by a previous call to balance_nonroot(). Its contents are
@@ -8293,7 +8762,7 @@ int sqlite3BtreeInsert(
     }
     rc = clearCell(pPage, oldCell, &info);
     if( info.nSize==szNew && info.nLocal==info.nPayload 
-     && (!ISAUTOVACUUM || szNew<pPage->minLocal)
+     && (!REQUIRE_PTRMAP || szNew<pPage->minLocal)
     ){
       /* Overwrite the old cell with the new if they are the same size.
       ** We could also try to do this if the old cell is smaller, then add
@@ -8726,7 +9195,8 @@ static int clearDatabasePage(
   BtShared *pBt,           /* The BTree that contains the table */
   Pgno pgno,               /* Page number to clear */
   int freePageFlag,        /* Deallocate page if true */
-  int *pnChange            /* Add number of Cells freed to this counter */
+  int *pnChange,           /* Add number of Cells freed to this counter */
+  Pgno pgnoRoot
 ){
   MemPage *pPage;
   int rc;
@@ -8741,6 +9211,7 @@ static int clearDatabasePage(
   }
   rc = getAndInitPage(pBt, pgno, &pPage, 0, 0);
   if( rc ) return rc;
+  setMempageRoot(pPage, pgnoRoot);
   if( pPage->bBusy ){
     rc = SQLITE_CORRUPT_BKPT;
     goto cleardatabasepage_out;
@@ -8750,14 +9221,16 @@ static int clearDatabasePage(
   for(i=0; i<pPage->nCell; i++){
     pCell = findCell(pPage, i);
     if( !pPage->leaf ){
-      rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange);
+      rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange, pgnoRoot);
       if( rc ) goto cleardatabasepage_out;
     }
     rc = clearCell(pPage, pCell, &info);
     if( rc ) goto cleardatabasepage_out;
   }
   if( !pPage->leaf ){
-    rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange);
+    rc = clearDatabasePage(
+        pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange, pgnoRoot
+    );
     if( rc ) goto cleardatabasepage_out;
   }else if( pnChange ){
     assert( pPage->intKey || CORRUPT_DB );
@@ -8802,7 +9275,7 @@ int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){
     ** is the root of a table b-tree - if it is not, the following call is
     ** a no-op).  */
     invalidateIncrblobCursors(p, (Pgno)iTable, 0, 1);
-    rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange);
+    rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange, (Pgno)iTable);
   }
   sqlite3BtreeLeave(p);
   return rc;
@@ -9663,9 +10136,11 @@ char *sqlite3BtreeIntegrityCheck(
   }
   pBt->db->flags = savedDbFlags;
 
-  /* Make sure every page in the file is referenced
-  */
-  for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){
+  /* Make sure every page in the file is referenced. Skip this if the
+  ** database is currently being written by a CONCURRENT transaction (it 
+  ** may fail as pages that were part of the free-list when the transaction
+  ** was opened cannot be counted).  */
+  for(i=1; ISCONCURRENT==0 && i<=sCheck.nPage && sCheck.mxErr; i++){
 #ifdef SQLITE_OMIT_AUTOVACUUM
     if( getPageReferenced(&sCheck, i)==0 ){
       checkAppendMsg(&sCheck, "Page %d is never used", i);
@@ -9969,6 +10444,86 @@ int sqlite3BtreeIsReadonly(Btree *p){
 */
 int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); }
 
+/*
+** This function is called to ensure that all locks required to commit the
+** current write-transaction to the database file are held. If the db is
+** in rollback mode, this means the EXCLUSIVE lock on the database file.
+**
+** Or, if this is an CONCURRENT transaction on a wal-mode database, the WRITER
+** lock on the wal file. In this case this function also checks that the
+** CONCURRENT transaction can be safely committed (does not commit with any
+** other transaction committed since it was opened).
+**
+** SQLITE_OK is returned if successful. SQLITE_BUSY if the required locks
+** cannot be obtained due to a conflicting lock. If the locks cannot be
+** obtained for an CONCURRENT transaction due to a conflict with an already
+** committed transaction, SQLITE_BUSY_SNAPSHOT is returned. Otherwise, if
+** some other error (OOM, IO, etc.) occurs, the relevant SQLite error code
+** is returned.
+*/
+int sqlite3BtreeExclusiveLock(Btree *p){
+  int rc;
+  Pgno pgno = 0;
+  BtShared *pBt = p->pBt;
+  assert( p->inTrans==TRANS_WRITE && pBt->pPage1 );
+  sqlite3BtreeEnter(p);
+  rc = sqlite3PagerExclusiveLock(pBt->pPager, pBt->pPage1->pDbPage, &pgno);
+#ifdef SQLITE_OMIT_CONCURRENT
+  assert( pgno==0 );
+#else
+  if( rc==SQLITE_BUSY_SNAPSHOT && pgno ){
+    PgHdr *pPg = 0;
+    int rc2 = sqlite3PagerGet(pBt->pPager, pgno, &pPg, 0);
+    if( rc2==SQLITE_OK ){
+      int bWrite = -1;
+      const char *zObj = 0;
+      const char *zTab = 0;
+
+      if( pPg ){
+        Pgno pgnoRoot = 0;
+        HashElem *pE;
+        Schema *pSchema;
+
+        pgnoRoot = ((MemPage*)sqlite3PagerGetExtra(pPg))->pgnoRoot;
+        bWrite = sqlite3PagerIswriteable(pPg);
+        sqlite3PagerUnref(pPg);
+
+        pSchema = sqlite3SchemaGet(p->db, p);
+        if( pSchema ){
+          for(pE=sqliteHashFirst(&pSchema->tblHash); pE; pE=sqliteHashNext(pE)){
+            Table *pTab = (Table *)sqliteHashData(pE);
+            if( pTab->tnum==(int)pgnoRoot ){
+              zObj = pTab->zName;
+              zTab = 0;
+            }else{
+              Index *pIdx;
+              for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
+                if( pIdx->tnum==(int)pgnoRoot ){
+                  zObj = pIdx->zName;
+                  zTab = pTab->zName;
+                }
+              }
+            }
+          }
+        }
+      }
+
+      sqlite3_log(SQLITE_OK,
+          "cannot commit CONCURRENT transaction "
+          "- conflict at page %d "
+          "(%s page; part of db %s %s%s%s)",
+          (int)pgno,
+          (bWrite==0?"read-only":(bWrite>0?"read/write":"unknown")),
+          (zTab ? "index" : "table"),
+          (zTab ? zTab : ""), (zTab ? "." : ""), (zObj ? zObj : "UNKNOWN")
+      );
+    }
+  }
+#endif
+  sqlite3BtreeLeave(p);
+  return rc;
+}
+
 #if !defined(SQLITE_OMIT_SHARED_CACHE)
 /*
 ** Return true if the Btree passed as the only argument is sharable.

src/btree.h

@@ -313,6 +313,8 @@ int sqlite3BtreeCursorHasHint(BtCursor*, unsigned int mask);
 int sqlite3BtreeIsReadonly(Btree *pBt);
 int sqlite3HeaderSizeBtree(void);
 
+int sqlite3BtreeExclusiveLock(Btree *pBt);
+
 #ifndef NDEBUG
 int sqlite3BtreeCursorIsValid(BtCursor*);
 #endif
@@ -371,5 +373,4 @@ void sqlite3BtreeCursorList(Btree*);
 # define sqlite3SchemaMutexHeld(X,Y,Z) 1
 #endif
 
-
 #endif /* SQLITE_BTREE_H */

src/btreeInt.h

@@ -232,6 +232,7 @@
 typedef struct MemPage MemPage;
 typedef struct BtLock BtLock;
 typedef struct CellInfo CellInfo;
+typedef struct BtreePtrmap BtreePtrmap;
 
 /*
 ** This is a magic string that appears at the beginning of every
@@ -276,6 +277,9 @@ struct MemPage {
   u8 intKey;           /* True if table b-trees.  False for index b-trees */
   u8 intKeyLeaf;       /* True if the leaf of an intKey table */
   Pgno pgno;           /* Page number for this page */
+#ifndef SQLITE_OMIT_CONCURRENT
+  Pgno pgnoRoot;       /* Root page of b-tree that this page belongs to */
+#endif
   /* Only the first 8 bytes (above) are zeroed by pager.c when a new page
   ** is allocated. All fields that follow must be initialized before use */
   u8 leaf;             /* True if a leaf page */
@@ -440,6 +444,9 @@ struct BtShared {
   Btree *pWriter;       /* Btree with currently open write transaction */
 #endif
   u8 *pTmpSpace;        /* Temp space sufficient to hold a single cell */
+#ifndef SQLITE_OMIT_CONCURRENT
+  BtreePtrmap *pMap;
+#endif
 };
 
 /*
@@ -648,12 +655,19 @@ struct BtCursor {
 ** (sqliteMallocRaw), it is not possible to use conditional compilation.
 ** So, this macro is defined instead.
 */
-#ifndef SQLITE_OMIT_AUTOVACUUM
-#define ISAUTOVACUUM (pBt->autoVacuum)
-#else
+#ifdef SQLITE_OMIT_AUTOVACUUM
 #define ISAUTOVACUUM 0
+#else
+#define ISAUTOVACUUM (pBt->autoVacuum)
 #endif
 
+#ifdef SQLITE_OMIT_CONCURRENT
+# define ISCONCURRENT 0
+#else
+# define ISCONCURRENT (pBt->pMap!=0)
+#endif
+
+#define REQUIRE_PTRMAP (ISAUTOVACUUM || ISCONCURRENT)
 
 /*
 ** This structure is passed around through all the sanity checking routines

src/build.c

@@ -59,8 +59,10 @@ void sqlite3TableLock(
   TableLock *p;
   assert( iDb>=0 );
 
+#ifdef SQLITE_OMIT_CONCURRENT
   if( iDb==1 ) return;
   if( !sqlite3BtreeSharable(pParse->db->aDb[iDb].pBt) ) return;
+#endif
   for(i=0; i<pToplevel->nTableLock; i++){
     p = &pToplevel->aTableLock[i];
     if( p->iDb==iDb && p->iTab==iTab ){
@@ -3978,13 +3980,13 @@ void sqlite3BeginTransaction(Parse *pParse, int type){
   }
   v = sqlite3GetVdbe(pParse);
   if( !v ) return;
-  if( type!=TK_DEFERRED ){
+  if( type==TK_IMMEDIATE || type==TK_EXCLUSIVE ){
     for(i=0; i<db->nDb; i++){
       sqlite3VdbeAddOp2(v, OP_Transaction, i, (type==TK_EXCLUSIVE)+1);
       sqlite3VdbeUsesBtree(v, i);
     }
   }
-  sqlite3VdbeAddOp0(v, OP_AutoCommit);
+  sqlite3VdbeAddOp3(v, OP_AutoCommit, 0, 0, (type==TK_CONCURRENT));
 }
 
 /*

src/main.c

@@ -4152,6 +4152,35 @@ void sqlite3_snapshot_free(sqlite3_snapshot *pSnapshot){
 }
 #endif /* SQLITE_ENABLE_SNAPSHOT */
 
+SQLITE_EXPERIMENTAL int sqlite3_wal_info(
+  sqlite3 *db, const char *zDb, 
+  unsigned int *pnPrior, unsigned int *pnFrame
+){
+  int rc = SQLITE_OK;
+
+#ifndef SQLITE_OMIT_WAL
+  Btree *pBt;
+  int iDb;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( !sqlite3SafetyCheckOk(db) ){
+    return SQLITE_MISUSE_BKPT;
+  }
+#endif
+
+  sqlite3_mutex_enter(db->mutex);
+  iDb = sqlite3FindDbName(db, zDb);
+  if( iDb<0 ){
+    return SQLITE_ERROR;
+  }
+  pBt = db->aDb[iDb].pBt;
+  rc = sqlite3PagerWalInfo(sqlite3BtreePager(pBt), pnPrior, pnFrame);
+  sqlite3_mutex_leave(db->mutex);
+#endif   /* SQLITE_OMIT_WAL */
+
+  return rc;
+}
+
 #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS
 /*
 ** Given the name of a compile-time option, return true if that option

src/os_unix.c

@@ -1124,6 +1124,10 @@ struct unixInodeInfo {
   sem_t *pSem;                    /* Named POSIX semaphore */
   char aSemName[MAX_PATHNAME+2];  /* Name of that semaphore */
 #endif
+#ifdef SQLITE_SHARED_MAPPING
+  sqlite3_int64 nSharedMapping;   /* Size of mapped region in bytes */
+  void *pSharedMapping;           /* Memory mapped region */
+#endif
 };
 
 /*
@@ -1258,6 +1262,13 @@ static void releaseInodeInfo(unixFile *pFile){
     pInode->nRef--;
     if( pInode->nRef==0 ){
       assert( pInode->pShmNode==0 );
+#ifdef SQLITE_SHARED_MAPPING
+      if( pInode->pSharedMapping ){
+        osMunmap(pInode->pSharedMapping, pInode->nSharedMapping);
+        pInode->pSharedMapping = 0;
+        pInode->nSharedMapping = 0;
+      }
+#endif
       closePendingFds(pFile);
       if( pInode->pPrev ){
         assert( pInode->pPrev->pNext==pInode );
@@ -2064,6 +2075,14 @@ static int nolockUnlock(sqlite3_file *NotUsed, int NotUsed2){
 ** Close the file.
 */
 static int nolockClose(sqlite3_file *id) {
+#ifdef SQLITE_SHARED_MAPPING
+  unixFile *pFd = (unixFile*)id;
+  if( pFd->pInode ){
+    unixEnterMutex();
+    releaseInodeInfo(pFd);
+    unixLeaveMutex();
+  }
+#endif
   return closeUnixFile(id);
 }
 
@@ -3883,6 +3902,9 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){
       *(i64*)pArg = pFile->mmapSizeMax;
       if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){
         pFile->mmapSizeMax = newLimit;
+#ifdef SQLITE_SHARED_MAPPING
+        if( pFile->pInode==0 )
+#endif
         if( pFile->mmapSize>0 ){
           unixUnmapfile(pFile);
           rc = unixMapfile(pFile, -1);
@@ -4835,6 +4857,9 @@ static int unixShmUnmap(
 */
 static void unixUnmapfile(unixFile *pFd){
   assert( pFd->nFetchOut==0 );
+#ifdef SQLITE_SHARED_MAPPING
+  if( pFd->pInode ) return;
+#endif
   if( pFd->pMapRegion ){
     osMunmap(pFd->pMapRegion, pFd->mmapSizeActual);
     pFd->pMapRegion = 0;
@@ -4966,6 +4991,28 @@ static int unixMapfile(unixFile *pFd, i64 nMap){
     nMap = pFd->mmapSizeMax;
   }
 
+#ifdef SQLITE_SHARED_MAPPING
+  if( pFd->pInode ){
+    unixInodeInfo *pInode = pFd->pInode;
+    if( pFd->pMapRegion ) return SQLITE_OK;
+    unixEnterMutex();
+    if( pInode->pSharedMapping==0 ){
+      u8 *pNew = osMmap(0, nMap, PROT_READ, MAP_SHARED, pFd->h, 0);
+      if( pNew==MAP_FAILED ){
+        unixLogError(SQLITE_OK, "mmap", pFd->zPath);
+        pFd->mmapSizeMax = 0;
+      }else{
+        pInode->pSharedMapping = pNew;
+        pInode->nSharedMapping = nMap;
+      }
+    }
+    pFd->pMapRegion = pInode->pSharedMapping;
+    pFd->mmapSizeActual = pFd->mmapSize = pInode->nSharedMapping;
+    unixLeaveMutex();
+    return SQLITE_OK;
+  }
+#endif
+
   assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) );
   if( nMap!=pFd->mmapSize ){
     unixRemapfile(pFd, nMap);
@@ -5403,6 +5450,9 @@ static int fillInUnixFile(
   if( pLockingStyle == &posixIoMethods
 #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE
     || pLockingStyle == &nfsIoMethods
+#endif
+#ifdef SQLITE_SHARED_MAPPING
+    || pLockingStyle == &nolockIoMethods
 #endif
   ){
     unixEnterMutex();

src/pager.c

@@ -670,6 +670,9 @@ struct Pager {
   u32 cksumInit;              /* Quasi-random value added to every checksum */
   u32 nSubRec;                /* Number of records written to sub-journal */
   Bitvec *pInJournal;         /* One bit for each page in the database file */
+#ifndef SQLITE_OMIT_CONCURRENT
+  Bitvec *pAllRead;           /* Pages read within current CONCURRENT trans. */
+#endif
   sqlite3_file *fd;           /* File descriptor for database */
   sqlite3_file *jfd;          /* File descriptor for main journal */
   sqlite3_file *sjfd;         /* File descriptor for sub-journal */
@@ -926,7 +929,9 @@ static int assert_pager_state(Pager *p){
       if( !pagerUseWal(pPager) ){
         assert( p->eLock>=RESERVED_LOCK );
       }
-      assert( pPager->dbSize==pPager->dbOrigSize );
+#ifndef SQLITE_OMIT_CONCURRENT
+      assert( pPager->dbSize==pPager->dbOrigSize || pPager->pAllRead );
+#endif
       assert( pPager->dbOrigSize==pPager->dbFileSize );
       assert( pPager->dbOrigSize==pPager->dbHintSize );
       assert( pPager->setMaster==0 );
@@ -1807,6 +1812,53 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){
   return rc;
 }
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** If they are not already, begin recording all pages read from the pager layer
+** by the b-tree layer This is used by concurrent transactions. Return
+** SQLITE_OK if successful, or an SQLite error code (SQLITE_NOMEM) if an error
+** occurs.
+*/
+int sqlite3PagerBeginConcurrent(Pager *pPager){
+  int rc = SQLITE_OK;
+  if( pPager->pAllRead==0 ){
+    pPager->pAllRead = sqlite3BitvecCreate(pPager->dbSize);
+    pPager->dbOrigSize = pPager->dbSize;
+    if( pPager->pAllRead==0 ){
+      rc = SQLITE_NOMEM;
+    }
+  }
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Stop recording all pages read from the pager layer by the b-tree layer
+** and discard any current records.
+*/
+void sqlite3PagerEndConcurrent(Pager *pPager){
+  sqlite3BitvecDestroy(pPager->pAllRead);
+  pPager->pAllRead = 0;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Return true if the database is in wal mode. False otherwise.
+*/
+int sqlite3PagerIsWal(Pager *pPager){
+  return pPager->pWal!=0;
+}
+#endif /* SQLITE_OMIT_CONCURRENT */
+
+/*
+** Free the Pager.pInJournal and Pager.pAllRead bitvec objects.
+*/
+static void pagerFreeBitvecs(Pager *pPager){
+  sqlite3BitvecDestroy(pPager->pInJournal);
+  pPager->pInJournal = 0;
+  sqlite3PagerEndConcurrent(pPager);
+}
+
 /*
 ** This function is a no-op if the pager is in exclusive mode and not
 ** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN
@@ -1831,8 +1883,7 @@ static void pager_unlock(Pager *pPager){
        || pPager->eState==PAGER_ERROR 
   );
 
-  sqlite3BitvecDestroy(pPager->pInJournal);
-  pPager->pInJournal = 0;
+  pagerFreeBitvecs(pPager);
   releaseAllSavepoints(pPager);
 
   if( pagerUseWal(pPager) ){
@@ -2100,8 +2151,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){
   }
 #endif
 
-  sqlite3BitvecDestroy(pPager->pInJournal);
-  pPager->pInJournal = 0;
+  pagerFreeBitvecs(pPager);
   pPager->nRec = 0;
   if( rc==SQLITE_OK ){
     if( MEMDB || pagerFlushOnCommit(pPager, bCommit) ){
@@ -3153,6 +3203,16 @@ static int pagerRollbackWal(Pager *pPager){
   pPager->dbSize = pPager->dbOrigSize;
   rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager);
   pList = sqlite3PcacheDirtyList(pPager->pPCache);
+
+#ifndef SQLITE_OMIT_CONCURRENT
+  /* If this is an CONCURRENT transaction, then page 1 must be reread from 
+  ** the db file, even if it is not dirty. This is because the b-tree layer 
+  ** may have already zeroed the nFree and iTrunk header fields.  */
+  if( rc==SQLITE_OK && (pList==0 || pList->pgno!=1) && pPager->pAllRead ){
+    rc = pagerUndoCallback((void*)pPager, 1);
+  }
+#endif
+
   while( pList && rc==SQLITE_OK ){
     PgHdr *pNext = pList->pDirty;
     rc = pagerUndoCallback((void *)pPager, pList->pgno);
@@ -3202,6 +3262,8 @@ static int pagerWalFrames(
       if( p->pgno<=nTruncate ){
         ppNext = &p->pDirty;
         nList++;
+        PAGERTRACE(("TO-WAL %d page %d hash(%08x)\n",
+                     PAGERID(pPager), p->pgno, pager_pagehash(p)));
       }
     }
     assert( pList );
@@ -4264,7 +4326,7 @@ static int syncJournal(Pager *pPager, int newHdr){
   assert( assert_pager_state(pPager) );
   assert( !pagerUseWal(pPager) );
 
-  rc = sqlite3PagerExclusiveLock(pPager);
+  rc = sqlite3PagerExclusiveLock(pPager, 0, 0);
   if( rc!=SQLITE_OK ) return rc;
 
   if( !pPager->noSync ){
@@ -4622,6 +4684,12 @@ static int pagerStress(void *p, PgHdr *pPg){
   pPager->aStat[PAGER_STAT_SPILL]++;
   pPg->pDirty = 0;
   if( pagerUseWal(pPager) ){
+#ifndef SQLITE_OMIT_CONCURRENT
+    /* If the transaction is a "BEGIN CONCURRENT" transaction, the page 
+    ** cannot be flushed to disk. Return early in this case. */
+    if( pPager->pAllRead ) return SQLITE_OK;
+#endif
+
     /* Write a single frame for this page to the log. */
     rc = subjournalPageIfRequired(pPg); 
     if( rc==SQLITE_OK ){
@@ -5455,6 +5523,18 @@ static int getPageNormal(
   assert( assert_pager_state(pPager) );
   assert( pPager->hasHeldSharedLock==1 );
 
+#ifndef SQLITE_OMIT_CONCURRENT
+  /* If this is an CONCURRENT transaction and the page being read was
+  ** present in the database file when the transaction was opened,
+  ** mark it as read in the pAllRead vector.  */
+  pPg = 0;
+  if( pPager->pAllRead && pgno<=pPager->dbOrigSize ){
+    PAGERTRACE(("USING page %d\n", pgno));
+    rc = sqlite3BitvecSet(pPager->pAllRead, pgno);
+    if( rc!=SQLITE_OK ) goto pager_acquire_err;
+  }
+#endif
+
   if( pgno==0 ) return SQLITE_CORRUPT_BKPT;
   pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3);
   if( pBase==0 ){
@@ -5795,11 +5875,14 @@ static int pager_open_journal(Pager *pPager){
 ** Begin a write-transaction on the specified pager object. If a 
 ** write-transaction has already been opened, this function is a no-op.
 **
-** If the exFlag argument is false, then acquire at least a RESERVED
-** lock on the database file. If exFlag is true, then acquire at least
+** If the exFlag argument is 0, then acquire at least a RESERVED
+** lock on the database file. If exFlag is >0, then acquire at least
 ** an EXCLUSIVE lock. If such a lock is already held, no locking 
 ** functions need be called.
 **
+** If (exFlag<0) and the database is in WAL mode, do not take any locks.
+** The transaction will run in CONCURRENT mode instead.
+**
 ** If the subjInMemory argument is non-zero, then any sub-journal opened
 ** within this transaction will be opened as an in-memory file. This
 ** has no effect if the sub-journal is already opened (as it may be when
@@ -5817,7 +5900,6 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){
 
   if( ALWAYS(pPager->eState==PAGER_READER) ){
     assert( pPager->pInJournal==0 );
-
     if( pagerUseWal(pPager) ){
       /* If the pager is configured to use locking_mode=exclusive, and an
       ** exclusive lock on the database is not already held, obtain it now.
@@ -5833,9 +5915,10 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){
       /* Grab the write lock on the log file. If successful, upgrade to
       ** PAGER_RESERVED state. Otherwise, return an error code to the caller.
       ** The busy-handler is not invoked if another connection already
-      ** holds the write-lock. If possible, the upper layer will call it.
-      */
-      rc = sqlite3WalBeginWriteTransaction(pPager->pWal);
+      ** holds the write-lock. If possible, the upper layer will call it.  */
+      if( exFlag>=0 ){
+        rc = sqlite3WalBeginWriteTransaction(pPager->pWal);
+      }
     }else{
       /* Obtain a RESERVED lock on the database file. If the exFlag parameter
       ** is true, then immediately upgrade this to an EXCLUSIVE lock. The
@@ -5843,7 +5926,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){
       ** lock, but not when obtaining the RESERVED lock.
       */
       rc = pagerLockDb(pPager, RESERVED_LOCK);
-      if( rc==SQLITE_OK && exFlag ){
+      if( rc==SQLITE_OK && exFlag>0 ){
         rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK);
       }
     }
@@ -6143,7 +6226,7 @@ int sqlite3PagerWrite(PgHdr *pPg){
 ** to sqlite3PagerWrite().  In other words, return TRUE if it is ok
 ** to change the content of the page.
 */
-#ifndef NDEBUG
+#if !defined(SQLITE_OMIT_CONCURRENT) || !defined(NDEBUG)
 int sqlite3PagerIswriteable(DbPage *pPg){
   return pPg->flags & PGHDR_WRITEABLE;
 }
@@ -6302,17 +6385,26 @@ int sqlite3PagerSync(Pager *pPager, const char *zMaster){
 }
 
 /*
-** This function may only be called while a write-transaction is active in
-** rollback. If the connection is in WAL mode, this call is a no-op. 
-** Otherwise, if the connection does not already have an EXCLUSIVE lock on 
-** the database file, an attempt is made to obtain one.
+** This function is called to ensure that all locks required to commit the
+** current write-transaction to the database file are held. If the db is
+** in rollback mode, this means the EXCLUSIVE lock on the database file.
 **
-** If the EXCLUSIVE lock is already held or the attempt to obtain it is
-** successful, or the connection is in WAL mode, SQLITE_OK is returned.
-** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is 
-** returned.
+** Or, if this is a non-CONCURRENT transaction on a wal-mode database, this
+** function is a no-op.
+**
+** If this is an CONCURRENT transaction on a wal-mode database, this function
+** attempts to obtain the WRITER lock on the wal file and also checks to
+** see that the transaction can be safely committed (does not commit with 
+** any other transaction committed since it was opened).
+**
+** If the required locks are already held or successfully obtained and
+** the transaction can be committed, SQLITE_OK is returned. If a required lock
+** cannot be obtained, SQLITE_BUSY is returned. Or, if the current transaction
+** is CONCURRENT and cannot be committed due to a conflict, SQLITE_BUSY_SNAPSHOT
+** is returned. Otherwise, if some other error occurs (IO error, OOM etc.),
+** and SQLite error code is returned.
 */
-int sqlite3PagerExclusiveLock(Pager *pPager){
+int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1, Pgno *piConflict){
   int rc = pPager->errCode;
   assert( assert_pager_state(pPager) );
   if( rc==SQLITE_OK ){
@@ -6324,10 +6416,73 @@ int sqlite3PagerExclusiveLock(Pager *pPager){
     if( 0==pagerUseWal(pPager) ){
       rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK);
     }
+#ifndef SQLITE_OMIT_CONCURRENT
+    else{
+      if( pPager->pAllRead ){
+        /* This is an CONCURRENT transaction. Attempt to lock the wal database
+        ** here. If SQLITE_BUSY (but not SQLITE_BUSY_SNAPSHOT) is returned,
+        ** invoke the busy-handler and try again for as long as it returns
+        ** non-zero.  */
+        do {
+          rc = sqlite3WalLockForCommit(
+              pPager->pWal, pPage1, pPager->pAllRead, piConflict
+          );
+        }while( rc==SQLITE_BUSY 
+             && pPager->xBusyHandler(pPager->pBusyHandlerArg) 
+        );
+      }
+    }
+#endif /* SQLITE_OMIT_CONCURRENT */
   }
   return rc;
 }
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** This function is called as part of committing an CONCURRENT transaction.
+** At this point the wal WRITER lock is held, and all pages in the cache 
+** except for page 1 are compatible with the snapshot at the head of the
+** wal file. 
+**
+** This function updates the in-memory data structures and reloads the
+** contents of page 1 so that the client is operating on the snapshot 
+** at the head of the wal file.
+**
+** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
+*/
+int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage *pPage1){
+  int rc;
+
+  assert( pPager->pWal && pPager->pAllRead );
+  rc = sqlite3WalUpgradeSnapshot(pPager->pWal);
+  if( rc==SQLITE_OK ){
+    rc = readDbPage(pPage1);
+  }
+
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Set the in-memory cache of the database file size to nSz pages.
+*/
+void sqlite3PagerSetDbsize(Pager *pPager, Pgno nSz){
+  pPager->dbSize = nSz;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** If this is a WAL mode connection and the WRITER lock is currently held,
+** relinquish it.
+*/
+void sqlite3PagerDropExclusiveLock(Pager *pPager){
+  if( pagerUseWal(pPager) ){
+    sqlite3WalEndWriteTransaction(pPager->pWal);
+  }
+}
+#endif  /* SQLITE_OMIT_CONCURRENT */
+
+
 /*
 ** Sync the database file for the pager pPager. zMaster points to the name
 ** of a master journal file that should be written into the individual
@@ -7636,6 +7791,11 @@ int sqlite3PagerSnapshotRecover(Pager *pPager){
   return rc;
 }
 #endif /* SQLITE_ENABLE_SNAPSHOT */
+
+int sqlite3PagerWalInfo(Pager *pPager, u32 *pnPrior, u32 *pnFrame){
+  return sqlite3WalInfo(pPager->pWal, pnPrior, pnFrame);
+}
+
 #endif /* !SQLITE_OMIT_WAL */
 
 #ifdef SQLITE_ENABLE_ZIPVFS

src/pager.h

@@ -165,7 +165,7 @@ void *sqlite3PagerGetExtra(DbPage *);
 void sqlite3PagerPagecount(Pager*, int*);
 int sqlite3PagerBegin(Pager*, int exFlag, int);
 int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int);
-int sqlite3PagerExclusiveLock(Pager*);
+int sqlite3PagerExclusiveLock(Pager*, DbPage *pPage1, Pgno*);
 int sqlite3PagerSync(Pager *pPager, const char *zMaster);
 int sqlite3PagerCommitPhaseTwo(Pager*);
 int sqlite3PagerRollback(Pager*);
@@ -173,6 +173,7 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int n);
 int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint);
 int sqlite3PagerSharedLock(Pager *pPager);
 
+
 #ifndef SQLITE_OMIT_WAL
   int sqlite3PagerCheckpoint(Pager *pPager, sqlite3*, int, int*, int*);
   int sqlite3PagerWalSupported(Pager *pPager);
@@ -218,6 +219,23 @@ void sqlite3PagerTruncateImage(Pager*,Pgno);
 
 void sqlite3PagerRekey(DbPage*, Pgno, u16);
 
+#ifndef SQLITE_OMIT_CONCURRENT
+void sqlite3PagerEndConcurrent(Pager*);
+int sqlite3PagerBeginConcurrent(Pager*);
+void sqlite3PagerDropExclusiveLock(Pager*);
+int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage*);
+void sqlite3PagerSetDbsize(Pager *pPager, Pgno);
+int sqlite3PagerIsWal(Pager*);
+#else
+# define sqlite3PagerEndConcurrent(x)
+#endif
+
+#if defined(SQLITE_DEBUG) || !defined(SQLITE_OMIT_CONCURRENT)
+int sqlite3PagerIswriteable(DbPage*);
+#endif
+
+int sqlite3PagerWalInfo(Pager*, u32 *pnPrior, u32 *pnFrame);
+
 #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL)
 void *sqlite3PagerCodec(DbPage *);
 #endif
@@ -225,7 +243,6 @@ void *sqlite3PagerCodec(DbPage *);
 /* Functions to support testing and debugging. */
 #if !defined(NDEBUG) || defined(SQLITE_TEST)
   Pgno sqlite3PagerPagenumber(DbPage*);
-  int sqlite3PagerIswriteable(DbPage*);
 #endif
 #ifdef SQLITE_TEST
   int *sqlite3PagerStats(Pager*);

src/parse.y

@@ -98,6 +98,13 @@
 */
 struct TrigEvent { int a; IdList * b; };
 
+/*
+** Generate a syntax error
+*/
+static void parserSyntaxError(Parse *pParse, Token *p){
+  sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", p);
+}
+
 /*
 ** Disable lookaside memory allocation for objects that might be
 ** shared across database connections.
@@ -133,7 +140,16 @@ trans_opt ::= TRANSACTION nm.
 transtype(A) ::= .             {A = TK_DEFERRED;}
 transtype(A) ::= DEFERRED(X).  {A = @X; /*A-overwrites-X*/}
 transtype(A) ::= IMMEDIATE(X). {A = @X; /*A-overwrites-X*/}
-transtype(A) ::= EXCLUSIVE(X). {A = @X; /*A-overwrites-X*/}
+transtype(A) ::= ID(X). {
+   Token *p = &X;
+   if( p->n==9 && sqlite3_strnicmp(p->z,"exclusive",9)==0 ){
+     A = TK_EXCLUSIVE;
+   }else if( p->n==10 && sqlite3_strnicmp(p->z,"concurrent",10)==0 ){
+     A = TK_CONCURRENT;  /*A-overwrites-X*/
+   }else{
+     parserSyntaxError(pParse, p);
+   }
+}
 cmd ::= COMMIT|END(X) trans_opt.   {sqlite3EndTransaction(pParse,@X);}
 cmd ::= ROLLBACK(X) trans_opt.     {sqlite3EndTransaction(pParse,@X);}
 
@@ -245,7 +261,6 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);}
 //
 %token_class id  ID|INDEXED.
 
-
 // And "ids" is an identifer-or-string.
 //
 %token_class ids  ID|STRING.
@@ -919,7 +934,7 @@ expr(A) ::= VARIABLE(X).     {
     Token t = X; /*A-overwrites-X*/
     assert( t.n>=2 );
     if( pParse->nested==0 ){
-      sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t);
+      parserSyntaxError(pParse, &t);
       A = 0;
     }else{
       A = sqlite3PExpr(pParse, TK_REGISTER, 0, 0);

src/pragma.h

@@ -440,6 +440,15 @@ static const PragmaName aPragmaName[] = {
   /* iArg:      */ 0 },
 #endif
 #endif
+#endif
+#if !defined(SQLITE_OMIT_FLAG_PRAGMAS)
+#if defined(SQLITE_ENABLE_NOOP_UPDATE)
+ {/* zName:     */ "noop_update",
+  /* ePragTyp:  */ PragTyp_FLAG,
+  /* ePragFlg:  */ PragFlg_Result0|PragFlg_NoColumns1,
+  /* ColNames:  */ 0, 0,
+  /* iArg:      */ SQLITE_NoopUpdate },
+#endif
 #endif
  {/* zName:     */ "optimize",
   /* ePragTyp:  */ PragTyp_OPTIMIZE,
@@ -646,4 +655,4 @@ static const PragmaName aPragmaName[] = {
   /* iArg:      */ SQLITE_WriteSchema },
 #endif
 };
-/* Number of pragmas: 60 on by default, 77 total. */
+/* Number of pragmas: 60 on by default, 78 total. */

src/sqlite.h.in

@@ -8748,6 +8748,31 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp(
 */
 SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);
 
+/*
+** CAPI3REF: Wal related information regarding the most recent COMMIT
+** EXPERIMENTAL
+**
+** This function reports on the state of the wal file (if any) for database 
+** zDb, which should be "main", "temp", or the name of the attached database.
+** Its results - the values written to the output parameters - are only
+** defined if the most recent SQL command on the connection was a successful 
+** COMMIT that wrote data to wal-mode database zDb.
+**
+** Assuming the above conditions are met, output parameter (*pnFrame) is set
+** to the total number of frames in the wal file. Parameter (*pnPrior) is
+** set to the number of frames that were present in the wal file before the
+** most recent transaction was committed. So that the number of frames written
+** by the most recent transaction is (*pnFrame)-(*pnPrior).
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. It
+** is not an error if this function is called at a time when the results
+** are undefined.
+*/
+SQLITE_EXPERIMENTAL int sqlite3_wal_info(
+  sqlite3 *db, const char *zDb, 
+  unsigned int *pnPrior, unsigned int *pnFrame
+);
+
 /*
 ** CAPI3REF: Serialize a database
 **

src/sqliteInt.h

@@ -1344,6 +1344,7 @@ struct sqlite3 {
   u16 dbOptFlags;               /* Flags to enable/disable optimizations */
   u8 enc;                       /* Text encoding */
   u8 autoCommit;                /* The auto-commit flag. */
+  u8 bConcurrent;               /* Current transaction is "CONCURRENT" */
   u8 temp_store;                /* 1: file 2: memory 0: default */
   u8 mallocFailed;              /* True if we have seen a malloc failure */
   u8 bBenignMalloc;             /* Do not require OOMs if true */
@@ -1498,6 +1499,7 @@ struct sqlite3 {
 #define SQLITE_EnableQPSG     0x00800000  /* Query Planner Stability Guarantee*/
 #define SQLITE_TriggerEQP     0x01000000  /* Show trigger EXPLAIN QUERY PLAN */
 
+#define SQLITE_NoopUpdate     0x01000000  /* UPDATE operations are no-ops */
 /* Flags used only if debugging */
 #ifdef SQLITE_DEBUG
 #define SQLITE_SqlTrace       0x08000000  /* Debug print SQL as it executes */

src/test1.c

@@ -7449,6 +7449,41 @@ static int SQLITE_TCLAPI test_dbconfig_maindbname_icecube(
   }
 }
 
+/*
+** Usage: sqlite3_wal_info DB DBNAME
+*/
+static int SQLITE_TCLAPI test_wal_info(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int rc;
+  sqlite3 *db;
+  char *zName;
+  unsigned int nPrior;
+  unsigned int nFrame;
+
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME");
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+  zName = Tcl_GetString(objv[2]);
+
+  rc = sqlite3_wal_info(db, zName, &nPrior, &nFrame);
+  if( rc!=SQLITE_OK ){
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+    return TCL_ERROR;
+  }else{
+    Tcl_Obj *pNew = Tcl_NewObj();
+    Tcl_ListObjAppendElement(interp, pNew, Tcl_NewWideIntObj((i64)nPrior));
+    Tcl_ListObjAppendElement(interp, pNew, Tcl_NewWideIntObj((i64)nFrame));
+    Tcl_SetObjResult(interp, pNew);
+  }
+  return TCL_OK;
+}
+
 /*
 ** Usage: sqlite3_mmap_warm DB DBNAME
 */
@@ -7748,8 +7783,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_snapshot_open_blob", test_snapshot_open_blob, 0 },
      { "sqlite3_snapshot_cmp_blob", test_snapshot_cmp_blob, 0 },
 #endif
-     { "sqlite3_delete_database", test_delete_database,    0 },
-     { "atomic_batch_write",      test_atomic_batch_write, 0 },
+     { "sqlite3_delete_database", test_delete_database, 0 },
+     { "sqlite3_wal_info", test_wal_info, 0 },
+     { "atomic_batch_write",      test_atomic_batch_write,     0   },
      { "sqlite3_mmap_warm",       test_mmap_warm,          0 },
   };
   static int bitmask_size = sizeof(Bitmask)*8;

src/test_config.c

@@ -672,6 +672,12 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
   Tcl_SetVar2(interp, "sqlite_options", "truncate_opt", "1", TCL_GLOBAL_ONLY);
 #endif
 
+#ifndef SQLITE_OMIT_CONCURRENT
+  Tcl_SetVar2(interp, "sqlite_options", "concurrent", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "concurrent", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_OMIT_UTF16
   Tcl_SetVar2(interp, "sqlite_options", "utf16", "0", TCL_GLOBAL_ONLY);
 #else

src/update.c

@@ -235,6 +235,17 @@ void sqlite3Update(
   */
   chngRowid = chngPk = 0;
   for(i=0; i<pChanges->nExpr; i++){
+#if defined(SQLITE_ENABLE_NOOP_UPDATE) && !defined(SQLITE_OMIT_FLAG_PRAGMAS)
+    if( db->flags & SQLITE_NoopUpdate ){
+      Token x;
+      sqlite3ExprDelete(db, pChanges->a[i].pExpr);
+      x.z = pChanges->a[i].zName;
+      x.n = sqlite3Strlen30(x.z);
+      pChanges->a[i].pExpr =
+         sqlite3PExpr(pParse, TK_UPLUS, sqlite3ExprAlloc(db, TK_ID, &x, 0), 0);
+      if( db->mallocFailed ) goto update_cleanup;
+    }
+#endif
     if( sqlite3ResolveExprNames(&sNC, pChanges->a[i].pExpr) ){
       goto update_cleanup;
     }

src/vacuum.c

@@ -354,6 +354,7 @@ end_of_vacuum:
   ** is closed by the DETACH.
   */
   db->autoCommit = 1;
+  assert( db->bConcurrent==0 );
 
   if( pDb ){
     sqlite3BtreeClose(pDb->pBt);

src/vdbe.c

@@ -3020,6 +3020,7 @@ case OP_Savepoint: {
       ** is committed. 
       */
       int isTransaction = pSavepoint->pNext==0 && db->isTransactionSavepoint;
+      assert( db->bConcurrent==0 || db->isTransactionSavepoint==0 );
       if( isTransaction && p1==SAVEPOINT_RELEASE ){
         if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){
           goto vdbe_return;
@@ -3096,23 +3097,31 @@ case OP_Savepoint: {
   break;
 }
 
-/* Opcode: AutoCommit P1 P2 * * *
+/* Opcode: AutoCommit P1 P2 P3 * *
 **
 ** Set the database auto-commit flag to P1 (1 or 0). If P2 is true, roll
 ** back any currently active btree transactions. If there are any active
 ** VMs (apart from this one), then a ROLLBACK fails.  A COMMIT fails if
 ** there are active writing VMs or active VMs that use shared cache.
 **
+** If P3 is non-zero, then this instruction is being executed as part of
+** a "BEGIN CONCURRENT" command.
+**
 ** This instruction causes the VM to halt.
 */
 case OP_AutoCommit: {
   int desiredAutoCommit;
   int iRollback;
+  int bConcurrent;
+  int hrc;
 
   desiredAutoCommit = pOp->p1;
   iRollback = pOp->p2;
+  bConcurrent = pOp->p3;
   assert( desiredAutoCommit==1 || desiredAutoCommit==0 );
   assert( desiredAutoCommit==1 || iRollback==0 );
+  assert( desiredAutoCommit==0 || bConcurrent==0 );
+  assert( db->autoCommit==0 || db->bConcurrent==0 );
   assert( db->nVdbeActive>0 );  /* At least this one VM is active */
   assert( p->bIsReader );
 
@@ -3121,10 +3130,17 @@ case OP_AutoCommit: {
       assert( desiredAutoCommit==1 );
       sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK);
       db->autoCommit = 1;
-    }else if( desiredAutoCommit && db->nVdbeWrite>0 ){
-      /* If this instruction implements a COMMIT and other VMs are writing
-      ** return an error indicating that the other VMs must complete first. 
-      */
+      db->bConcurrent = 0;
+    }else if( desiredAutoCommit
+            && (db->nVdbeWrite>0 || (db->bConcurrent && db->nVdbeActive>1)) ){
+      /* A transaction may only be committed if there are no other active
+      ** writer VMs. If the transaction is CONCURRENT, then it may only be
+      ** committed if there are no active VMs at all (readers or writers).
+      **
+      ** If this instruction is a COMMIT and the transaction may not be
+      ** committed due to one of the conditions above, return an error
+      ** indicating that other VMs must complete before the COMMIT can 
+      ** be processed.  */
       sqlite3VdbeError(p, "cannot commit transaction - "
                           "SQL statements in progress");
       rc = SQLITE_BUSY;
@@ -3134,12 +3150,15 @@ case OP_AutoCommit: {
     }else{
       db->autoCommit = (u8)desiredAutoCommit;
     }
-    if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){
+    hrc = sqlite3VdbeHalt(p);
+    if( (hrc & 0xFF)==SQLITE_BUSY ){
       p->pc = (int)(pOp - aOp);
       db->autoCommit = (u8)(1-desiredAutoCommit);
-      p->rc = rc = SQLITE_BUSY;
+      p->rc = hrc;
+      rc = SQLITE_BUSY;
       goto vdbe_return;
     }
+    db->bConcurrent = (u8)bConcurrent;
     assert( db->nStatement==0 );
     sqlite3CloseSavepoints(db);
     if( p->rc==SQLITE_OK ){
@@ -3331,10 +3350,22 @@ case OP_SetCookie: {
   pDb = &db->aDb[pOp->p1];
   assert( pDb->pBt!=0 );
   assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) );
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( db->bConcurrent 
+   && (pOp->p2==BTREE_USER_VERSION || pOp->p2==BTREE_APPLICATION_ID)
+  ){
+    rc = SQLITE_ERROR;
+    sqlite3VdbeError(p, "cannot modify %s within CONCURRENT transaction",
+        pOp->p2==BTREE_USER_VERSION ? "user_version" : "application_id"
+    );
+    goto abort_due_to_error;
+  }
+#endif
   /* See note about index shifting on OP_ReadCookie */
   rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, pOp->p3);
   if( pOp->p2==BTREE_SCHEMA_VERSION ){
     /* When the schema cookie changes, record the new cookie internally */
+    assert( pOp->p1==1 || db->bConcurrent==0 );
     pDb->pSchema->schema_cookie = pOp->p3;
     db->mDbFlags |= DBFLAG_SchemaChange;
   }else if( pOp->p2==BTREE_FILE_FORMAT ){
@@ -6545,6 +6576,14 @@ case OP_Expire: {
 */
 case OP_TableLock: {
   u8 isWriteLock = (u8)pOp->p3;
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( isWriteLock && db->bConcurrent && pOp->p2==1 && pOp->p1!=1 ){
+    rc = SQLITE_ERROR;
+    sqlite3VdbeError(p, 
+        "cannot modify database schema within CONCURRENT transaction");
+    goto abort_due_to_error;
+  }
+#endif
   if( isWriteLock || 0==(db->flags&SQLITE_ReadUncommit) ){
     int p1 = pOp->p1; 
     assert( p1>=0 && p1<db->nDb );

src/vdbeaux.c

@@ -2287,10 +2287,27 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
         assert( i!=1 );
         nTrans++;
       }
-      rc = sqlite3PagerExclusiveLock(pPager);
+      rc = sqlite3BtreeExclusiveLock(pBt);
       sqlite3BtreeLeave(pBt);
     }
   }
+
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( db->bConcurrent && (rc & 0xFF)==SQLITE_BUSY ){
+    /* An SQLITE_BUSY or SQLITE_BUSY_SNAPSHOT was encountered while 
+    ** attempting to take the WRITER lock on a wal file. Release the
+    ** WRITER locks on all wal files and return early.  */
+    for(i=0; i<db->nDb; i++){
+      Btree *pBt = db->aDb[i].pBt;
+      if( sqlite3BtreeIsInTrans(pBt) ){
+        sqlite3BtreeEnter(pBt);
+        sqlite3PagerDropExclusiveLock(sqlite3BtreePager(pBt));
+        sqlite3BtreeLeave(pBt);
+      }
+    }
+  }
+#endif
+
   if( rc!=SQLITE_OK ){
     return rc;
   }
@@ -2686,6 +2703,7 @@ int sqlite3VdbeHalt(Vdbe *p){
           sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK);
           sqlite3CloseSavepoints(db);
           db->autoCommit = 1;
+          db->bConcurrent = 0;
           p->nChange = 0;
         }
       }
@@ -2721,9 +2739,9 @@ int sqlite3VdbeHalt(Vdbe *p){
           ** is required. */
           rc = vdbeCommit(db, p);
         }
-        if( rc==SQLITE_BUSY && p->readOnly ){
+        if( (rc & 0xFF)==SQLITE_BUSY && p->readOnly ){
           sqlite3VdbeLeave(p);
-          return SQLITE_BUSY;
+          return rc;
         }else if( rc!=SQLITE_OK ){
           p->rc = rc;
           sqlite3RollbackAll(db, SQLITE_OK);
@@ -2748,6 +2766,7 @@ int sqlite3VdbeHalt(Vdbe *p){
         sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK);
         sqlite3CloseSavepoints(db);
         db->autoCommit = 1;
+        db->bConcurrent = 0;
         p->nChange = 0;
       }
     }
@@ -2769,6 +2788,7 @@ int sqlite3VdbeHalt(Vdbe *p){
         sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK);
         sqlite3CloseSavepoints(db);
         db->autoCommit = 1;
+        db->bConcurrent = 0;
         p->nChange = 0;
       }
     }

src/wal.c

@@ -459,6 +459,7 @@ struct Wal {
   WalIndexHdr hdr;           /* Wal-index header for current transaction */
   u32 minFrame;              /* Ignore wal frames before this one */
   u32 iReCksum;              /* On commit, recalculate checksums from here */
+  u32 nPriorFrame;           /* For sqlite3WalInfo() */
   const char *zWalName;      /* Name of WAL file */
   u32 nCkpt;                 /* Checkpoint sequence counter in the wal-header */
 #ifdef SQLITE_DEBUG
@@ -828,7 +829,7 @@ static const char *walLockName(int lockIdx){
 /*
 ** Set or release locks on the WAL.  Locks are either shared or exclusive.
 ** A lock cannot be moved directly between shared and exclusive - it must go
-** through the unlocked state first.
+** through the concurrent state first.
 **
 ** In locking_mode=EXCLUSIVE, all of these routines become no-ops.
 */
@@ -1126,7 +1127,7 @@ static int walIndexRecover(Wal *pWal){
   /* Obtain an exclusive lock on all byte in the locking range not already
   ** locked by the caller. The caller is guaranteed to have locked the
   ** WAL_WRITE_LOCK byte, and may have also locked the WAL_CKPT_LOCK byte.
-  ** If successful, the same bytes that are locked here are unlocked before
+  ** If successful, the same bytes that are locked here are concurrent before
   ** this function returns.
   */
   assert( pWal->ckptLock==1 || pWal->ckptLock==0 );
@@ -2029,6 +2030,49 @@ int sqlite3WalClose(
   return rc;
 }
 
+/*
+** Try to copy the wal-index header from shared-memory into (*pHdr). Return
+** zero if successful or non-zero otherwise. If the header is corrupted
+** (either because the two copies are inconsistent or because the checksum 
+** values are incorrect), the read fails and non-zero is returned.
+*/
+static int walIndexLoadHdr(Wal *pWal, WalIndexHdr *pHdr){
+  u32 aCksum[2];                  /* Checksum on the header content */
+  WalIndexHdr h2;                 /* Second copy of the header content */
+  WalIndexHdr volatile *aHdr;     /* Header in shared memory */
+
+  /* The first page of the wal-index must be mapped at this point. */
+  assert( pWal->nWiData>0 && pWal->apWiData[0] );
+
+  /* Read the header. This might happen concurrently with a write to the
+  ** same area of shared memory on a different CPU in a SMP,
+  ** meaning it is possible that an inconsistent snapshot is read
+  ** from the file. If this happens, return non-zero.
+  **
+  ** There are two copies of the header at the beginning of the wal-index.
+  ** When reading, read [0] first then [1].  Writes are in the reverse order.
+  ** Memory barriers are used to prevent the compiler or the hardware from
+  ** reordering the reads and writes.
+  */
+  aHdr = walIndexHdr(pWal);
+  memcpy(pHdr, (void *)&aHdr[0], sizeof(h2));
+  walShmBarrier(pWal);
+  memcpy(&h2, (void *)&aHdr[1], sizeof(h2));
+
+  if( memcmp(&h2, pHdr, sizeof(h2))!=0 ){
+    return 1;   /* Dirty read */
+  }  
+  if( h2.isInit==0 ){
+    return 1;   /* Malformed header - probably all zeros */
+  }
+  walChecksumBytes(1, (u8*)&h2, sizeof(h2)-sizeof(h2.aCksum), 0, aCksum);
+  if( aCksum[0]!=h2.aCksum[0] || aCksum[1]!=h2.aCksum[1] ){
+    return 1;   /* Checksum does not match */
+  }
+
+  return 0;
+}
+
 /*
 ** Try to read the wal-index header.  Return 0 on success and 1 if
 ** there is a problem.
@@ -2047,37 +2091,10 @@ int sqlite3WalClose(
 ** is read successfully and the checksum verified, return zero.
 */
 static int walIndexTryHdr(Wal *pWal, int *pChanged){
-  u32 aCksum[2];                  /* Checksum on the header content */
-  WalIndexHdr h1, h2;             /* Two copies of the header content */
-  WalIndexHdr volatile *aHdr;     /* Header in shared memory */
+  WalIndexHdr h1;                 /* Copy of the header content */
 
-  /* The first page of the wal-index must be mapped at this point. */
-  assert( pWal->nWiData>0 && pWal->apWiData[0] );
-
-  /* Read the header. This might happen concurrently with a write to the
-  ** same area of shared memory on a different CPU in a SMP,
-  ** meaning it is possible that an inconsistent snapshot is read
-  ** from the file. If this happens, return non-zero.
-  **
-  ** There are two copies of the header at the beginning of the wal-index.
-  ** When reading, read [0] first then [1].  Writes are in the reverse order.
-  ** Memory barriers are used to prevent the compiler or the hardware from
-  ** reordering the reads and writes.
-  */
-  aHdr = walIndexHdr(pWal);
-  memcpy(&h1, (void *)&aHdr[0], sizeof(h1));
-  walShmBarrier(pWal);
-  memcpy(&h2, (void *)&aHdr[1], sizeof(h2));
-
-  if( memcmp(&h1, &h2, sizeof(h1))!=0 ){
-    return 1;   /* Dirty read */
-  }  
-  if( h1.isInit==0 ){
-    return 1;   /* Malformed header - probably all zeros */
-  }
-  walChecksumBytes(1, (u8*)&h1, sizeof(h1)-sizeof(h1.aCksum), 0, aCksum);
-  if( aCksum[0]!=h1.aCksum[0] || aCksum[1]!=h1.aCksum[1] ){
-    return 1;   /* Checksum does not match */
+  if( walIndexLoadHdr(pWal, &h1) ){
+    return 1;
   }
 
   if( memcmp(&pWal->hdr, &h1, sizeof(WalIndexHdr)) ){
@@ -2746,6 +2763,7 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){
   testcase( rc==SQLITE_PROTOCOL );
   testcase( rc==SQLITE_OK );
 
+  pWal->nPriorFrame = pWal->hdr.mxFrame;
 #ifdef SQLITE_ENABLE_SNAPSHOT
   if( rc==SQLITE_OK ){
     if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){
@@ -2825,39 +2843,20 @@ void sqlite3WalEndReadTransaction(Wal *pWal){
 }
 
 /*
-** Search the wal file for page pgno. If found, set *piRead to the frame that
-** contains the page. Otherwise, if pgno is not in the wal file, set *piRead
-** to zero.
-**
-** Return SQLITE_OK if successful, or an error code if an error occurs. If an
-** error does occur, the final value of *piRead is undefined.
+** Search the hash tables for an entry matching page number pgno. Ignore
+** any entries that lie after frame iLast within the wal file.
 */
-int sqlite3WalFindFrame(
-  Wal *pWal,                      /* WAL handle */
-  Pgno pgno,                      /* Database page number to read data for */
-  u32 *piRead                     /* OUT: Frame number (or zero) */
+static int walFindFrame(
+  Wal *pWal, 
+  Pgno pgno, 
+  u32 iLast, 
+  u32 *piRead
 ){
-  u32 iRead = 0;                  /* If !=0, WAL frame to return data from */
-  u32 iLast = pWal->hdr.mxFrame;  /* Last page in WAL for this reader */
   int iHash;                      /* Used to loop through N hash tables */
+  u32 iRead = 0;
   int iMinHash;
 
-  /* This routine is only be called from within a read transaction. */
-  assert( pWal->readLock>=0 || pWal->lockError );
-
-  /* If the "last page" field of the wal-index header snapshot is 0, then
-  ** no data will be read from the wal under any circumstances. Return early
-  ** in this case as an optimization.  Likewise, if pWal->readLock==0, 
-  ** then the WAL is ignored by the reader so return early, as if the 
-  ** WAL were empty.
-  */
-  if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){
-    *piRead = 0;
-    return SQLITE_OK;
-  }
-
-  /* Search the hash table or tables for an entry matching page number
-  ** pgno. Each iteration of the following for() loop searches one
+  /* Each iteration of the following for() loop searches one
   ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames).
   **
   ** This code might run concurrently to the code in walIndexAppend()
@@ -2908,11 +2907,48 @@ int sqlite3WalFindFrame(
     if( iRead ) break;
   }
 
+  *piRead = iRead;
+  return SQLITE_OK;
+}
+
+/*
+** Search the wal file for page pgno. If found, set *piRead to the frame that
+** contains the page. Otherwise, if pgno is not in the wal file, set *piRead
+** to zero.
+**
+** Return SQLITE_OK if successful, or an error code if an error occurs. If an
+** error does occur, the final value of *piRead is undefined.
+*/
+int sqlite3WalFindFrame(
+  Wal *pWal,                      /* WAL handle */
+  Pgno pgno,                      /* Database page number to read data for */
+  u32 *piRead                     /* OUT: Frame number (or zero) */
+){
+  u32 iRead = 0;                  /* If !=0, WAL frame to return data from */
+  u32 iLast = pWal->hdr.mxFrame;  /* Last page in WAL for this reader */
+  int rc;
+
+  /* This routine is only be called from within a read transaction. */
+  assert( pWal->readLock>=0 || pWal->lockError );
+
+  /* If the "last page" field of the wal-index header snapshot is 0, then
+  ** no data will be read from the wal under any circumstances. Return early
+  ** in this case as an optimization.  Likewise, if pWal->readLock==0, 
+  ** then the WAL is ignored by the reader so return early, as if the 
+  ** WAL were empty.
+  */
+  if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){
+    *piRead = 0;
+    return SQLITE_OK;
+  }
+
+  rc = walFindFrame(pWal, pgno, iLast, &iRead);
+
 #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT
   /* If expensive assert() statements are available, do a linear search
   ** of the wal-index file content. Make sure the results agree with the
   ** result obtained using the hash indexes above.  */
-  {
+  if( rc==SQLITE_OK ){
     u32 iRead2 = 0;
     u32 iTest;
     assert( pWal->bShmUnreliable || pWal->minFrame>0 );
@@ -2927,7 +2963,7 @@ int sqlite3WalFindFrame(
 #endif
 
   *piRead = iRead;
-  return SQLITE_OK;
+  return rc;
 }
 
 /*
@@ -2962,6 +2998,35 @@ Pgno sqlite3WalDbsize(Wal *pWal){
   return 0;
 }
 
+/*
+** Take the WRITER lock on the WAL file. Return SQLITE_OK if successful,
+** or an SQLite error code otherwise. This routine does not invoke any
+** busy-handler callbacks, that is done at a higher level.
+*/
+static int walWriteLock(Wal *pWal){
+  int rc;
+
+  /* Cannot start a write transaction without first holding a read lock */
+  assert( pWal->readLock>=0 );
+  assert( pWal->writeLock==0 );
+  assert( pWal->iReCksum==0 );
+
+  /* If this is a read-only connection, obtaining a write-lock is not
+  ** possible. In this case return SQLITE_READONLY. Otherwise, attempt
+  ** to grab the WRITER lock. Set Wal.writeLock to true and return
+  ** SQLITE_OK if successful, or leave Wal.writeLock clear and return 
+  ** an SQLite error code (possibly SQLITE_BUSY) otherwise. */
+  if( pWal->readOnly ){
+    rc = SQLITE_READONLY;
+  }else{
+    rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1);
+    if( rc==SQLITE_OK ){
+      pWal->writeLock = 1;
+    }
+  }
+
+  return rc;
+}
 
 /* 
 ** This function starts a write transaction on the WAL.
@@ -2977,39 +3042,205 @@ Pgno sqlite3WalDbsize(Wal *pWal){
 ** There can only be a single writer active at a time.
 */
 int sqlite3WalBeginWriteTransaction(Wal *pWal){
-  int rc;
-
-  /* Cannot start a write transaction without first holding a read
-  ** transaction. */
-  assert( pWal->readLock>=0 );
-  assert( pWal->writeLock==0 && pWal->iReCksum==0 );
-
-  if( pWal->readOnly ){
-    return SQLITE_READONLY;
+  int rc = walWriteLock(pWal);
+  if( rc==SQLITE_OK ){
+    /* If another connection has written to the database file since the
+    ** time the read transaction on this connection was started, then
+    ** the write is disallowed. Release the WRITER lock and return
+    ** SQLITE_BUSY_SNAPSHOT in this case.  */
+    if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){
+      walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1);
+      pWal->writeLock = 0;
+      rc = SQLITE_BUSY_SNAPSHOT;
+    }
   }
-
-  /* Only one writer allowed at a time.  Get the write lock.  Return
-  ** SQLITE_BUSY if unable.
-  */
-  rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1);
-  if( rc ){
-    return rc;
-  }
-  pWal->writeLock = 1;
-
-  /* If another connection has written to the database file since the
-  ** time the read transaction on this connection was started, then
-  ** the write is disallowed.
-  */
-  if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){
-    walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1);
-    pWal->writeLock = 0;
-    rc = SQLITE_BUSY_SNAPSHOT;
-  }
-
   return rc;
 }
 
+/*
+** This function is called by a writer that has a read-lock on aReadmark[0]
+** (pWal->readLock==0). This function relinquishes that lock and takes a
+** lock on a different aReadmark[] slot. 
+**
+** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
+*/
+static int walUpgradeReadlock(Wal *pWal){
+  int cnt;
+  int rc;
+  assert( pWal->writeLock && pWal->readLock==0 );
+  walUnlockShared(pWal, WAL_READ_LOCK(0));
+  pWal->readLock = -1;
+  cnt = 0;
+  do{
+    int notUsed;
+    rc = walTryBeginRead(pWal, &notUsed, 1, ++cnt);
+  }while( rc==WAL_RETRY );
+  assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */
+  testcase( (rc&0xff)==SQLITE_IOERR );
+  testcase( rc==SQLITE_PROTOCOL );
+  testcase( rc==SQLITE_OK );
+  return rc;
+}
+
+
+#ifndef SQLITE_OMIT_CONCURRENT
+/* 
+** This function is only ever called when committing a "BEGIN CONCURRENT"
+** transaction. It may be assumed that no frames have been written to
+** the wal file. The second parameter is a pointer to the in-memory 
+** representation of page 1 of the database (which may or may not be
+** dirty). The third is a bitvec with a bit set for each page in the
+** database file that was read by the current concurrent transaction.
+**
+** This function performs three tasks:
+**
+**   1) It obtains the WRITER lock on the wal file,
+**
+**   2) It checks that there are no conflicts between the current
+**      transaction and any transactions committed to the wal file since
+**      it was opened, and
+**
+**   3) It ejects any non-dirty pages from the page-cache that have been
+**      written by another client since the CONCURRENT transaction was started
+**      (so as to avoid ending up with an inconsistent cache after the
+**      current transaction is committed).
+**
+** If no error occurs and the caller may proceed with committing the 
+** transaction, SQLITE_OK is returned. SQLITE_BUSY is returned if the WRITER
+** lock cannot be obtained. Or, if the WRITER lock can be obtained but there
+** are conflicts with a committed transaction, SQLITE_BUSY_SNAPSHOT. Finally,
+** if an error (i.e. an OOM condition or IO error), an SQLite error code
+** is returned.
+*/
+int sqlite3WalLockForCommit(
+  Wal *pWal, 
+  PgHdr *pPage1, 
+  Bitvec *pAllRead, 
+  Pgno *piConflict
+){
+  Pager *pPager = pPage1->pPager;
+  int rc = walWriteLock(pWal);
+
+  /* If the database has been modified since this transaction was started,
+  ** check if it is still possible to commit. The transaction can be 
+  ** committed if:
+  **
+  **   a) None of the pages in pList have been modified since the 
+  **      transaction opened, and
+  **
+  **   b) The database schema cookie has not been modified since the
+  **      transaction was started.
+  */
+  if( rc==SQLITE_OK ){
+    WalIndexHdr head;
+
+    if( walIndexLoadHdr(pWal, &head) ){
+      /* This branch is taken if the wal-index header is corrupted. This 
+      ** occurs if some other writer has crashed while committing a 
+      ** transaction to this database since the current concurrent transaction
+      ** was opened.  */
+      rc = SQLITE_BUSY_SNAPSHOT;
+    }else if( memcmp(&pWal->hdr, (void*)&head, sizeof(WalIndexHdr))!=0 ){
+      int iHash;
+      int iLastHash = walFramePage(head.mxFrame);
+      u32 iFirst = pWal->hdr.mxFrame+1;     /* First wal frame to check */
+      if( memcmp(pWal->hdr.aSalt, (u32*)head.aSalt, sizeof(u32)*2) ){
+        assert( pWal->readLock==0 );
+        iFirst = 1;
+      }
+      for(iHash=walFramePage(iFirst); iHash<=iLastHash; iHash++){
+        volatile ht_slot *aHash;
+        volatile u32 *aPgno;
+        u32 iZero;
+
+        rc = walHashGet(pWal, iHash, &aHash, &aPgno, &iZero);
+        if( rc==SQLITE_OK ){
+          u32 i, iMin, iMax;
+          assert( head.mxFrame>=iZero );
+          iMin = (iZero >= iFirst) ? 1 : (iFirst - iZero);
+          iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE;
+          if( iMax>(head.mxFrame-iZero) ) iMax = (head.mxFrame-iZero);
+          for(i=iMin; rc==SQLITE_OK && i<=iMax; i++){
+            PgHdr *pPg;
+            if( aPgno[i]==1 ){
+              /* Check that the schema cookie has not been modified. If
+              ** it has not, the commit can proceed. */
+              u8 aNew[4];
+              u8 *aOld = &((u8*)pPage1->pData)[40];
+              int sz;
+              i64 iOffset;
+              sz = pWal->hdr.szPage;
+              sz = (sz&0xfe00) + ((sz&0x0001)<<16);
+              iOffset = walFrameOffset(i+iZero, sz) + WAL_FRAME_HDRSIZE + 40;
+              rc = sqlite3OsRead(pWal->pWalFd, aNew, sizeof(aNew), iOffset);
+              if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){
+                rc = SQLITE_BUSY_SNAPSHOT;
+              }
+            }else if( sqlite3BitvecTestNotNull(pAllRead, aPgno[i]) ){
+              *piConflict = aPgno[i];
+              rc = SQLITE_BUSY_SNAPSHOT;
+            }else if( (pPg = sqlite3PagerLookup(pPager, aPgno[i])) ){
+              /* Page aPgno[i], which is present in the pager cache, has been
+              ** modified since the current CONCURRENT transaction was started.
+              ** However it was not read by the current transaction, so is not
+              ** a conflict. There are two possibilities: (a) the page was
+              ** allocated at the of the file by the current transaction or 
+              ** (b) was present in the cache at the start of the transaction.
+              **
+              ** For case (a), do nothing. This page will be moved within the
+              ** database file by the commit code to avoid the conflict. The
+              ** call to PagerUnref() is to release the reference grabbed by
+              ** the sqlite3PagerLookup() above.  
+              **
+              ** In case (b), drop the page from the cache - otherwise
+              ** following the snapshot upgrade the cache would be inconsistent
+              ** with the database as stored on disk. */
+              if( sqlite3PagerIswriteable(pPg) ){
+                sqlite3PagerUnref(pPg);
+              }else{
+                sqlite3PcacheDrop(pPg);
+              }
+            }
+          }
+        }
+        if( rc!=SQLITE_OK ) break;
+      }
+    }
+  }
+
+  pWal->nPriorFrame = pWal->hdr.mxFrame;
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** This function is called as part of committing an CONCURRENT transaction.
+** It is assumed that sqlite3WalLockForCommit() has already been successfully
+** called and so (a) the WRITER lock is held and (b) it is known that the
+** wal-index-header stored in shared memory is not corrupt.
+**
+** Before returning, this function upgrades the client so that it is 
+** operating on the database snapshot currently at the head of the wal file
+** (even if the CONCURRENT transaction ran against an older snapshot).
+**
+** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
+*/
+int sqlite3WalUpgradeSnapshot(Wal *pWal){
+  int rc = SQLITE_OK;
+  assert( pWal->writeLock );
+  memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr));
+
+  /* If this client has its read-lock on slot aReadmark[0] and the entire
+  ** wal has not been checkpointed, switch it to a different slot. Otherwise
+  ** any reads performed between now and committing the transaction will
+  ** read from the old snapshot - not the one just upgraded to.  */
+  if( pWal->readLock==0 && pWal->hdr.mxFrame!=walCkptInfo(pWal)->nBackfill ){
+    rc = walUpgradeReadlock(pWal);
+  }
+  return rc;
+}
+#endif   /* SQLITE_OMIT_CONCURRENT */
+
 /*
 ** End a write transaction.  The commit has already been done.  This
 ** routine merely releases the lock.
@@ -3038,7 +3269,7 @@ int sqlite3WalEndWriteTransaction(Wal *pWal){
 */
 int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){
   int rc = SQLITE_OK;
-  if( ALWAYS(pWal->writeLock) ){
+  if( pWal->writeLock ){
     Pgno iMax = pWal->hdr.mxFrame;
     Pgno iFrame;
   
@@ -3077,7 +3308,6 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){
 ** point in the event of a savepoint rollback (via WalSavepointUndo()).
 */
 void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){
-  assert( pWal->writeLock );
   aWalData[0] = pWal->hdr.mxFrame;
   aWalData[1] = pWal->hdr.aFrameCksum[0];
   aWalData[2] = pWal->hdr.aFrameCksum[1];
@@ -3093,7 +3323,7 @@ void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){
 int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){
   int rc = SQLITE_OK;
 
-  assert( pWal->writeLock );
+  assert( pWal->writeLock || aWalData[0]==pWal->hdr.mxFrame );
   assert( aWalData[3]!=pWal->nCkpt || aWalData[0]<=pWal->hdr.mxFrame );
 
   if( aWalData[3]!=pWal->nCkpt ){
@@ -3129,7 +3359,6 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){
 */
 static int walRestartLog(Wal *pWal){
   int rc = SQLITE_OK;
-  int cnt;
 
   if( pWal->readLock==0 ){
     volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
@@ -3150,21 +3379,21 @@ static int walRestartLog(Wal *pWal){
         ** to handle if this transaction is rolled back.  */
         walRestartHdr(pWal, salt1);
         walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
+        pWal->nPriorFrame = 0;
       }else if( rc!=SQLITE_BUSY ){
         return rc;
       }
     }
-    walUnlockShared(pWal, WAL_READ_LOCK(0));
-    pWal->readLock = -1;
-    cnt = 0;
-    do{
-      int notUsed;
-      rc = walTryBeginRead(pWal, &notUsed, 1, ++cnt);
-    }while( rc==WAL_RETRY );
-    assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */
-    testcase( (rc&0xff)==SQLITE_IOERR );
-    testcase( rc==SQLITE_PROTOCOL );
-    testcase( rc==SQLITE_OK );
+
+    /* Regardless of whether or not the wal file was restarted, change the
+    ** read-lock held by this client to a slot other than aReadmark[0]. 
+    ** Clients with a lock on aReadmark[0] read from the database file 
+    ** only - never from the wal file. This means that if a writer holding
+    ** a lock on aReadmark[0] were to commit a transaction but not close the
+    ** read-transaction, subsequent read operations would read directly from
+    ** the database file - ignoring the new pages just appended
+    ** to the wal file. */
+    rc = walUpgradeReadlock(pWal);
   }
   return rc;
 }
@@ -3433,6 +3662,7 @@ int sqlite3WalFrames(
     p->flags |= PGHDR_WAL_APPEND;
   }
 
+
   /* Recalculate checksums within the wal file if required. */
   if( isCommit && pWal->iReCksum ){
     rc = walRewriteChecksums(pWal, iFrame);
@@ -3607,7 +3837,6 @@ int sqlite3WalCheckpoint(
 
   /* Copy data from the log to the database file. */
   if( rc==SQLITE_OK ){
-
     if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){
       rc = SQLITE_CORRUPT_BKPT;
     }else{
@@ -3792,4 +4021,16 @@ sqlite3_file *sqlite3WalFile(Wal *pWal){
   return pWal->pWalFd;
 }
 
+/* 
+** Return the values required by sqlite3_wal_info().
+*/
+int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame){
+  int rc = SQLITE_OK;
+  if( pWal ){
+    *pnFrame = pWal->hdr.mxFrame;
+    *pnPrior = pWal->nPriorFrame;
+  }
+  return rc;
+}
+
 #endif /* #ifndef SQLITE_OMIT_WAL */

src/wal.h

@@ -134,6 +134,15 @@ void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot);
 int sqlite3WalSnapshotRecover(Wal *pWal);
 #endif
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/* Tell the wal layer that we want to commit a concurrent transaction */
+int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPg, Bitvec *pRead, Pgno*);
+
+/* Upgrade the state of the client to take into account changes written
+** by other connections */
+int sqlite3WalUpgradeSnapshot(Wal *pWal);
+#endif /* SQLITE_OMIT_CONCURRENT */
+
 #ifdef SQLITE_ENABLE_ZIPVFS
 /* If the WAL file is not empty, return the number of bytes of content
 ** stored in each frame (i.e. the db page-size when the WAL was created).
@@ -144,5 +153,8 @@ int sqlite3WalFramesize(Wal *pWal);
 /* Return the sqlite3_file object for the WAL file */
 sqlite3_file *sqlite3WalFile(Wal *pWal);
 
+/* sqlite3_wal_info() data */
+int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame);
+
 #endif /* ifndef SQLITE_OMIT_WAL */
 #endif /* SQLITE_WAL_H */

src/where.c

@@ -3353,6 +3353,7 @@ static int whereLoopAddOr(
   for(pTerm=pWC->a; pTerm<pWCEnd && rc==SQLITE_OK; pTerm++){
     if( (pTerm->eOperator & WO_OR)!=0
      && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0 
+     && !ExprHasProperty(pTerm->pExpr, EP_FromJoin)
     ){
       WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc;
       WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm];

test/bc_test1.c

@@ -0,0 +1,556 @@
+/*
+** 2016-05-07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+
+#include <sqlite3.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include "tt3_core.c"
+
+#ifdef USE_OSINST
+# include "../src/test_osinst.c"
+#else
+# define vfslog_time() 0
+#endif
+
+typedef struct Config Config;
+typedef struct ThreadCtx ThreadCtx;
+
+#define THREAD_TIME_INSERT   0
+#define THREAD_TIME_COMMIT   1
+#define THREAD_TIME_ROLLBACK 2
+#define THREAD_TIME_WRITER   3
+#define THREAD_TIME_CKPT     4
+
+struct ThreadCtx {
+  Config *pConfig;
+  Sqlite *pDb;
+  Error *pErr;
+  sqlite3_int64 aTime[5];
+};
+
+struct Config {
+  int nIPT;                       /* --inserts-per-transaction */
+  int nThread;                    /* --threads */
+  int nSecond;                    /* --seconds */
+  int bMutex;                     /* --mutex */
+  int nAutoCkpt;                  /* --autockpt */
+  int bRm;                        /* --rm */
+  int bClearCache;                /* --clear-cache */
+  int nMmap;                      /* mmap limit in MB */
+  char *zFile;
+  int bOsinst;                    /* True to use osinst */
+
+  ThreadCtx *aCtx;                /* Array of size nThread */
+
+  pthread_cond_t cond;
+  pthread_mutex_t mutex;
+  int nCondWait;                  /* Number of threads waiting on hCond */
+  sqlite3_vfs *pVfs;
+};
+
+
+typedef struct VfsWrapperFd VfsWrapperFd;
+struct VfsWrapperFd {
+  sqlite3_file base;              /* Base class */
+  int bWriter;                    /* True if holding shm WRITER lock */
+  int iTid;
+  Config *pConfig;
+  sqlite3_file *pFd;              /* Underlying file descriptor */
+};
+
+/* Methods of the wrapper VFS */
+static int vfsWrapOpen(sqlite3_vfs*, const char*, sqlite3_file*, int, int*);
+static int vfsWrapDelete(sqlite3_vfs*, const char*, int);
+static int vfsWrapAccess(sqlite3_vfs*, const char*, int, int*);
+static int vfsWrapFullPathname(sqlite3_vfs*, const char *, int, char*);
+static void *vfsWrapDlOpen(sqlite3_vfs*, const char*);
+static void vfsWrapDlError(sqlite3_vfs*, int, char*);
+static void (*vfsWrapDlSym(sqlite3_vfs*,void*, const char*))(void);
+static void vfsWrapDlClose(sqlite3_vfs*, void*);
+static int vfsWrapRandomness(sqlite3_vfs*, int, char*);
+static int vfsWrapSleep(sqlite3_vfs*, int);
+static int vfsWrapCurrentTime(sqlite3_vfs*, double*);
+static int vfsWrapGetLastError(sqlite3_vfs*, int, char*);
+static int vfsWrapCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
+static int vfsWrapSetSystemCall(sqlite3_vfs*, const char*, sqlite3_syscall_ptr);
+static sqlite3_syscall_ptr vfsWrapGetSystemCall(sqlite3_vfs*, const char*);
+static const char *vfsWrapNextSystemCall(sqlite3_vfs*, const char*);
+
+/* Methods of wrapper sqlite3_io_methods object (see vfsWrapOpen()) */
+static int vfsWrapClose(sqlite3_file*);
+static int vfsWrapRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
+static int vfsWrapWrite(sqlite3_file*, const void*, int iAmt, sqlite3_int64);
+static int vfsWrapTruncate(sqlite3_file*, sqlite3_int64 size);
+static int vfsWrapSync(sqlite3_file*, int flags);
+static int vfsWrapFileSize(sqlite3_file*, sqlite3_int64 *pSize);
+static int vfsWrapLock(sqlite3_file*, int);
+static int vfsWrapUnlock(sqlite3_file*, int);
+static int vfsWrapCheckReservedLock(sqlite3_file*, int *pResOut);
+static int vfsWrapFileControl(sqlite3_file*, int op, void *pArg);
+static int vfsWrapSectorSize(sqlite3_file*);
+static int vfsWrapDeviceCharacteristics(sqlite3_file*);
+static int vfsWrapShmMap(sqlite3_file*, int iPg, int, int, void volatile**);
+static int vfsWrapShmLock(sqlite3_file*, int offset, int n, int flags);
+static void vfsWrapShmBarrier(sqlite3_file*);
+static int vfsWrapShmUnmap(sqlite3_file*, int deleteFlag);
+static int vfsWrapFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **);
+static int vfsWrapUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
+
+static int vfsWrapOpen(
+  sqlite3_vfs *pVfs, 
+  const char *zName, 
+  sqlite3_file *pFd, 
+  int flags, 
+  int *fout
+){
+  static sqlite3_io_methods methods = {
+    3,
+    vfsWrapClose, vfsWrapRead, vfsWrapWrite,
+    vfsWrapTruncate, vfsWrapSync, vfsWrapFileSize,
+    vfsWrapLock, vfsWrapUnlock, vfsWrapCheckReservedLock,
+    vfsWrapFileControl, vfsWrapSectorSize, vfsWrapDeviceCharacteristics,
+    vfsWrapShmMap, vfsWrapShmLock, vfsWrapShmBarrier,
+    vfsWrapShmUnmap, vfsWrapFetch, vfsWrapUnfetch
+  };
+
+  Config *pConfig = (Config*)pVfs->pAppData;
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  int rc;
+
+  memset(pWrapper, 0, sizeof(VfsWrapperFd));
+  if( flags & SQLITE_OPEN_MAIN_DB ){
+    pWrapper->iTid = (int)sqlite3_uri_int64(zName, "tid", 0);
+  }
+
+  pWrapper->pFd = (sqlite3_file*)&pWrapper[1];
+  pWrapper->pConfig = pConfig;
+  rc = pConfig->pVfs->xOpen(pConfig->pVfs, zName, pWrapper->pFd, flags, fout);
+  if( rc==SQLITE_OK ){
+    pWrapper->base.pMethods = &methods;
+  }
+  return rc;
+}
+
+static int vfsWrapDelete(sqlite3_vfs *pVfs, const char *a, int b){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xDelete(pConfig->pVfs, a, b);
+}
+static int vfsWrapAccess(sqlite3_vfs *pVfs, const char *a, int b, int *c){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xAccess(pConfig->pVfs, a, b, c);
+}
+static int vfsWrapFullPathname(sqlite3_vfs *pVfs, const char *a, int b, char*c){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xFullPathname(pConfig->pVfs, a, b, c);
+}
+static void *vfsWrapDlOpen(sqlite3_vfs *pVfs, const char *a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xDlOpen(pConfig->pVfs, a);
+}
+static void vfsWrapDlError(sqlite3_vfs *pVfs, int a, char *b){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xDlError(pConfig->pVfs, a, b);
+}
+static void (*vfsWrapDlSym(sqlite3_vfs *pVfs, void *a, const char *b))(void){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xDlSym(pConfig->pVfs, a, b);
+}
+static void vfsWrapDlClose(sqlite3_vfs *pVfs, void *a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xDlClose(pConfig->pVfs, a);
+}
+static int vfsWrapRandomness(sqlite3_vfs *pVfs, int a, char *b){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xRandomness(pConfig->pVfs, a, b);
+}
+static int vfsWrapSleep(sqlite3_vfs *pVfs, int a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xSleep(pConfig->pVfs, a);
+}
+static int vfsWrapCurrentTime(sqlite3_vfs *pVfs, double *a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xCurrentTime(pConfig->pVfs, a);
+}
+static int vfsWrapGetLastError(sqlite3_vfs *pVfs, int a, char *b){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xGetLastError(pConfig->pVfs, a, b);
+}
+static int vfsWrapCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xCurrentTimeInt64(pConfig->pVfs, a);
+}
+static int vfsWrapSetSystemCall(
+  sqlite3_vfs *pVfs, 
+  const char *a, 
+  sqlite3_syscall_ptr b
+){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xSetSystemCall(pConfig->pVfs, a, b);
+}
+static sqlite3_syscall_ptr vfsWrapGetSystemCall(
+  sqlite3_vfs *pVfs, 
+  const char *a
+){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xGetSystemCall(pConfig->pVfs, a);
+}
+static const char *vfsWrapNextSystemCall(sqlite3_vfs *pVfs, const char *a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xNextSystemCall(pConfig->pVfs, a);
+}
+
+static int vfsWrapClose(sqlite3_file *pFd){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  pWrapper->pFd->pMethods->xClose(pWrapper->pFd);
+  pWrapper->pFd = 0;
+  return SQLITE_OK;
+}
+static int vfsWrapRead(sqlite3_file *pFd, void *a, int b, sqlite3_int64 c){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xRead(pWrapper->pFd, a, b, c);
+}
+static int vfsWrapWrite(
+  sqlite3_file *pFd, 
+  const void *a, int b, 
+  sqlite3_int64 c
+){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xWrite(pWrapper->pFd, a, b, c);
+}
+static int vfsWrapTruncate(sqlite3_file *pFd, sqlite3_int64 a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xTruncate(pWrapper->pFd, a);
+}
+static int vfsWrapSync(sqlite3_file *pFd, int a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xSync(pWrapper->pFd, a);
+}
+static int vfsWrapFileSize(sqlite3_file *pFd, sqlite3_int64 *a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xFileSize(pWrapper->pFd, a);
+}
+static int vfsWrapLock(sqlite3_file *pFd, int a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xLock(pWrapper->pFd, a);
+}
+static int vfsWrapUnlock(sqlite3_file *pFd, int a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xUnlock(pWrapper->pFd, a);
+}
+static int vfsWrapCheckReservedLock(sqlite3_file *pFd, int *a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xCheckReservedLock(pWrapper->pFd, a);
+}
+static int vfsWrapFileControl(sqlite3_file *pFd, int a, void *b){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xFileControl(pWrapper->pFd, a, b);
+}
+static int vfsWrapSectorSize(sqlite3_file *pFd){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xSectorSize(pWrapper->pFd);
+}
+static int vfsWrapDeviceCharacteristics(sqlite3_file *pFd){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xDeviceCharacteristics(pWrapper->pFd);
+}
+static int vfsWrapShmMap(
+  sqlite3_file *pFd, 
+  int a, int b, int c, 
+  void volatile **d
+){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xShmMap(pWrapper->pFd, a, b, c, d);
+}
+static int vfsWrapShmLock(sqlite3_file *pFd, int offset, int n, int flags){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  Config *pConfig = pWrapper->pConfig;
+  int bMutex = 0;
+  int rc;
+
+  if(  (offset==0 && n==1)
+    && (flags & SQLITE_SHM_LOCK) && (flags & SQLITE_SHM_EXCLUSIVE)
+  ){
+    pthread_mutex_lock(&pConfig->mutex);
+    pWrapper->bWriter = 1;
+    bMutex = 1;
+    if( pWrapper->iTid ){
+      sqlite3_int64 t = vfslog_time();
+      pConfig->aCtx[pWrapper->iTid-1].aTime[THREAD_TIME_WRITER] -= t;
+    }
+  }
+
+  rc = pWrapper->pFd->pMethods->xShmLock(pWrapper->pFd, offset, n, flags);
+
+  if( (rc!=SQLITE_OK && bMutex)
+   || (offset==0 && (flags & SQLITE_SHM_UNLOCK) && pWrapper->bWriter)
+  ){
+    assert( pWrapper->bWriter );
+    pthread_mutex_unlock(&pConfig->mutex);
+    pWrapper->bWriter = 0;
+    if( pWrapper->iTid ){
+      sqlite3_int64 t = vfslog_time();
+      pConfig->aCtx[pWrapper->iTid-1].aTime[THREAD_TIME_WRITER] += t;
+    }
+  }
+
+  return rc;
+}
+static void vfsWrapShmBarrier(sqlite3_file *pFd){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xShmBarrier(pWrapper->pFd);
+}
+static int vfsWrapShmUnmap(sqlite3_file *pFd, int a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xShmUnmap(pWrapper->pFd, a);
+}
+static int vfsWrapFetch(sqlite3_file *pFd, sqlite3_int64 a, int b, void **c){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xFetch(pWrapper->pFd, a, b, c);
+}
+static int vfsWrapUnfetch(sqlite3_file *pFd, sqlite3_int64 a, void *b){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xUnfetch(pWrapper->pFd, a, b);
+}
+
+static void create_vfs(Config *pConfig){
+  static sqlite3_vfs vfs = {
+    3, 0, 0, 0, "wrapper", 0,
+    vfsWrapOpen, vfsWrapDelete, vfsWrapAccess,
+    vfsWrapFullPathname, vfsWrapDlOpen, vfsWrapDlError,
+    vfsWrapDlSym, vfsWrapDlClose, vfsWrapRandomness,
+    vfsWrapSleep, vfsWrapCurrentTime, vfsWrapGetLastError,
+    vfsWrapCurrentTimeInt64, vfsWrapSetSystemCall, vfsWrapGetSystemCall,
+    vfsWrapNextSystemCall
+  };
+  sqlite3_vfs *pVfs;
+
+  pVfs = sqlite3_vfs_find(0);
+  vfs.mxPathname = pVfs->mxPathname;
+  vfs.szOsFile = pVfs->szOsFile + sizeof(VfsWrapperFd);
+  vfs.pAppData = (void*)pConfig;
+  pConfig->pVfs = pVfs;
+
+  sqlite3_vfs_register(&vfs, 1);
+}
+
+
+/*
+** Wal hook used by connections in thread_main().
+*/
+static int thread_wal_hook(
+  void *pArg,                     /* Pointer to ThreadCtx object */
+  sqlite3 *db,
+  const char *zDb, 
+  int nFrame
+){
+  ThreadCtx *pCtx = (ThreadCtx*)pArg;
+  Config *pConfig = pCtx->pConfig;
+
+  if( pConfig->nAutoCkpt && nFrame>=pConfig->nAutoCkpt ){
+    pCtx->aTime[THREAD_TIME_CKPT] -= vfslog_time();
+    pthread_mutex_lock(&pConfig->mutex);
+    if( pConfig->nCondWait>=0 ){
+      pConfig->nCondWait++;
+      if( pConfig->nCondWait==pConfig->nThread ){
+        execsql(pCtx->pErr, pCtx->pDb, "PRAGMA wal_checkpoint");
+        pthread_cond_broadcast(&pConfig->cond);
+      }else{
+        pthread_cond_wait(&pConfig->cond, &pConfig->mutex);
+      }
+      pConfig->nCondWait--;
+    }
+    pthread_mutex_unlock(&pConfig->mutex);
+    pCtx->aTime[THREAD_TIME_CKPT] += vfslog_time();
+  }
+
+  return SQLITE_OK;
+}
+
+
+static char *thread_main(int iTid, void *pArg){
+  Config *pConfig = (Config*)pArg;
+  Error err = {0};                /* Error code and message */
+  Sqlite db = {0};                /* SQLite database connection */
+  int nAttempt = 0;               /* Attempted transactions */
+  int nCommit = 0;                /* Successful transactions */
+  int j;
+  ThreadCtx *pCtx = &pConfig->aCtx[iTid-1];
+  char *zUri = 0;
+
+#ifdef USE_OSINST
+  char *zOsinstName = 0;
+  char *zLogName = 0;
+  if( pConfig->bOsinst ){
+    zOsinstName = sqlite3_mprintf("osinst%d", iTid);
+    zLogName = sqlite3_mprintf("bc_test1.log.%d.%d", (int)getpid(), iTid);
+    zUri = sqlite3_mprintf(
+        "file:%s?vfs=%s&tid=%d", pConfig->zFile, zOsinstName, iTid
+    );
+    sqlite3_vfslog_new(zOsinstName, 0, zLogName);
+    opendb(&err, &db, zUri, 0);
+  }else
+#endif
+  {
+    zUri = sqlite3_mprintf("file:%s?tid=%d", pConfig->zFile, iTid);
+    opendb(&err, &db, zUri, 0);
+  }
+
+  sqlite3_busy_handler(db.db, 0, 0);
+  sql_script_printf(&err, &db, 
+      "PRAGMA wal_autocheckpoint = 0;"
+      "PRAGMA synchronous = 0;"
+      "PRAGMA mmap_size = %lld;",
+      (i64)(pConfig->nMmap) * 1024 * 1024
+  );
+
+  pCtx->pConfig = pConfig;
+  pCtx->pErr = &err;
+  pCtx->pDb = &db;
+  sqlite3_wal_hook(db.db, thread_wal_hook, (void*)pCtx);
+
+  while( !timetostop(&err) ){
+    execsql(&err, &db, "BEGIN CONCURRENT");
+
+    pCtx->aTime[THREAD_TIME_INSERT] -= vfslog_time();
+    for(j=0; j<pConfig->nIPT; j++){
+      execsql(&err, &db, 
+          "INSERT INTO t1 VALUES"
+          "(randomblob(10), randomblob(20), randomblob(30), randomblob(200))"
+      );
+    }
+    pCtx->aTime[THREAD_TIME_INSERT] += vfslog_time();
+
+    pCtx->aTime[THREAD_TIME_COMMIT] -= vfslog_time();
+    execsql(&err, &db, "COMMIT");
+    pCtx->aTime[THREAD_TIME_COMMIT] += vfslog_time();
+
+    pCtx->aTime[THREAD_TIME_ROLLBACK] -= vfslog_time();
+    nAttempt++;
+    if( err.rc==SQLITE_OK ){
+      nCommit++;
+    }else{
+      clear_error(&err, SQLITE_BUSY);
+      execsql(&err, &db, "ROLLBACK");
+    }
+    pCtx->aTime[THREAD_TIME_ROLLBACK] += vfslog_time();
+
+    if( pConfig->bClearCache ){
+      sqlite3_db_release_memory(db.db);
+    }
+  }
+
+  closedb(&err, &db);
+
+#ifdef USE_OSINST
+  if( pConfig->bOsinst ){
+    sqlite3_vfslog_finalize(zOsinstName);
+    sqlite3_free(zOsinstName);
+    sqlite3_free(zLogName);
+  }
+#endif
+  sqlite3_free(zUri);
+
+  pthread_mutex_lock(&pConfig->mutex);
+  pConfig->nCondWait = -1;
+  pthread_cond_broadcast(&pConfig->cond);
+  pthread_mutex_unlock(&pConfig->mutex);
+
+  return sqlite3_mprintf("commits: %d/%d insert: %dms"
+      " commit: %dms"
+      " rollback: %dms" 
+      " writer: %dms"
+      " checkpoint: %dms", 
+      nCommit, nAttempt, 
+      (int)(pCtx->aTime[THREAD_TIME_INSERT]/1000), 
+      (int)(pCtx->aTime[THREAD_TIME_COMMIT]/1000), 
+      (int)(pCtx->aTime[THREAD_TIME_ROLLBACK]/1000),
+      (int)(pCtx->aTime[THREAD_TIME_WRITER]/1000),
+      (int)(pCtx->aTime[THREAD_TIME_CKPT]/1000)
+  );
+}
+
+int main(int argc, const char **argv){
+  Error err = {0};                /* Error code and message */
+  Sqlite db = {0};                /* SQLite database connection */
+  Threadset threads = {0};        /* Test threads */
+  Config conf = {5, 3, 5};
+  int i;
+
+  CmdlineArg apArg[] = {
+    { "-seconds", CMDLINE_INT,  offsetof(Config, nSecond) },
+    { "-inserts", CMDLINE_INT,  offsetof(Config, nIPT) },
+    { "-threads", CMDLINE_INT,  offsetof(Config, nThread) },
+    { "-mutex",   CMDLINE_BOOL, offsetof(Config, bMutex) },
+    { "-rm",      CMDLINE_BOOL, offsetof(Config, bRm) },
+    { "-autockpt",CMDLINE_INT,  offsetof(Config, nAutoCkpt) },
+    { "-mmap",    CMDLINE_INT,  offsetof(Config, nMmap) },
+    { "-clear-cache",    CMDLINE_BOOL,  offsetof(Config, bClearCache) },
+    { "-file",    CMDLINE_STRING,  offsetof(Config, zFile) },
+    { "-osinst",  CMDLINE_BOOL,  offsetof(Config, bOsinst) },
+    { 0, 0, 0 }
+  };
+
+  conf.nAutoCkpt = 1000;
+  cmdline_process(apArg, argc, argv, (void*)&conf);
+  if( err.rc==SQLITE_OK ){
+    char *z = cmdline_construct(apArg, (void*)&conf);
+    printf("With: %s\n", z);
+    sqlite3_free(z);
+  }
+  if( conf.zFile==0 ){
+    conf.zFile = "xyz.db";
+  }
+
+  /* Create the special VFS - "wrapper". And the mutex and condition 
+  ** variable. */
+  create_vfs(&conf);
+  pthread_mutex_init(&conf.mutex, 0);
+  pthread_cond_init(&conf.cond, 0);
+
+  conf.aCtx = sqlite3_malloc(sizeof(ThreadCtx) * conf.nThread);
+  memset(conf.aCtx, 0, sizeof(ThreadCtx) * conf.nThread);
+
+  /* Ensure the schema has been created */
+  opendb(&err, &db, conf.zFile, conf.bRm);
+  sql_script(&err, &db,
+      "PRAGMA journal_mode = wal;"
+      "CREATE TABLE IF NOT EXISTS t1(a PRIMARY KEY, b, c, d) WITHOUT ROWID;"
+      "CREATE INDEX IF NOT EXISTS t1b ON t1(b);"
+      "CREATE INDEX IF NOT EXISTS t1c ON t1(c);"
+  );
+
+  setstoptime(&err, conf.nSecond*1000);
+  if( conf.nThread==1 ){
+    char *z = thread_main(1, (void*)&conf);
+    printf("Thread 0 says: %s\n", (z==0 ? "..." : z));
+    fflush(stdout);
+  }else{
+    for(i=0; i<conf.nThread; i++){
+      launch_thread(&err, &threads, thread_main, (void*)&conf);
+    }
+    join_all_threads(&err, &threads);
+  }
+
+  if( err.rc==SQLITE_OK ){
+    printf("Database is %dK\n", (int)(filesize(&err, conf.zFile) / 1024));
+  }
+  if( err.rc==SQLITE_OK ){
+    char *zWal = sqlite3_mprintf("%s-wal", conf.zFile);
+    printf("Wal file is %dK\n", (int)(filesize(&err, zWal) / 1024));
+  }
+
+  closedb(&err, &db);
+  print_and_free_err(&err);
+  return 0;
+}

test/concfault.test

@@ -0,0 +1,86 @@
+# 2015 Aug 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains fault injection tests designed to test the concurrent
+# transactions feature.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set testprefix concfault
+
+# This test will not work with an in-memory journal, as the database will
+# become corrupt if an error is injected into a transaction after it starts
+# writing data out to the db file.
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+do_test 1-pre1 {
+  execsql {
+    PRAGMA journal_mode = wal;
+    CREATE TABLE t1(a PRIMARY KEY, b);
+    INSERT INTO t1 VALUES(randomblob(1000), randomblob(100));
+    INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
+    INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
+    INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
+    INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
+    DELETE FROM t1 WHERE rowid%2;
+  }
+  faultsim_save_and_close
+} {}
+
+do_faultsim_test 1.1 -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { 
+    BEGIN CONCURRENT;
+      INSERT INTO t1 VALUES(randomblob(1000), randomblob(100));
+    COMMIT;
+  }
+} -test {
+  faultsim_test_result {0 {}} 
+  catchsql { ROLLBACK }
+  faultsim_integrity_check
+}
+
+do_faultsim_test 1.2 -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { 
+    BEGIN CONCURRENT;
+      INSERT INTO t1 VALUES(randomblob(1000), randomblob(100));
+    ROLLBACK;
+  }
+} -test {
+  faultsim_test_result {0 {}} 
+  catchsql { ROLLBACK }
+  faultsim_integrity_check
+}
+
+do_faultsim_test 1.3 -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { 
+    BEGIN CONCURRENT;
+      DELETE FROM t1;
+    COMMIT;
+  }
+} -test {
+  faultsim_test_result {0 {}} 
+  catchsql { ROLLBACK }
+  faultsim_integrity_check
+}
+
+finish_test
+

test/concurrent.test

@@ -0,0 +1,678 @@
+# 2015 July 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set ::testprefix concurrent
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal;
+} {wal}
+
+do_execsql_test 1.1 {
+  CREATE TABLE t1(k INTEGER PRIMARY KEY, v);
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(1, 'abcd');
+  COMMIT;
+}
+
+do_execsql_test 1.2 {
+  SELECT * FROM t1;
+} {1 abcd}
+
+do_execsql_test 1.3 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(2, 'efgh');
+  ROLLBACK;
+}
+
+do_execsql_test 1.4 {
+  SELECT * FROM t1;
+} {1 abcd}
+
+
+#-------------------------------------------------------------------------
+# CONCURRENT transactions cannot do cache spills.
+#
+foreach {tn trans spill} {
+  1 {BEGIN CONCURRENT}  0
+  2 {BEGIN}           1
+} {
+  do_test 1.5.$tn {
+    sqlite3 db2 test.db
+    set walsz [file size test.db-wal]
+
+    execsql { PRAGMA cache_size = 10 } db2
+    execsql $trans db2
+    execsql {
+      WITH cnt(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM cnt WHERE i<50)
+        INSERT INTO t1(v) SELECT randomblob(900) FROM cnt;
+    } db2
+
+    expr {[file size test.db-wal]==$walsz}
+  } [expr !$spill]
+
+  execsql ROLLBACK db2
+  db2 close
+}
+
+#-------------------------------------------------------------------------
+# CONCURRENT transactions man not be committed while there are active
+# readers.
+do_execsql_test 1.6.setup {
+  DROP TABLE t1;
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 2);
+  INSERT INTO t1 VALUES(3, 4);
+  INSERT INTO t1 VALUES(5, 6);
+}
+foreach {tn trans commit_ok} {
+  1 {BEGIN CONCURRENT}  0
+  2 {BEGIN}           1
+} {
+  do_test 1.6.$tn.1 {
+    set stmt [sqlite3_prepare db "SELECT * FROM t1" -1 dummy]
+    sqlite3_step $stmt
+  } SQLITE_ROW
+  do_test 1.6.$tn.2 {
+    execsql $trans
+    execsql { INSERT INTO t1 VALUES(7, 8) }
+  } {}
+
+  if { $commit_ok } {
+    do_test 1.6.$tn.3 { catchsql COMMIT } {0 {}}
+  } else {
+    do_test 1.6.$tn.4 { catchsql COMMIT } {/1 {cannot commit transaction .*}/}
+  }
+
+  sqlite3_finalize $stmt
+  catchsql ROLLBACK
+}
+
+#-------------------------------------------------------------------------
+# CONCURRENT transactions may not modify the db schema.
+#
+foreach {tn sql} {
+  1 { CREATE TABLE xx(a, b) }
+  2 { DROP TABLE t1 }
+  3 { CREATE INDEX i1 ON t1(a) }
+  4 { CREATE VIEW v1 AS SELECT * FROM t1 }
+} {
+  do_catchsql_test 1.7.0.$tn.1 "
+    BEGIN CONCURRENT;
+    $sql
+  " {1 {cannot modify database schema within CONCURRENT transaction}}
+
+  do_execsql_test 1.7.0.$tn.2 {
+    SELECT sql FROM sqlite_master;
+    SELECT sql FROM sqlite_temp_master;
+  } {{CREATE TABLE t1(a, b)}}
+
+  do_execsql_test 1.7.0.$tn.3 COMMIT
+}
+
+# Except the temp db schema.
+foreach {tn sql} {
+  1 { CREATE TEMP TABLE xx(a, b) }
+  2 { DROP TABLE xx }
+  3 { CREATE TEMP TABLE yy(a, b) }
+  4 { CREATE VIEW temp.v1 AS SELECT * FROM t1 }
+  5 { CREATE INDEX yyi1 ON yy(a); }
+  6 { CREATE TABLE temp.zz(a, b) }
+} {
+  do_catchsql_test 1.7.1.$tn.1 "
+    BEGIN CONCURRENT;
+    $sql
+  " {0 {}}
+
+  do_execsql_test 1.7.1.$tn.2 COMMIT
+}
+
+
+do_execsql_test 1.7.1.x {
+  SELECT sql FROM sqlite_master;
+  SELECT sql FROM sqlite_temp_master;
+} {
+  {CREATE TABLE t1(a, b)}
+  {CREATE TABLE yy(a, b)} 
+  {CREATE VIEW v1 AS SELECT * FROM t1} 
+  {CREATE INDEX yyi1 ON yy(a)} 
+  {CREATE TABLE zz(a, b)}
+}
+
+#-------------------------------------------------------------------------
+# If an auto-vacuum database is written within an CONCURRENT transaction, it
+# is handled in the same way as for a non-CONCURRENT transaction.
+#
+reset_db
+do_execsql_test 1.8.1 {
+  PRAGMA auto_vacuum = 1;
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x, y);
+  INSERT INTO t1 VALUES('x', 'y');
+} {wal}
+
+do_execsql_test 1.8.2 {
+  BEGIN CONCURRENT;
+    SELECT * FROM t1;
+  COMMIT;
+} {x y}
+
+do_catchsql_test 1.8.3 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES('a', 'b');
+} {0 {}}
+
+do_test 1.8.4 {
+  sqlite3 db2 test.db
+  catchsql {
+    BEGIN CONCURRENT;
+      INSERT INTO t1 VALUES('c', 'd');
+  } db2
+} {1 {database is locked}}
+
+do_test 1.8.5 {
+  db eval COMMIT
+  db2 eval COMMIT
+} {}
+db close
+db2 close
+
+do_multiclient_test tn {
+
+  #-----------------------------------------------------------------------
+  # 1. Start an CONCURRENT transaction using [db1].
+  #
+  # 2. Start and then rollback a regular transaction using [db2]. This 
+  #    can be done as the ongoing [db1] transaction is CONCURRENT.
+  #
+  # 3. The [db1] transaction can now be committed, as [db2] has relinquished
+  #    the write lock.
+  #
+  do_test 2.$tn.1.1 {
+    sql1 { 
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(k INTEGER PRIMARY KEY, v);
+      INSERT INTO t1 VALUES(1, 'one');
+    }
+    sql1 { 
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(2, 'two');
+    }
+    code1 { sqlite3_get_autocommit db }
+  } 0
+
+  do_test 2.$tn.1.2 {
+    sql2 {
+      BEGIN;
+        INSERT INTO t1 VALUES(3, 'three');
+      ROLLBACK;
+    }
+  } {}
+
+  do_test 2.$tn.1.3 {
+    sql1 COMMIT
+    sql2 { SELECT * FROM t1 }
+  } {1 one 2 two}
+  
+  #-----------------------------------------------------------------------
+  # 1. Start an CONCURRENT transaction using [db1].
+  #
+  # 2. Commit a transaction using [db2].
+  #
+  # 3. Try to commit with [db1]. Check that SQLITE_BUSY_SNAPSHOT is returned,
+  #    and the transaction is not rolled back.
+  #
+  do_test 2.$tn.2.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(-1, 'hello world');
+    }
+  } {}
+
+  do_test 2.$tn.2.2 {
+    sql2 {
+      INSERT INTO t1 VALUES(3, 'three');
+    }
+  } {}
+
+  do_test 2.$tn.2.3.1 {
+    set rc [catch { sql1 COMMIT } msg]
+    list $rc $msg
+  } {1 {database is locked}}
+
+  do_test 2.$tn.2.3.2 {
+    code1 { list [sqlite3_extended_errcode db] [sqlite3_get_autocommit db] }
+  } {SQLITE_BUSY_SNAPSHOT 0}
+
+  do_test 2.$tn.2.3.3 {
+    sql1 {
+      SELECT * FROM t1;
+      ROLLBACK;
+    }
+  } {-1 {hello world} 1 one 2 two}
+  
+  #-----------------------------------------------------------------------
+  # 1. Start an CONCURRENT transaction using [db1].
+  #
+  # 2. Open a transaction using [db2].
+  #
+  # 3. Try to commit with [db1]. Check that SQLITE_BUSY is returned,
+  #    and the transaction is not rolled back.
+  #
+  # 4. Have [db2] roll its transaction back. Then check that [db1] can
+  #    commit.
+  #
+  do_test 2.$tn.3.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(4, 'four');
+    }
+  } {}
+
+  do_test 2.$tn.3.2 {
+    sql2 {
+      BEGIN;
+        INSERT INTO t1 VALUES(-1, 'xyz');
+    }
+  } {}
+
+  do_test 2.$tn.3.3.1 {
+    set rc [catch { sql1 COMMIT } msg]
+    list $rc $msg
+  } {1 {database is locked}}
+
+  do_test 2.$tn.3.3.2 {
+    code1 { list [sqlite3_extended_errcode db] [sqlite3_get_autocommit db] }
+  } {SQLITE_BUSY 0}
+
+  do_test 2.$tn.3.3.3 {
+    sql1 { SELECT * FROM t1; }
+  } {1 one 2 two 3 three 4 four}
+
+  do_test 2.$tn.3.4 {
+    sql2 ROLLBACK
+    sql1 COMMIT
+    sql1 { SELECT * FROM t1; }
+  } {1 one 2 two 3 three 4 four}
+
+  #-----------------------------------------------------------------------
+  # 1. Create a second table - t2.
+  #
+  # 2. Write to t1 with [db] and t2 with [db2].
+  #
+  # 3. See if it worked.
+  #
+  do_test 2.$tn.4.1 {
+    sql1 { CREATE TABLE t2(a, b) }
+  } {}
+  do_test 2.$tn.4.2 {
+    sql2 {
+      BEGIN CONCURRENT;
+        INSERT INTO t2 VALUES('i', 'n');
+    }
+
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(5, 'five');
+      COMMIT;
+    }
+
+    sql2 COMMIT
+  } {}
+
+  do_test 2.$tn.4.3.1 {
+    sql2 {SELECT * FROM t1}
+  } {1 one 2 two 3 three 4 four 5 five}
+  do_test 2.$tn.4.3.2 {
+    sql1 {SELECT * FROM t1}
+  } {1 one 2 two 3 three 4 four 5 five}
+
+  do_test 2.$tn.4.3.3 { sql2 {SELECT * FROM t2} } {i n}
+  do_test 2.$tn.4.3.4 { sql1 {SELECT * FROM t2} } {i n}
+
+  #-----------------------------------------------------------------------
+  # The "schema cookie" issue.
+  #
+  # 1. Begin and CONCURRENT write to "t1" using [db]
+  #
+  # 2. Create an index on t1 using [db2].
+  #
+  # 3. Attempt to commit the CONCURRENT write. This is an SQLITE_BUSY_SNAPSHOT,
+  #    even though there is no page collision.
+  #
+  do_test 2.$tn.5.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(6, 'six');
+    }
+  } {}
+
+  do_test 2.$tn.5.2 {
+    sql2 { CREATE INDEX i1 ON t1(v); }
+  } {}
+
+  do_test 2.$tn.5.3 {
+    list [catch { sql1 { COMMIT } } msg] $msg [sqlite3_errcode db]
+  } {1 {database is locked} SQLITE_BUSY_SNAPSHOT}
+
+  do_test 2.$tn.5.4 {
+    sql2 { PRAGMA integrity_check }
+  } {ok}
+  catch { sql1 ROLLBACK }
+
+  #-----------------------------------------------------------------------
+  #
+  # 1. Begin an CONCURRENT write to "t1" using [db]
+  #
+  # 2. Lots of inserts into t2. Enough to grow the db file and modify page 1.
+  #
+  # 3. Check that the CONCURRENT transaction can not be committed.
+  #
+  do_test 2.$tn.6.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(6, 'six');
+    }
+  } {}
+
+  do_test 2.$tn.6.2 {
+    sql2 { 
+      WITH src(a,b) AS (
+        VALUES(1,1) UNION ALL SELECT a+1,b+1 FROM src WHERE a<10000
+      ) INSERT INTO t2 SELECT * FROM src;
+    }
+  } {}
+
+  do_test 2.$tn.6.3 {
+    sql1 { SELECT count(*) FROM t2 }
+    list [catch { sql1 { COMMIT } } msg] $msg [sqlite3_errcode db]
+  } {1 {database is locked} SQLITE_BUSY_SNAPSHOT}
+  sql1 ROLLBACK
+
+  do_test 2.$tn.6.4 {
+    sql1 {
+      SELECT count(*) FROM t1;
+      SELECT count(*) FROM t2;
+    }
+  } {5 10001}
+
+  #-----------------------------------------------------------------------
+  # 
+  # 1. Begin an big CONCURRENT write to "t1" using [db] - large enough to
+  #    grow the db file.
+  #
+  # 2. Lots of inserts into t2. Also enough to grow the db file.
+  #
+  # 3. Check that the CONCURRENT transaction cannot be committed (due to a clash
+  #    on page 1 - the db size field).
+  #
+  do_test 2.$tn.7.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        WITH src(a,b) AS (
+          VALUES(10000,10000) UNION ALL SELECT a+1,b+1 FROM src WHERE a<20000
+        ) INSERT INTO t1 SELECT * FROM src;
+    }
+  } {}
+
+  do_test 2.$tn.7.2 {
+    sql2 { 
+      WITH src(a,b) AS (
+        VALUES(1,1) UNION ALL SELECT a+1,b+1 FROM src WHERE a<10000
+      ) INSERT INTO t2 SELECT * FROM src;
+    }
+  } {}
+
+  do_test 2.$tn.7.3 {
+    list [catch { sql1 { COMMIT } } msg] $msg [sqlite3_errcode db]
+  } {0 {} SQLITE_OK}
+
+  do_test 2.$tn.7.4 { sql3 { PRAGMA integrity_check } } ok
+}
+
+#-------------------------------------------------------------------------
+# Concurrent transactions may not modify the user_version or application_id.
+#
+reset_db
+do_execsql_test 3.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x, y);
+  INSERT INTO t1 VALUES('a', 'b');
+  PRAGMA user_version = 10;
+} {wal}
+do_execsql_test 3.1 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES('c', 'd');
+    SELECT * FROM t1;
+} {a b c d}
+do_catchsql_test 3.2 {
+  PRAGMA user_version = 11;
+} {1 {cannot modify user_version within CONCURRENT transaction}}
+do_execsql_test 3.3 {
+  PRAGMA user_version;
+  SELECT * FROM t1;
+} {10 a b c d}
+do_catchsql_test 3.4 {
+  PRAGMA application_id = 11;
+} {1 {cannot modify application_id within CONCURRENT transaction}}
+do_execsql_test 3.5 {
+  COMMIT;
+  PRAGMA user_version;
+  PRAGMA application_id;
+  SELECT * FROM t1;
+} {10 0 a b c d}
+
+#-------------------------------------------------------------------------
+# However, another transaction modifying the user_version or application_id
+# should not cause a conflict. And committing a concurrent transaction does not
+# clobber the modification - even if the concurrent transaction allocates or
+# frees database pages.
+#
+do_multiclient_test tn {
+  do_test 4.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE ttt(y UNIQUE, z UNIQUE);
+      PRAGMA user_version = 14;
+      BEGIN CONCURRENT;
+        INSERT INTO ttt VALUES('y', 'z');
+    }
+  } {wal}
+  do_test 4.$tn.2 {
+    sql2 { PRAGMA user_version = 16 }
+    sql1 COMMIT
+    sql1 { PRAGMA user_version }
+  } {16}
+
+  do_test 4.$tn.3 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO ttt VALUES(randomblob(10000), randomblob(4));
+        PRAGMA user_version;
+    }
+  } {16}
+  do_test 4.$tn.4 {
+    sql2 { PRAGMA user_version = 1234 }
+    sql1 {
+        PRAGMA user_version;
+      COMMIT;
+      PRAGMA user_version;
+      PRAGMA integrity_check;
+    }
+  } {16 1234 ok}
+
+  do_test 4.$tn.5 {
+    sql1 {
+      BEGIN CONCURRENT;
+        DELETE FROM ttt;
+        PRAGMA user_version;
+    }
+  } {1234}
+  do_test 4.$tn.4 {
+    sql2 { PRAGMA user_version = 5678 }
+    sql1 {
+        PRAGMA user_version;
+      COMMIT;
+      PRAGMA user_version;
+      PRAGMA integrity_check;
+    }
+  } {1234 5678 ok}
+}
+
+do_multiclient_test tn {
+  do_test 5.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE tt(a INTEGER PRIMARY KEY, b);
+      CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
+      INSERT INTO tt VALUES(1, randomblob(400));
+      BEGIN CONCURRENT;
+    }
+  } {wal}
+
+  do_test 5.$tn.2 {
+    sql1 { UPDATE t2 SET b=5 WHERE a=3 }
+    sql2 { INSERT INTO tt VALUES(2, randomblob(6000)) }
+  } {}
+
+  do_test 5.$tn.3 {
+    sql1 { COMMIT }
+  } {}
+}
+
+do_multiclient_test tn {
+  do_test 6.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+      CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
+      INSERT INTO t1 VALUES(1, 'one');
+      INSERT INTO t2 VALUES(2, 'two');
+    }
+  } {wal}
+
+  do_test 6.$tn.2 {
+    sql2 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t2;
+        INSERT INTO t1 VALUES(3, 'three');
+    }
+  } {2 two}
+
+  do_test 6.$tn.3 {
+    sql1 {
+      INSERT INTO t2 VALUES(3, 'three');
+    }
+  } {}
+
+  do_test 6.$tn.2 {
+    list [catch { sql2 { COMMIT } } msg] $msg
+  } {1 {database is locked}}
+}
+
+do_multiclient_test tn {
+  do_test 7.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+      WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100) 
+      INSERT INTO t1 SELECT NULL, randomblob(400) FROM s;
+
+      CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
+      WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<50000) 
+      INSERT INTO t2 SELECT NULL, randomblob(400) FROM s;
+
+      CREATE TABLE t3(a INTEGER PRIMARY KEY, b);
+      WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100) 
+      INSERT INTO t3 SELECT NULL, randomblob(400) FROM s;
+
+      CREATE TABLE t4(a INTEGER PRIMARY KEY, b);
+
+    }
+    set {} {}
+  } {}
+
+  do_test 7.$tn.2 {
+    sql2 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t1;
+        INSERT INTO t4 VALUES(1, 2);
+    }
+    set {} {}
+  } {}
+
+  do_test 7.$tn.3 {
+    sql3 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t3;
+        INSERT INTO t4 VALUES(1, 2);
+    }
+    set {} {}
+  } {}
+
+  do_test 7.$tn.4 {
+    sql1 {
+      UPDATE t1 SET b=randomblob(400);
+      UPDATE t2 SET b=randomblob(400);
+      UPDATE t3 SET b=randomblob(400);
+    }
+  } {}
+
+  do_test 7.$tn.5 {
+    csql2 { COMMIT } 
+  } {1 {database is locked}}
+
+  do_test 7.$tn.6 {
+    csql3 { COMMIT } 
+  } {1 {database is locked}}
+
+
+  csql2 ROLLBACK
+  csql3 ROLLBACK
+
+  # The following test works with $tn==1 (sql2 and sql3 use separate 
+  # processes), but is quite slow. So only run it with $tn==2 (all
+  # connections in the same process).
+  #
+  if {$tn==2} {
+    do_test 7.$tn.7 {
+      for {set i 1} {$i < 10000} {incr i} {
+        sql3 { 
+          PRAGMA wal_checkpoint;
+          BEGIN CONCURRENT;
+          SELECT * FROM t3;
+          INSERT INTO t4 VALUES(1, 2);
+        }
+
+        sql1 {
+          UPDATE t2 SET b = randomblob(400) WHERE rowid <= $i;
+          UPDATE t3 SET b = randomblob(400) WHERE rowid = 1;
+        }
+
+        if {[csql3 COMMIT]!={1 {database is locked}}} {
+          error "Failed at i=$i"
+        }
+        csql3 ROLLBACK
+      }
+    } {}
+  }
+
+}
+
+finish_test

test/concurrent2.test

@@ -0,0 +1,623 @@
+# 2015 July 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Miscellaneous tests for transactions started with BEGIN CONCURRENT. 
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+set ::testprefix concurrent2
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+do_multiclient_test tn {
+
+  do_test 1.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      CREATE TABLE t2(y);
+    }
+  } {wal}
+  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+
+  # Test that an CONCURRENT transaction that allocates/frees no pages does
+  # not conflict with a transaction that does allocate pages.
+  do_test 1.$tn.2  {
+    sql1 { 
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(4);
+    }
+    sql2 {
+      INSERT INTO t2 VALUES(randomblob(1500));
+    }
+    sql1 {
+      COMMIT;
+    }
+  } {}
+  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+  
+  # But that an CONCURRENT transaction does conflict with a transaction
+  # that modifies the db schema.
+  do_test 1.$tn.3  {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(5);
+    }
+    sql2 {
+      CREATE TABLE t3(z);
+    }
+    list [catch { sql1 COMMIT } msg] $msg
+  } {1 {database is locked}}
+  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+  
+  # Test that an CONCURRENT transaction that allocates at least one page 
+  # does not conflict with a transaction that allocates no pages.
+  do_test 1.$tn.4  {
+    sql1 { 
+      ROLLBACK;
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(randomblob(1500));
+    }
+    sql2 {
+      INSERT INTO t2 VALUES(8);
+    }
+    sql1 {
+      COMMIT;
+    }
+  } {}
+
+  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+}
+
+do_multiclient_test tn {
+  do_test 2.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x UNIQUE);
+      CREATE TABLE t2(y UNIQUE);
+    }
+  } {wal}
+
+  do_test 2.$tn.2  {
+    sql1 { 
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(randomblob(1500));
+    }
+    sql2 {
+      INSERT INTO t2 VALUES(randomblob(1500));
+    }
+    sql1 COMMIT
+  } {}
+
+  do_test 2.$tn.3 { sql3 { PRAGMA integrity_check } } {ok}
+
+  do_test 2.$tn.4  {
+    sql1 { 
+      BEGIN CONCURRENT;
+        DELETE FROM t1;
+    }
+    sql2 {
+      DELETE FROM t2;
+    }
+    sql1 COMMIT
+  } {}
+
+  do_test 2.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+
+  do_test 2.$tn.6 {
+    sql1 {
+      INSERT INTO t1 VALUES(randomblob(1500));
+      INSERT INTO t1 VALUES(randomblob(1500));
+      INSERT INTO t2 VALUES(randomblob(1500));
+      DELETE FROM t1 WHERE rowid=1;
+    }
+
+    sql1 {
+      BEGIN CONCURRENT;
+        DELETE FROM t1 WHERE rowid=2;
+    }
+
+    sql2 {
+      DELETE FROM t2;
+    }
+
+    sql1 COMMIT
+  } {}
+
+  do_test 2.$tn.7 { sql3 { PRAGMA integrity_check } } {ok}
+}
+
+#-------------------------------------------------------------------------
+# When an CONCURRENT transaction is opened on a database, the nFree and 
+# iTrunk header fields of the cached version of page 1 are both set 
+# to 0. This allows an CONCURRENT transaction to use its own private 
+# free-page-list, which is merged with the main database free-list when
+# the transaction is committed.
+#
+# The following tests check that nFree/iTrunk are correctly restored if
+# an CONCURRENT transaction is rolled back, and that savepoint rollbacks
+# that occur within CONCURRENT transactions do not incorrectly restore
+# these fields to their on-disk values.
+#
+reset_db
+do_execsql_test 3.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x, y);
+  INSERT INTO t1 VALUES(randomblob(1500), randomblob(1500));
+  DELETE FROM t1;
+} {wal}
+
+do_execsql_test 3.1 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(1, 2);
+  ROLLBACK;
+}
+
+do_execsql_test 3.2 { PRAGMA integrity_check } {ok}
+do_execsql_test 3.3 { PRAGMA freelist_count } {2}
+
+do_execsql_test 3.4.1 {
+  BEGIN CONCURRENT;
+    PRAGMA freelist_count;
+} {2}
+do_execsql_test 3.4.2 {
+  SAVEPOINT xyz;
+    INSERT INTO t1 VALUES(randomblob(1500), NULL);
+    PRAGMA freelist_count;
+} {0}
+do_execsql_test 3.4.3 {
+  ROLLBACK TO xyz;
+} {}
+do_execsql_test 3.4.4 { PRAGMA freelist_count } {0}
+do_execsql_test 3.4.5 { COMMIT; PRAGMA freelist_count } {2}
+do_execsql_test 3.4.6 { PRAGMA integrity_check } {ok}
+
+do_execsql_test 3.5.1 {
+  BEGIN CONCURRENT;
+    UPDATE t1 SET x=randomblob(10) WHERE y=555;
+    PRAGMA freelist_count;
+} {0}
+do_execsql_test 3.5.2 {
+  ROLLBACK;
+  PRAGMA freelist_count;
+} {2}
+do_execsql_test 3.5.3 { PRAGMA integrity_check } {ok}
+
+#-------------------------------------------------------------------------
+# Test that nothing goes wrong if an CONCURRENT transaction allocates a
+# page at the end of the file, frees it within the same transaction, and
+# then has to move the same page to avoid a conflict on COMMIT.
+#
+do_multiclient_test tn {
+  do_test 4.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      CREATE TABLE t2(x);
+    }
+  } {wal}
+
+  do_test 4.$tn.2 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(randomblob(1500));
+        INSERT INTO t1 VALUES(randomblob(1500));
+        DELETE FROM t1 WHERE rowid = 1;
+    }
+
+    sql2 {
+      INSERT INTO t2 VALUES(randomblob(1500));
+      INSERT INTO t2 VALUES(randomblob(1500));
+      INSERT INTO t2 VALUES(randomblob(1500));
+      INSERT INTO t2 VALUES(randomblob(1500));
+      DELETE FROM t2 WHERE rowid IN (1, 2);
+    }
+
+    sql1 COMMIT
+  } {}
+}
+
+#-------------------------------------------------------------------------
+#
+do_multiclient_test tn {
+  do_test 5.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      CREATE TABLE t2(x);
+      INSERT INTO t1 VALUES(randomblob(1500));
+      PRAGMA page_count;
+    }
+  } {wal 4}
+
+  do_test 5.$tn.2 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t2 VALUES(randomblob(1500));
+        PRAGMA page_count;
+    }
+  } {5}
+
+  do_test 5.$tn.3 {
+    sql2 { 
+      DELETE FROM t1;
+      PRAGMA freelist_count;
+      PRAGMA page_count;
+    }
+  } {1 4}
+
+  do_test 5.$tn.4 { sql1 COMMIT } {}
+  do_test 5.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+}
+
+#-------------------------------------------------------------------------
+#
+do_multiclient_test tn {
+  do_test 6.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      INSERT INTO t1 VALUES(randomblob(1500));
+      PRAGMA wal_checkpoint;
+    }
+  } {wal 0 5 5}
+
+  do_test 6.$tn.2 {
+    sql1 { 
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(randomblob(1500));
+        INSERT INTO t1 VALUES(randomblob(1500));
+    }
+  } {}
+
+  do_test 6.$tn.3 {
+    sql2 {
+      BEGIN;
+        INSERT INTO t1 VALUES(randomblob(1500));
+        INSERT INTO t1 VALUES(randomblob(1500));
+      COMMIT;
+    }
+  } {}
+
+  do_test 6.$tn.4 { 
+    list [catch { sql1 COMMIT } msg] $msg
+  } {1 {database is locked}}
+  do_test 6.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+  do_test 6.$tn.5 { sql3 { SELECT count(*) from t1 } } {3}
+}
+
+#-------------------------------------------------------------------------
+# Test that if a corrupt wal-index-header is encountered when attempting
+# to commit a CONCURRENT transaction, the transaction is not committed
+# (or rolled back) and that SQLITE_BUSY_SNAPSHOT is returned to the user.
+#
+catch { db close }
+forcedelete test.db
+testvfs tvfs
+sqlite3 db test.db -vfs tvfs
+do_execsql_test 7.1 {
+  PRAGMA journal_mode = wal;
+  BEGIN;
+    CREATE TABLE t1(a, b, PRIMARY KEY(a));
+    INSERT INTO t1 VALUES(1, 2);
+    INSERT INTO t1 VALUES(3, 4);
+  COMMIT;
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(5, 6);
+    INSERT INTO t1 VALUES(7, 8);
+    SELECT * FROM t1;
+} {wal 1 2 3 4 5 6 7 8}
+
+# Corrupt the wal-index header
+incr_tvfs_hdr test.db 11 1
+
+do_catchsql_test 7.2.1 { COMMIT } {1 {database is locked}}
+do_test 7.2.2 { sqlite3_extended_errcode db } SQLITE_BUSY_SNAPSHOT
+
+do_execsql_test 7.3.1 {
+  SELECT * FROM t1;
+  ROLLBACK;
+} {1 2 3 4 5 6 7 8}
+do_execsql_test 7.3.2 {
+  SELECT * FROM t1;
+} {1 2 3 4}
+
+#-------------------------------------------------------------------------
+# Test that "PRAGMA integrity_check" works within a concurrent 
+# transaction. Within a concurrent transaction, "PRAGMA integrity_check"
+# is unable to detect unused database pages, but can detect other types
+# of corruption.
+#
+reset_db
+do_execsql_test 8.1 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE kv(k INTEGER PRIMARY KEY, v UNIQUE);
+  INSERT INTO kv VALUES(NULL, randomblob(750));
+  INSERT INTO kv SELECT NULL, randomblob(750) FROM kv;
+  INSERT INTO kv SELECT NULL, randomblob(750) FROM kv;
+  INSERT INTO kv SELECT NULL, randomblob(750) FROM kv;
+  INSERT INTO kv SELECT NULL, randomblob(750) FROM kv;
+  INSERT INTO kv SELECT NULL, randomblob(750) FROM kv;
+  DELETE FROM kv WHERE rowid%2;
+  PRAGMA freelist_count;
+} {wal 34}
+do_execsql_test 8.2 { PRAGMA integrity_check } ok
+do_execsql_test 8.3 { 
+  BEGIN CONCURRENT;
+    PRAGMA integrity_check;
+} {ok}
+do_execsql_test 8.4 { 
+    INSERT INTO kv VALUES(1100, 1100);
+    PRAGMA integrity_check;
+} {ok}
+do_execsql_test 8.5 { 
+  COMMIT;
+  PRAGMA integrity_check;
+} {ok}
+
+#-------------------------------------------------------------------------
+# Test that concurrent transactions do not allow foreign-key constraints
+# to be bypassed.
+#
+do_multiclient_test tn {
+  do_test 9.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE pp(i INTEGER PRIMARY KEY, j);
+      CREATE TABLE cc(a, b REFERENCES pp);
+
+      WITH seq(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM seq WHERE i<100)
+      INSERT INTO pp SELECT i, randomblob(1000) FROM seq;
+
+      PRAGMA foreign_keys = 1;
+    }
+  } {wal}
+
+
+  do_test 9.$tn.2.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO cc VALUES(42, 42);
+    }
+  } {}
+  do_test 9.$tn.2.2 {
+    sql2 { DELETE FROM pp WHERE i=42 }
+    list [catch { sql1 COMMIT } msg] $msg
+  } {1 {database is locked}}
+  do_test 9.$tn.2.3 {
+    sql1 ROLLBACK
+  } {}
+
+  do_test 9.$tn.3.1 {
+    sql1 {
+      PRAGMA foreign_keys = 0;
+      BEGIN CONCURRENT;
+        INSERT INTO cc VALUES(43, 43);
+    }
+  } {}
+  do_test 9.$tn.3.2 {
+    sql2 { DELETE FROM pp WHERE i=43 }
+    list [catch { sql1 COMMIT } msg] $msg
+  } {0 {}}
+
+  do_test 9.$tn.4.1 {
+    sql1 {
+      PRAGMA foreign_keys = on;
+      BEGIN CONCURRENT;
+        INSERT INTO cc VALUES(44, 44);
+    }
+  } {}
+  do_test 9.$tn.4.2 {
+    sql2 { DELETE FROM pp WHERE i=1 }
+    list [catch { sql1 COMMIT } msg] $msg
+  } {0 {}}
+}
+
+#-------------------------------------------------------------------------
+# Test that even if a SELECT statement appears before all writes within
+# a CONCURRENT transaction, the pages it reads are still considered when
+# considering whether or not the transaction may be committed.
+#
+do_multiclient_test tn {
+  do_test 10.$tn.1.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(a);
+      CREATE TABLE t2(b);
+      CREATE TABLE t3(c);
+      INSERT INTO t1 VALUES(1), (2), (3);
+      INSERT INTO t2 VALUES(1), (2), (3);
+      INSERT INTO t3 VALUES(1), (2), (3);
+    }
+  } {wal}
+
+  do_test 10.$tn.1.2 {
+    sql1 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t1;
+        INSERT INTO t2 VALUES(4);
+    }
+  } {1 2 3}
+
+  do_test 10.$tn.1.3 {
+    sql2 { INSERT INTO t1 VALUES(4) }
+    list [catch {sql1 COMMIT} msg] $msg
+  } {1 {database is locked}}
+  sql1 ROLLBACK
+
+  # In this case, because the "SELECT * FROM t1" is first stepped before
+  # the "BEGIN CONCURRENT", the pages it reads are not recorded by the
+  # pager object. And so the transaction can be committed. Technically
+  # this behaviour (the effect of an ongoing SELECT on a BEGIN CONCURRENT
+  # transacation) is undefined.
+  #
+  do_test 10.$tn.2.1 {
+    code1 {
+      set ::stmt [sqlite3_prepare db "SELECT * FROM t1" -1 dummy]
+      sqlite3_step $::stmt
+    }
+  } {SQLITE_ROW}
+  do_test 10.$tn.2.2 {
+    sql1 {
+      BEGIN CONCURRENT; 
+        INSERT INTO t2 VALUES(4);
+    }
+    code1 {
+      set res [list]
+      lappend res [sqlite3_column_int $::stmt 0]
+      while {[sqlite3_step $::stmt]=="SQLITE_ROW"} {
+        lappend res [sqlite3_column_int $::stmt 0]
+      }
+      sqlite3_finalize $::stmt
+      set res
+    }
+  } {1 2 3 4}
+  do_test 10.$tn.2.3 {
+    sql2 { INSERT INTO t1 VALUES(5) }
+    sql1 COMMIT
+  } {}
+
+  # More tests surrounding long-lived prepared statements and concurrent
+  # transactions.
+  do_test 10.$tn.3.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t1;
+      COMMIT;
+    }
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t2 VALUES(5);
+    }
+    sql2 {
+      INSERT INTO t1 VALUES(5);
+    }
+    sql1 COMMIT
+    sql3 {
+      SELECT * FROM t2;
+    }
+  } {1 2 3 4 5}
+  do_test 10.$tn.3.2 {
+    sql1 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t1;
+      ROLLBACK;
+    }
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t2 VALUES(6);
+    }
+    sql2 {
+      INSERT INTO t1 VALUES(6);
+    }
+    sql1 COMMIT
+    sql3 { SELECT * FROM t2 }
+  } {1 2 3 4 5 6}
+  do_test 10.$tn.3.3 {
+    sql1 { BEGIN CONCURRENT }
+    code1 {
+      set ::stmt [sqlite3_prepare db "SELECT * FROM t1" -1 dummy]
+      sqlite3_step $::stmt
+    }
+    sql1 {
+      INSERT INTO t2 VALUES(7);
+      SELECT * FROM t3;
+      ROLLBACK;
+      BEGIN CONCURRENT;
+    }
+    sql2 { INSERT INTO t3 VALUES(5) }
+    code1 { sqlite3_finalize $::stmt }
+    sql1 {
+      INSERT INTO t2 VALUES(8);
+      COMMIT;
+    }
+  } {}
+}
+
+do_multiclient_test tn {
+  do_test 11.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(a);
+    }
+  } {wal}
+
+  do_test 11.$tn.2 {
+    code1 { sqlite3_wal_info db main }
+  } {0 2}
+
+  do_test 11.$tn.3 {
+    sql1 { INSERT INTO t1 VALUES(1) }
+    code1 { sqlite3_wal_info db main }
+  } {2 3}
+
+  do_test 11.$tn.4 {
+    sql2 { INSERT INTO t1 VALUES(2) }
+    code2 { sqlite3_wal_info db2 main }
+  } {3 4}
+
+  do_test 11.$tn.5 {
+    sql1 { PRAGMA wal_checkpoint }
+    sql2 { INSERT INTO t1 VALUES(3) }
+    code2 { sqlite3_wal_info db2 main }
+  } {0 1}
+}
+
+reset_db
+do_execsql_test 12.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE tx(a INTEGER PRIMARY KEY, b);
+} {wal}
+do_test 12.1 {
+  for {set i 0} {$i < 50} {incr i} {
+    execsql {
+      BEGIN CONCURRENT;
+      INSERT INTO tx(b) VALUES( randomblob( 1200 ) );
+      COMMIT;
+    }
+  }
+  execsql { PRAGMA page_size }
+} {1024}
+do_execsql_test 12.2 {
+  DELETE FROM tx;
+}
+do_test 12.3 {
+  for {set i 0} {$i < 50} {incr i} {
+    execsql {
+      BEGIN CONCURRENT;
+      INSERT INTO tx(b) VALUES( randomblob( 1200 ) );
+      COMMIT;
+    }
+  }
+  execsql { PRAGMA page_size }
+} {1024}
+do_execsql_test 12.4 {
+  DELETE FROM tx;
+}
+do_test 12.5 {
+  execsql { BEGIN CONCURRENT }
+  for {set i 0} {$i < 5000} {incr i} {
+    execsql {
+      INSERT INTO tx(b) VALUES( randomblob( 1200 ) );
+    }
+  }
+  execsql { COMMIT }
+  execsql { PRAGMA page_size }
+} {1024}
+
+
+finish_test

test/concurrent3.test

@@ -0,0 +1,231 @@
+# 2015 July 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests for transactions started with BEGIN CONCURRENT. The tests in this
+# file focus on testing that deferred page allocation works properly.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set ::testprefix concurrent3
+
+if {$AUTOVACUUM} { finish_test ; return }
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+db close
+sqlite3_shutdown
+test_sqlite3_log xLog
+proc xLog {error_code msg} {
+  # puts "$error_code: $msg"
+  # Enable the previous for debugging
+}
+reset_db
+
+proc create_schema {} {
+  db eval {
+    PRAGMA journal_mode = wal;
+
+    CREATE TABLE t1(x, y);
+    CREATE TABLE t2(x, y);
+    CREATE TABLE t3(x, y);
+    CREATE TABLE t4(x, y);
+
+    CREATE INDEX i1 ON t1(y, x);
+    CREATE INDEX i2 ON t2(y, x);
+    CREATE INDEX i3 ON t3(y, x);
+    CREATE INDEX i4 ON t4(y, x);
+  }
+}
+
+proc do_sql_op {iTbl iOp} {
+  set db "db$iTbl"
+
+  switch $iOp {
+    "i" {
+      set sql "
+        WITH cnt(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM cnt WHERE i<10)
+        INSERT INTO t$iTbl SELECT randomblob(800), randomblob(800) FROM cnt;
+      "
+    }
+
+    "d" {
+      set sql "
+        DELETE FROM t$iTbl WHERE rowid IN (
+          SELECT rowid FROM t$iTbl ORDER BY 1 ASC LIMIT 10
+        )
+      "
+    }
+
+    "D" {
+      set sql "
+        DELETE FROM t$iTbl WHERE rowid IN (
+          SELECT rowid FROM t$iTbl o WHERE (
+            SELECT count(*) FROM t$iTbl i WHERE i.rowid<o.rowid
+          ) % 2
+        )
+      "
+    }
+
+    "I" {
+      set sql "
+        INSERT INTO t$iTbl SELECT randomblob(800), randomblob(800) FROM t$iTbl;
+      "
+    }
+
+    default {
+      error "bad iOp parameter: $iOp"
+    }
+  }
+
+  $db eval $sql
+}
+
+
+set DBLIST {db1 db2 db3 db4} 
+
+create_schema
+foreach {tn oplist} {
+  1  {1i   2i   3i   4i} 
+  2  {1iii 2iii 3iii 4iii}
+  3  {1d   2d   3d   4d} 
+  .  -----------------------
+  4  {1i}
+  5  {1d 2i}
+  .  -----------------------
+  6  {1iii 2iii 3iii 4iii}
+  7  {1di  2id  3iii 4ddd}
+  8  {1iii 2iii 3iii 4iii}
+  9  {1D  2II}
+  10 {1I  2D  3I  4D}
+  11 {1III 3dddddd 4III}
+} {
+  if {[string range $oplist 0 0]=="-"} {
+    reset_db
+    create_schema
+    continue
+  }
+  foreach db $DBLIST { sqlite3 $db test.db }
+
+  do_test 1.$tn {
+    foreach db $DBLIST { $db eval "BEGIN CONCURRENT" } 
+
+    foreach op $oplist {
+      set iTbl [string range $op 0 0]
+      foreach char [split [string range $op 1 end] {}] {
+        do_sql_op $iTbl $char
+      }
+    }
+
+    foreach db $DBLIST { $db eval "COMMIT" }
+    db eval {PRAGMA integrity_check}
+  } {ok}
+
+  foreach db $DBLIST { 
+    $db close 
+  }
+}
+
+#-------------------------------------------------------------------------
+#
+proc create_schema2 {} {
+  db eval {
+    PRAGMA journal_mode = wal;
+    CREATE TABLE t1(x INTEGER PRIMARY KEY, y);
+    CREATE INDEX i1 ON t1(y);
+  }
+}
+
+proc randint {nMax} {
+  db eval {SELECT abs(random() % $nMax)}
+}
+
+proc do_sql_op2 {db iOp} {
+  switch -- $iOp {
+    i {
+      # Insert 1 rows.
+      set r [randint 1000000000]
+      set ::rows($r) 1
+      #puts "insert row $r"
+      $db eval { INSERT OR IGNORE INTO t1 VALUES($r, randomblob(50)); }
+    }
+
+    d {
+      # Insert 1 row
+      set keys [array names ::rows]
+      set r [randint [llength $keys]]
+      set rowid [lindex $keys $r]
+      $db eval { DELETE FROM t1 WHERE x=$rowid }
+      unset ::rows($rowid)
+    }
+  }
+}
+
+foreach {tn nRepeat oplist} {
+  - - ----------------------------
+  1 100 { 1iiiiiiiiii }
+  2 100 { 1i 2d }
+  3 100 { 1d 2i }
+  4  50 { 1d 2i 3d }
+  5 500 { 1i 2i 3i 4i }
+  6 500 { 1i 2d 3d 4d }
+} {
+  if {[string range $oplist 0 0]=="-"} {
+    array unset rows
+    reset_db
+    create_schema2
+    continue
+  }
+
+  foreach db $DBLIST { 
+    sqlite3 $db test.db 
+    set stats($db,0) 0
+    set stats($db,1) 0
+  }
+  array unset used
+
+  do_test 2.$tn {
+
+    for {set i 0} {$i < $nRepeat} {incr i} {
+      foreach db $DBLIST { $db eval "BEGIN CONCURRENT" } 
+
+      foreach op $oplist {
+        set iDb [string range $op 0 0]
+        set used(db$iDb) 1
+        foreach char [split [string range $op 1 end] {}] {
+          do_sql_op2 "db$iDb" $char
+        }
+      }
+
+      foreach db $DBLIST { 
+        set rc [catch { $db eval COMMIT } msg]
+        if {$rc} { $db eval ROLLBACK }
+        incr stats($db,$rc)
+      }
+      set res [db eval {PRAGMA integrity_check}]
+      if {$res != "ok"} { puts "after $db $rc: $res" ; after 1000000 }
+    }
+  } {}
+
+  foreach db $DBLIST { 
+    $db close 
+  }
+  # foreach k [lsort [array names used]] {
+  #   puts "$k: $stats($k,0) committed, $stats($k,1) rolled back"
+  # }
+}
+
+
+
+finish_test

test/concurrent4.test

@@ -0,0 +1,194 @@
+# 2017 May 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Miscellaneous tests for transactions started with BEGIN CONCURRENT. 
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+set ::testprefix concurrent4
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x PRIMARY KEY, y UNIQUE);
+  WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s;
+  DELETE FROM t1 WHERE rowid<2;
+} {wal}
+
+do_execsql_test 1.1 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1(rowid, x, y) VALUES(1000, randomblob(3000), randomblob(3000));
+    SAVEPOINT abc;
+    DELETE FROM t1 WHERE rowid = 1000;
+}
+
+do_execsql_test 1.2 { ROLLBACK TO abc }
+do_execsql_test 1.3 { COMMIT }
+do_execsql_test 1.4 { PRAGMA integrity_check } {ok}
+
+do_multiclient_test tn {
+  do_test 2.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(a, b);
+      WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100)
+        INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s;
+
+      CREATE TABLE t2(a, b);
+      WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100)
+        INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM s;
+    }
+
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(randomblob(3000), randomblob(3000));
+    }
+  } {}
+
+  do_test 2.$tn.2 {
+    sql2 {
+      WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10)
+        INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM s;
+    }
+    sql2 {
+      DELETE FROM t2 WHERE rowid<10;
+    }
+  } {}
+
+  do_test 2.$tn.3 {
+    sql1 {
+      COMMIT;
+      PRAGMA integrity_check;
+    }
+  } {ok}
+  do_test 2.$tn.4 {
+    sql2 {
+      PRAGMA integrity_check;
+    }
+  } {ok}
+}
+
+reset_db
+do_execsql_test 3.1 {
+  PRAGMA page_size = 1024;
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t2(x);
+  INSERT INTO t2 VALUES(randomblob(5000));
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+  INSERT INTO t1 VALUES(25, randomblob(104));
+  DELETE FROM t2;
+} {wal}
+
+do_execsql_test 3.2 {
+  BEGIN CONCURRENT;
+    REPLACE INTO t1 VALUES(25, randomblob(1117));
+  COMMIT;
+} {}
+
+#-------------------------------------------------------------------------
+# Test the effect of BEGIN CONCURRENT transactions that consist entirely
+# of read-only statements.
+#
+reset_db
+do_execsql_test 4.0 {
+  PRAGMA page_size = 1024;
+  PRAGMA journal_mode = wal;
+  
+  CREATE TABLE t4(a, b);
+  INSERT INTO t4 VALUES(1, 2);
+  INSERT INTO t4 VALUES(3, 4);
+} {wal}
+ 
+sqlite3 db2 test.db
+do_test 4.1.1 {
+  db eval {
+    BEGIN CONCURRENT;
+      INSERT INTO t4 VALUES(5, 6);
+  }
+
+  db2 eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+    ROLLBACK;
+  }
+} {1 2 3 4}
+
+do_test 4.1.2 {
+  db eval { COMMIT }
+  db2 eval { SELECT * FROM t4 }
+} {1 2 3 4 5 6}
+
+do_test 4.2.1 {
+  db eval {
+    BEGIN CONCURRENT;
+      INSERT INTO t4 VALUES(7, 8);
+  }
+
+  db2 eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+    COMMIT;
+  }
+} {1 2 3 4 5 6}
+
+do_test 4.2.2 {
+  db eval { COMMIT }
+  db2 eval { SELECT * FROM t4 }
+} {1 2 3 4 5 6 7 8}
+
+do_test 4.3 {
+  db2 eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+  }
+
+  db eval {
+    BEGIN CONCURRENT;
+      INSERT INTO t4 VALUES(9, 10);
+    COMMIT;
+  }
+  db2 eval {
+    SELECT * FROM t4;
+    COMMIT;
+  }
+} {1 2 3 4 5 6 7 8}
+
+set sz [file size test.db-wal]
+do_test 4.4.1 {
+  db eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+      SELECT * FROM sqlite_master;
+  }
+
+  db eval COMMIT
+  file size test.db-wal
+} $sz
+do_test 4.4.2 {
+  db eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+      SELECT * FROM sqlite_master;
+    ROLLBACK;
+  }
+  file size test.db-wal
+} $sz
+
+finish_test
+

test/concurrent5.test

@@ -0,0 +1,133 @@
+# 2017 May 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+set ::testprefix concurrent5
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+db close
+sqlite3_shutdown
+test_sqlite3_log [list lappend ::log]
+set ::log [list]
+
+sqlite3 db test.db
+
+proc do_test_conflict_msg {tn msg} {
+  set msg "cannot commit CONCURRENT transaction - [string trim $msg]"
+  uplevel [list do_test $tn {lindex $::log end} $msg]
+}
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x, y);
+  CREATE TABLE t2(c);
+  WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s;
+} {wal}
+
+sqlite3 db2 test.db
+
+do_test 1.1.1 {
+  set ::log [list]
+  db2 eval {
+    BEGIN CONCURRENT;
+      SELECT count(*) FROM t1;
+      INSERT INTO t2 VALUES(10);
+  }
+
+  db eval {
+    INSERT INTO t1 VALUES(randomblob(200), randomblob(200));
+  }
+
+  catchsql COMMIT db2
+} {1 {database is locked}}
+do_test_conflict_msg 1.1.2 {
+  conflict at page 2 (read-only page; part of db table t1)
+}
+
+do_test 1.2.1 {
+  set ::log [list]
+  db2 eval {
+    ROLLBACK;
+    BEGIN CONCURRENT;
+      INSERT INTO t1 VALUES(11, 12);
+  }
+
+  db eval {
+    INSERT INTO t1 VALUES(12, 11);
+  }
+
+  catchsql COMMIT db2
+} {1 {database is locked}}
+
+do_test_conflict_msg 1.2.2 {
+  conflict at page 105 (read/write page; part of db table t1)
+}
+
+do_test 1.3.1 {
+  set ::log [list]
+  db2 eval {
+    ROLLBACK;
+    BEGIN CONCURRENT;
+      INSERT INTO t2 VALUES('x');
+  }
+
+  db eval {
+    INSERT INTO t2 VALUES('y');
+  }
+
+  catchsql COMMIT db2
+} {1 {database is locked}}
+
+do_test_conflict_msg 1.3.2 {
+  conflict at page 3 (read/write page; part of db table t2)
+}
+
+do_test 1.4.1 {
+  set ::log [list]
+
+  execsql {
+    ROLLBACK;
+    CREATE TABLE t3(a INTEGER PRIMARY KEY, b INTEGER);
+    CREATE INDEX i3 ON t3(b);
+
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<5000
+    ) INSERT INTO t3 SELECT i, i FROM s;
+
+    BEGIN CONCURRENT;
+      INSERT INTO t3 VALUES(0, 5001);
+  } db2
+
+  execsql { INSERT INTO t3 VALUES(NULL, 5002) } db
+  catchsql COMMIT db2
+} {1 {database is locked}}
+
+do_test_conflict_msg 1.3.2 {
+  conflict at page 211 (read/write page; part of db index t3.i3)
+}
+
+db close
+db2 close
+sqlite3_shutdown
+test_sqlite3_log 
+sqlite3_initialize
+finish_test
+

test/concurrent6.test

@@ -0,0 +1,60 @@
+# 2017 May 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+set ::testprefix concurrent6
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+sqlite3 db2 test.db
+
+do_execsql_test 1.0 {
+  PRAGMA page_size = 1024;
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(x);
+  CREATE TABLE t3(x);
+  CREATE TABLE t4(x);
+
+  INSERT INTO t1 VALUES(zeroblob(1500));
+} {wal}
+
+do_execsql_test -db db2 1.1 {
+  BEGIN CONCURRENT;
+    INSERT INTO t3 VALUES(zeroblob(4000));
+    DELETE FROM t1;
+}
+
+do_execsql_test 1.2 {
+  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) 
+  INSERT INTO t2 SELECT zeroblob(1000) FROM s;
+
+  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) 
+  INSERT INTO t4  SELECT zeroblob(1000) FROM s;
+
+  DELETE FROM t4;
+}
+
+do_execsql_test -db db2 1.3 {
+  COMMIT;
+}
+
+
+finish_test
+

test/concurrent7.test

@@ -0,0 +1,52 @@
+# 2018 Jan 5
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix concurrent7
+
+sqlite3 db2 test.db
+
+do_execsql_test 1 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(x);
+} {wal}
+
+do_execsql_test -db db2 2 {
+  SELECT * FROM t1;
+}
+
+do_execsql_test 3 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(randomblob(1500));
+    INSERT INTO t1 VALUES(randomblob(1500));
+    DELETE FROM t1 WHERE rowid = 1;
+}
+
+do_execsql_test -db db2 4 {
+  INSERT INTO t2 VALUES(randomblob(1500));
+  INSERT INTO t2 VALUES(randomblob(1500));
+  INSERT INTO t2 VALUES(randomblob(1500));
+  INSERT INTO t2 VALUES(randomblob(1500));
+  DELETE FROM t2 WHERE rowid IN (1, 2);
+}
+
+do_execsql_test 5 {
+  COMMIT;
+  PRAGMA integrity_check;
+} {ok}
+
+finish_test
+
+

test/parser1.test

@@ -1,102 +0,0 @@
-# 2014-08-24
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-# This file implements regression tests for SQLite library.
-# The focus of this script is testing details of the SQL language parser.
-# 
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-do_catchsql_test parser1-1.1 {
-  CREATE TABLE t1(
-    a TEXT PRIMARY KEY,
-    b TEXT,
-    FOREIGN KEY(b COLLATE nocase DESC) REFERENCES t1(a COLLATE binary ASC)
-  );
-} {1 {syntax error after column name "b"}}
-
-
-# Verify that a legacy schema in the sqlite_master file is allowed to have
-# COLLATE, ASC, and DESC keywords on the id list of a FK constraint, and that
-# those keywords are silently ignored.
-#
-do_execsql_test parser1-1.2 {
-  CREATE TABLE t1(
-    a TEXT PRIMARY KEY,
-    b TEXT,
-    FOREIGN KEY(b) REFERENCES t1(a)
-  );
-  INSERT INTO t1 VALUES('abc',NULL),('xyz','abc');
-  PRAGMA writable_schema=on;
-  UPDATE sqlite_master SET sql='CREATE TABLE t1(
-    a TEXT PRIMARY KEY,
-    b TEXT,
-    FOREIGN KEY(b COLLATE nocase) REFERENCES t1(a)
-  )' WHERE name='t1';
-  SELECT name FROM sqlite_master WHERE sql LIKE '%collate%';
-} {t1}
-sqlite3 db2 test.db
-do_test parser1-1.3 {
-  sqlite3 db2 test.db
-  db2 eval {SELECT * FROM t1 ORDER BY 1}
-} {abc {} xyz abc}
-db2 close
-
-do_execsql_test parser1-1.4 {
-  UPDATE sqlite_master SET sql='CREATE TABLE t1(
-    a TEXT PRIMARY KEY,
-    b TEXT,
-    FOREIGN KEY(b ASC) REFERENCES t1(a)
-  )' WHERE name='t1';
-  SELECT name FROM sqlite_master WHERE sql LIKE '%ASC%';
-} {t1}
-sqlite3 db2 test.db
-do_test parser1-1.5 {
-  sqlite3 db2 test.db
-  db2 eval {SELECT * FROM t1 ORDER BY 1}
-} {abc {} xyz abc}
-db2 close
-
-do_catchsql_test parser1-2.1 {
-  WITH RECURSIVE
-    c(x COLLATE binary) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<5)
-  SELECT x FROM c;
-} {1 {syntax error after column name "x"}}
-do_catchsql_test parser1-2.2 {
-  WITH RECURSIVE
-    c(x ASC) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<5)
-  SELECT x FROM c;
-} {1 {syntax error after column name "x"}}
-
-# Verify that the comma between multiple table constraints is
-# optional.
-#
-# The missing comma is technically a syntax error.  But we have to support
-# it because there might be legacy databases that omit the commas in their
-# sqlite_master tables.
-#
-do_execsql_test parser1-3.1 {
-  CREATE TABLE t300(id INTEGER PRIMARY KEY);
-  CREATE TABLE t301(
-    id INTEGER PRIMARY KEY,
-    c1 INTEGER NOT NULL,
-    c2 INTEGER NOT NULL,
-    c3 BOOLEAN NOT NULL DEFAULT 0,
-    FOREIGN KEY(c1) REFERENCES t300(id) ON DELETE CASCADE ON UPDATE RESTRICT
-        /* no comma */
-    FOREIGN KEY(c2) REFERENCES t300(id) ON DELETE CASCADE ON UPDATE RESTRICT
-        /* no comma */
-    UNIQUE(c1, c2)
-  );
-  PRAGMA foreign_key_list(t301);
-} {0 0 t300 c2 id RESTRICT CASCADE NONE 1 0 t300 c1 id RESTRICT CASCADE NONE}
-
-finish_test

test/permutations.test

@@ -89,7 +89,6 @@ foreach f [glob $testdir/*.test] { lappend alltests [file tail $f] }
 foreach f [glob -nocomplain            \
     $testdir/../ext/rtree/*.test       \
     $testdir/../ext/fts5/test/*.test   \
-    $testdir/../ext/expert/*.test      \
     $testdir/../ext/lsm1/test/*.test   \
 ] {
   lappend alltests $f 

test/threadtest3.c

@@ -32,885 +32,20 @@
 ** The exit status is non-zero if any test fails.
 */
 
-/* 
-** The "Set Error Line" macro.
-*/
-#define SEL(e) ((e)->iLine = ((e)->rc ? (e)->iLine : __LINE__))
-
-/* Database functions */
-#define opendb(w,x,y,z)         (SEL(w), opendb_x(w,x,y,z))
-#define closedb(y,z)            (SEL(y), closedb_x(y,z))
-
-/* Functions to execute SQL */
-#define sql_script(x,y,z)       (SEL(x), sql_script_x(x,y,z))
-#define integrity_check(x,y)    (SEL(x), integrity_check_x(x,y))
-#define execsql_i64(x,y,...)    (SEL(x), execsql_i64_x(x,y,__VA_ARGS__))
-#define execsql_text(x,y,z,...) (SEL(x), execsql_text_x(x,y,z,__VA_ARGS__))
-#define execsql(x,y,...)        (SEL(x), (void)execsql_i64_x(x,y,__VA_ARGS__))
-#define sql_script_printf(x,y,z,...) (                \
-    SEL(x), sql_script_printf_x(x,y,z,__VA_ARGS__)    \
-) 
-
-/* Thread functions */
-#define launch_thread(w,x,y,z)     (SEL(w), launch_thread_x(w,x,y,z))
-#define join_all_threads(y,z)      (SEL(y), join_all_threads_x(y,z))
-
-/* Timer functions */
-#define setstoptime(y,z)        (SEL(y), setstoptime_x(y,z))
-#define timetostop(z)           (SEL(z), timetostop_x(z))
-
-/* Report/clear errors. */
-#define test_error(z, ...)      test_error_x(z, sqlite3_mprintf(__VA_ARGS__))
-#define clear_error(y,z)        clear_error_x(y, z)
-
-/* File-system operations */
-#define filesize(y,z)           (SEL(y), filesize_x(y,z))
-#define filecopy(x,y,z)         (SEL(x), filecopy_x(x,y,z))
-
-#define PTR2INT(x) ((int)((intptr_t)x))
-#define INT2PTR(x) ((void*)((intptr_t)x))
-
-/*
-** End of test code/infrastructure interface macros.
-*************************************************************************/
 
 
 
 
 #include <sqlite3.h>
-#include <unistd.h>
-#include <stdio.h>
-#include <pthread.h>
-#include <assert.h>
-#include <sys/types.h> 
-#include <sys/stat.h> 
-#include <string.h>
-#include <fcntl.h>
-#include <errno.h>
 
 #include "test_multiplex.h"
+#include "tt3_core.c"
 
 /* Required to link test_multiplex.c */
 #ifndef SQLITE_OMIT_WSD
 int sqlite3PendingByte = 0x40000000;
 #endif
 
-/*
- * This code implements the MD5 message-digest algorithm.
- * The algorithm is due to Ron Rivest.  This code was
- * written by Colin Plumb in 1993, no copyright is claimed.
- * This code is in the public domain; do with it what you wish.
- *
- * Equivalent code is available from RSA Data Security, Inc.
- * This code has been tested against that, and is equivalent,
- * except that you don't need to include two pages of legalese
- * with every copy.
- *
- * To compute the message digest of a chunk of bytes, declare an
- * MD5Context structure, pass it to MD5Init, call MD5Update as
- * needed on buffers full of bytes, and then call MD5Final, which
- * will fill a supplied 16-byte array with the digest.
- */
-
-/*
- * If compiled on a machine that doesn't have a 32-bit integer,
- * you just set "uint32" to the appropriate datatype for an
- * unsigned 32-bit integer.  For example:
- *
- *       cc -Duint32='unsigned long' md5.c
- *
- */
-#ifndef uint32
-#  define uint32 unsigned int
-#endif
-
-struct MD5Context {
-  int isInit;
-  uint32 buf[4];
-  uint32 bits[2];
-  union {
-    unsigned char in[64];
-    uint32 in32[16];
-  } u;
-};
-typedef struct MD5Context MD5Context;
-
-/*
- * Note: this code is harmless on little-endian machines.
- */
-static void byteReverse (unsigned char *buf, unsigned longs){
-  uint32 t;
-  do {
-    t = (uint32)((unsigned)buf[3]<<8 | buf[2]) << 16 |
-          ((unsigned)buf[1]<<8 | buf[0]);
-    *(uint32 *)buf = t;
-    buf += 4;
-  } while (--longs);
-}
-/* The four core functions - F1 is optimized somewhat */
-
-/* #define F1(x, y, z) (x & y | ~x & z) */
-#define F1(x, y, z) (z ^ (x & (y ^ z)))
-#define F2(x, y, z) F1(z, x, y)
-#define F3(x, y, z) (x ^ y ^ z)
-#define F4(x, y, z) (y ^ (x | ~z))
-
-/* This is the central step in the MD5 algorithm. */
-#define MD5STEP(f, w, x, y, z, data, s) \
-  ( w += f(x, y, z) + data,  w = w<<s | w>>(32-s),  w += x )
-
-/*
- * The core of the MD5 algorithm, this alters an existing MD5 hash to
- * reflect the addition of 16 longwords of new data.  MD5Update blocks
- * the data and converts bytes into longwords for this routine.
- */
-static void MD5Transform(uint32 buf[4], const uint32 in[16]){
-  register uint32 a, b, c, d;
-
-  a = buf[0];
-  b = buf[1];
-  c = buf[2];
-  d = buf[3];
-
-  MD5STEP(F1, a, b, c, d, in[ 0]+0xd76aa478,  7);
-  MD5STEP(F1, d, a, b, c, in[ 1]+0xe8c7b756, 12);
-  MD5STEP(F1, c, d, a, b, in[ 2]+0x242070db, 17);
-  MD5STEP(F1, b, c, d, a, in[ 3]+0xc1bdceee, 22);
-  MD5STEP(F1, a, b, c, d, in[ 4]+0xf57c0faf,  7);
-  MD5STEP(F1, d, a, b, c, in[ 5]+0x4787c62a, 12);
-  MD5STEP(F1, c, d, a, b, in[ 6]+0xa8304613, 17);
-  MD5STEP(F1, b, c, d, a, in[ 7]+0xfd469501, 22);
-  MD5STEP(F1, a, b, c, d, in[ 8]+0x698098d8,  7);
-  MD5STEP(F1, d, a, b, c, in[ 9]+0x8b44f7af, 12);
-  MD5STEP(F1, c, d, a, b, in[10]+0xffff5bb1, 17);
-  MD5STEP(F1, b, c, d, a, in[11]+0x895cd7be, 22);
-  MD5STEP(F1, a, b, c, d, in[12]+0x6b901122,  7);
-  MD5STEP(F1, d, a, b, c, in[13]+0xfd987193, 12);
-  MD5STEP(F1, c, d, a, b, in[14]+0xa679438e, 17);
-  MD5STEP(F1, b, c, d, a, in[15]+0x49b40821, 22);
-
-  MD5STEP(F2, a, b, c, d, in[ 1]+0xf61e2562,  5);
-  MD5STEP(F2, d, a, b, c, in[ 6]+0xc040b340,  9);
-  MD5STEP(F2, c, d, a, b, in[11]+0x265e5a51, 14);
-  MD5STEP(F2, b, c, d, a, in[ 0]+0xe9b6c7aa, 20);
-  MD5STEP(F2, a, b, c, d, in[ 5]+0xd62f105d,  5);
-  MD5STEP(F2, d, a, b, c, in[10]+0x02441453,  9);
-  MD5STEP(F2, c, d, a, b, in[15]+0xd8a1e681, 14);
-  MD5STEP(F2, b, c, d, a, in[ 4]+0xe7d3fbc8, 20);
-  MD5STEP(F2, a, b, c, d, in[ 9]+0x21e1cde6,  5);
-  MD5STEP(F2, d, a, b, c, in[14]+0xc33707d6,  9);
-  MD5STEP(F2, c, d, a, b, in[ 3]+0xf4d50d87, 14);
-  MD5STEP(F2, b, c, d, a, in[ 8]+0x455a14ed, 20);
-  MD5STEP(F2, a, b, c, d, in[13]+0xa9e3e905,  5);
-  MD5STEP(F2, d, a, b, c, in[ 2]+0xfcefa3f8,  9);
-  MD5STEP(F2, c, d, a, b, in[ 7]+0x676f02d9, 14);
-  MD5STEP(F2, b, c, d, a, in[12]+0x8d2a4c8a, 20);
-
-  MD5STEP(F3, a, b, c, d, in[ 5]+0xfffa3942,  4);
-  MD5STEP(F3, d, a, b, c, in[ 8]+0x8771f681, 11);
-  MD5STEP(F3, c, d, a, b, in[11]+0x6d9d6122, 16);
-  MD5STEP(F3, b, c, d, a, in[14]+0xfde5380c, 23);
-  MD5STEP(F3, a, b, c, d, in[ 1]+0xa4beea44,  4);
-  MD5STEP(F3, d, a, b, c, in[ 4]+0x4bdecfa9, 11);
-  MD5STEP(F3, c, d, a, b, in[ 7]+0xf6bb4b60, 16);
-  MD5STEP(F3, b, c, d, a, in[10]+0xbebfbc70, 23);
-  MD5STEP(F3, a, b, c, d, in[13]+0x289b7ec6,  4);
-  MD5STEP(F3, d, a, b, c, in[ 0]+0xeaa127fa, 11);
-  MD5STEP(F3, c, d, a, b, in[ 3]+0xd4ef3085, 16);
-  MD5STEP(F3, b, c, d, a, in[ 6]+0x04881d05, 23);
-  MD5STEP(F3, a, b, c, d, in[ 9]+0xd9d4d039,  4);
-  MD5STEP(F3, d, a, b, c, in[12]+0xe6db99e5, 11);
-  MD5STEP(F3, c, d, a, b, in[15]+0x1fa27cf8, 16);
-  MD5STEP(F3, b, c, d, a, in[ 2]+0xc4ac5665, 23);
-
-  MD5STEP(F4, a, b, c, d, in[ 0]+0xf4292244,  6);
-  MD5STEP(F4, d, a, b, c, in[ 7]+0x432aff97, 10);
-  MD5STEP(F4, c, d, a, b, in[14]+0xab9423a7, 15);
-  MD5STEP(F4, b, c, d, a, in[ 5]+0xfc93a039, 21);
-  MD5STEP(F4, a, b, c, d, in[12]+0x655b59c3,  6);
-  MD5STEP(F4, d, a, b, c, in[ 3]+0x8f0ccc92, 10);
-  MD5STEP(F4, c, d, a, b, in[10]+0xffeff47d, 15);
-  MD5STEP(F4, b, c, d, a, in[ 1]+0x85845dd1, 21);
-  MD5STEP(F4, a, b, c, d, in[ 8]+0x6fa87e4f,  6);
-  MD5STEP(F4, d, a, b, c, in[15]+0xfe2ce6e0, 10);
-  MD5STEP(F4, c, d, a, b, in[ 6]+0xa3014314, 15);
-  MD5STEP(F4, b, c, d, a, in[13]+0x4e0811a1, 21);
-  MD5STEP(F4, a, b, c, d, in[ 4]+0xf7537e82,  6);
-  MD5STEP(F4, d, a, b, c, in[11]+0xbd3af235, 10);
-  MD5STEP(F4, c, d, a, b, in[ 2]+0x2ad7d2bb, 15);
-  MD5STEP(F4, b, c, d, a, in[ 9]+0xeb86d391, 21);
-
-  buf[0] += a;
-  buf[1] += b;
-  buf[2] += c;
-  buf[3] += d;
-}
-
-/*
- * Start MD5 accumulation.  Set bit count to 0 and buffer to mysterious
- * initialization constants.
- */
-static void MD5Init(MD5Context *ctx){
-  ctx->isInit = 1;
-  ctx->buf[0] = 0x67452301;
-  ctx->buf[1] = 0xefcdab89;
-  ctx->buf[2] = 0x98badcfe;
-  ctx->buf[3] = 0x10325476;
-  ctx->bits[0] = 0;
-  ctx->bits[1] = 0;
-}
-
-/*
- * Update context to reflect the concatenation of another buffer full
- * of bytes.
- */
-static 
-void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){
-  uint32 t;
-
-  /* Update bitcount */
-
-  t = ctx->bits[0];
-  if ((ctx->bits[0] = t + ((uint32)len << 3)) < t)
-    ctx->bits[1]++; /* Carry from low to high */
-  ctx->bits[1] += len >> 29;
-
-  t = (t >> 3) & 0x3f;    /* Bytes already in shsInfo->data */
-
-  /* Handle any leading odd-sized chunks */
-
-  if ( t ) {
-    unsigned char *p = (unsigned char *)ctx->u.in + t;
-
-    t = 64-t;
-    if (len < t) {
-      memcpy(p, buf, len);
-      return;
-    }
-    memcpy(p, buf, t);
-    byteReverse(ctx->u.in, 16);
-    MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
-    buf += t;
-    len -= t;
-  }
-
-  /* Process data in 64-byte chunks */
-
-  while (len >= 64) {
-    memcpy(ctx->u.in, buf, 64);
-    byteReverse(ctx->u.in, 16);
-    MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
-    buf += 64;
-    len -= 64;
-  }
-
-  /* Handle any remaining bytes of data. */
-
-  memcpy(ctx->u.in, buf, len);
-}
-
-/*
- * Final wrapup - pad to 64-byte boundary with the bit pattern 
- * 1 0* (64-bit count of bits processed, MSB-first)
- */
-static void MD5Final(unsigned char digest[16], MD5Context *ctx){
-  unsigned count;
-  unsigned char *p;
-
-  /* Compute number of bytes mod 64 */
-  count = (ctx->bits[0] >> 3) & 0x3F;
-
-  /* Set the first char of padding to 0x80.  This is safe since there is
-     always at least one byte free */
-  p = ctx->u.in + count;
-  *p++ = 0x80;
-
-  /* Bytes of padding needed to make 64 bytes */
-  count = 64 - 1 - count;
-
-  /* Pad out to 56 mod 64 */
-  if (count < 8) {
-    /* Two lots of padding:  Pad the first block to 64 bytes */
-    memset(p, 0, count);
-    byteReverse(ctx->u.in, 16);
-    MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
-
-    /* Now fill the next block with 56 bytes */
-    memset(ctx->u.in, 0, 56);
-  } else {
-    /* Pad block to 56 bytes */
-    memset(p, 0, count-8);
-  }
-  byteReverse(ctx->u.in, 14);
-
-  /* Append length in bits and transform */
-  ctx->u.in32[14] = ctx->bits[0];
-  ctx->u.in32[15] = ctx->bits[1];
-
-  MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
-  byteReverse((unsigned char *)ctx->buf, 4);
-  memcpy(digest, ctx->buf, 16);
-  memset(ctx, 0, sizeof(*ctx));    /* In case it is sensitive */
-}
-
-/*
-** Convert a 128-bit MD5 digest into a 32-digit base-16 number.
-*/
-static void MD5DigestToBase16(unsigned char *digest, char *zBuf){
-  static char const zEncode[] = "0123456789abcdef";
-  int i, j;
-
-  for(j=i=0; i<16; i++){
-    int a = digest[i];
-    zBuf[j++] = zEncode[(a>>4)&0xf];
-    zBuf[j++] = zEncode[a & 0xf];
-  }
-  zBuf[j] = 0;
-}
-
-/*
-** During testing, the special md5sum() aggregate function is available.
-** inside SQLite.  The following routines implement that function.
-*/
-static void md5step(sqlite3_context *context, int argc, sqlite3_value **argv){
-  MD5Context *p;
-  int i;
-  if( argc<1 ) return;
-  p = sqlite3_aggregate_context(context, sizeof(*p));
-  if( p==0 ) return;
-  if( !p->isInit ){
-    MD5Init(p);
-  }
-  for(i=0; i<argc; i++){
-    const char *zData = (char*)sqlite3_value_text(argv[i]);
-    if( zData ){
-      MD5Update(p, (unsigned char*)zData, strlen(zData));
-    }
-  }
-}
-static void md5finalize(sqlite3_context *context){
-  MD5Context *p;
-  unsigned char digest[16];
-  char zBuf[33];
-  p = sqlite3_aggregate_context(context, sizeof(*p));
-  MD5Final(digest,p);
-  MD5DigestToBase16(digest, zBuf);
-  sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
-}
-
-/*
-** End of copied md5sum() code.
-**************************************************************************/
-
-typedef sqlite3_int64 i64;
-
-typedef struct Error Error;
-typedef struct Sqlite Sqlite;
-typedef struct Statement Statement;
-
-typedef struct Threadset Threadset;
-typedef struct Thread Thread;
-
-/* Total number of errors in this process so far. */
-static int nGlobalErr = 0;
-
-struct Error {
-  int rc;
-  int iLine;
-  char *zErr;
-};
-
-struct Sqlite {
-  sqlite3 *db;                    /* Database handle */
-  Statement *pCache;              /* Linked list of cached statements */
-  int nText;                      /* Size of array at aText[] */
-  char **aText;                   /* Stored text results */
-};
-
-struct Statement {
-  sqlite3_stmt *pStmt;            /* Pre-compiled statement handle */
-  Statement *pNext;               /* Next statement in linked-list */
-};
-
-struct Thread {
-  int iTid;                       /* Thread number within test */
-  void* pArg;                     /* Pointer argument passed by caller */
-
-  pthread_t tid;                  /* Thread id */
-  char *(*xProc)(int, void*);     /* Thread main proc */
-  Thread *pNext;                  /* Next in this list of threads */
-};
-
-struct Threadset {
-  int iMaxTid;                    /* Largest iTid value allocated so far */
-  Thread *pThread;                /* Linked list of threads */
-};
-
-static void free_err(Error *p){
-  sqlite3_free(p->zErr);
-  p->zErr = 0;
-  p->rc = 0;
-}
-
-static void print_err(Error *p){
-  if( p->rc!=SQLITE_OK ){
-    int isWarn = 0;
-    if( p->rc==SQLITE_SCHEMA ) isWarn = 1;
-    if( sqlite3_strglob("* - no such table: *",p->zErr)==0 ) isWarn = 1;
-    printf("%s: (%d) \"%s\" at line %d\n", isWarn ? "Warning" : "Error",
-            p->rc, p->zErr, p->iLine);
-    if( !isWarn ) nGlobalErr++;
-    fflush(stdout);
-  }
-}
-
-static void print_and_free_err(Error *p){
-  print_err(p);
-  free_err(p);
-}
-
-static void system_error(Error *pErr, int iSys){
-  pErr->rc = iSys;
-  pErr->zErr = (char *)sqlite3_malloc(512);
-  strerror_r(iSys, pErr->zErr, 512);
-  pErr->zErr[511] = '\0';
-}
-
-static void sqlite_error(
-  Error *pErr, 
-  Sqlite *pDb, 
-  const char *zFunc
-){
-  pErr->rc = sqlite3_errcode(pDb->db);
-  pErr->zErr = sqlite3_mprintf(
-      "sqlite3_%s() - %s (%d)", zFunc, sqlite3_errmsg(pDb->db),
-      sqlite3_extended_errcode(pDb->db)
-  );
-}
-
-static void test_error_x(
-  Error *pErr,
-  char *zErr
-){
-  if( pErr->rc==SQLITE_OK ){
-    pErr->rc = 1;
-    pErr->zErr = zErr;
-  }else{
-    sqlite3_free(zErr);
-  }
-}
-
-static void clear_error_x(
-  Error *pErr,
-  int rc
-){
-  if( pErr->rc==rc ){
-    pErr->rc = SQLITE_OK;
-    sqlite3_free(pErr->zErr);
-    pErr->zErr = 0;
-  }
-}
-
-static int busyhandler(void *pArg, int n){
-  usleep(10*1000);
-  return 1;
-}
-
-static void opendb_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb,                    /* OUT: Database handle */
-  const char *zFile,              /* Database file name */
-  int bDelete                     /* True to delete db file before opening */
-){
-  if( pErr->rc==SQLITE_OK ){
-    int rc;
-    int flags = SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI;
-    if( bDelete ) unlink(zFile);
-    rc = sqlite3_open_v2(zFile, &pDb->db, flags, 0);
-    if( rc ){
-      sqlite_error(pErr, pDb, "open");
-      sqlite3_close(pDb->db);
-      pDb->db = 0;
-    }else{
-      sqlite3_create_function(
-          pDb->db, "md5sum", -1, SQLITE_UTF8, 0, 0, md5step, md5finalize
-      );
-      sqlite3_busy_handler(pDb->db, busyhandler, 0);
-      sqlite3_exec(pDb->db, "PRAGMA synchronous=OFF", 0, 0, 0);
-    }
-  }
-}
-
-static void closedb_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb                     /* OUT: Database handle */
-){
-  int rc;
-  int i;
-  Statement *pIter;
-  Statement *pNext;
-  for(pIter=pDb->pCache; pIter; pIter=pNext){
-    pNext = pIter->pNext;
-    sqlite3_finalize(pIter->pStmt);
-    sqlite3_free(pIter);
-  }
-  for(i=0; i<pDb->nText; i++){
-    sqlite3_free(pDb->aText[i]);
-  }
-  sqlite3_free(pDb->aText);
-  rc = sqlite3_close(pDb->db);
-  if( rc && pErr->rc==SQLITE_OK ){
-    pErr->zErr = sqlite3_mprintf("%s", sqlite3_errmsg(pDb->db));
-  }
-  memset(pDb, 0, sizeof(Sqlite));
-}
-
-static void sql_script_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb,                    /* Database handle */
-  const char *zSql                /* SQL script to execute */
-){
-  if( pErr->rc==SQLITE_OK ){
-    pErr->rc = sqlite3_exec(pDb->db, zSql, 0, 0, &pErr->zErr);
-  }
-}
-
-static void sql_script_printf_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb,                    /* Database handle */
-  const char *zFormat,            /* SQL printf format string */
-  ...                             /* Printf args */
-){
-  va_list ap;                     /* ... printf arguments */
-  va_start(ap, zFormat);
-  if( pErr->rc==SQLITE_OK ){
-    char *zSql = sqlite3_vmprintf(zFormat, ap);
-    pErr->rc = sqlite3_exec(pDb->db, zSql, 0, 0, &pErr->zErr);
-    sqlite3_free(zSql);
-  }
-  va_end(ap);
-}
-
-static Statement *getSqlStatement(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb,                    /* Database handle */
-  const char *zSql                /* SQL statement */
-){
-  Statement *pRet;
-  int rc;
-
-  for(pRet=pDb->pCache; pRet; pRet=pRet->pNext){
-    if( 0==strcmp(sqlite3_sql(pRet->pStmt), zSql) ){
-      return pRet;
-    }
-  }
-
-  pRet = sqlite3_malloc(sizeof(Statement));
-  rc = sqlite3_prepare_v2(pDb->db, zSql, -1, &pRet->pStmt, 0);
-  if( rc!=SQLITE_OK ){
-    sqlite_error(pErr, pDb, "prepare_v2");
-    return 0;
-  }
-  assert( 0==strcmp(sqlite3_sql(pRet->pStmt), zSql) );
-
-  pRet->pNext = pDb->pCache;
-  pDb->pCache = pRet;
-  return pRet;
-}
-
-static sqlite3_stmt *getAndBindSqlStatement(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb,                    /* Database handle */
-  va_list ap                      /* SQL followed by parameters */
-){
-  Statement *pStatement;          /* The SQLite statement wrapper */
-  sqlite3_stmt *pStmt;            /* The SQLite statement to return */
-  int i;                          /* Used to iterate through parameters */
-
-  pStatement = getSqlStatement(pErr, pDb, va_arg(ap, const char *));
-  if( !pStatement ) return 0;
-  pStmt = pStatement->pStmt;
-  for(i=1; i<=sqlite3_bind_parameter_count(pStmt); i++){
-    const char *zName = sqlite3_bind_parameter_name(pStmt, i);
-    void * pArg = va_arg(ap, void*);
-
-    switch( zName[1] ){
-      case 'i':
-        sqlite3_bind_int64(pStmt, i, *(i64 *)pArg);
-        break;
-
-      default:
-        pErr->rc = 1;
-        pErr->zErr = sqlite3_mprintf("Cannot discern type: \"%s\"", zName);
-        pStmt = 0;
-        break;
-    }
-  }
-
-  return pStmt;
-}
-
-static i64 execsql_i64_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb,                    /* Database handle */
-  ...                             /* SQL and pointers to parameter values */
-){
-  i64 iRet = 0;
-  if( pErr->rc==SQLITE_OK ){
-    sqlite3_stmt *pStmt;          /* SQL statement to execute */
-    va_list ap;                   /* ... arguments */
-    va_start(ap, pDb);
-    pStmt = getAndBindSqlStatement(pErr, pDb, ap);
-    if( pStmt ){
-      int first = 1;
-      while( SQLITE_ROW==sqlite3_step(pStmt) ){
-        if( first && sqlite3_column_count(pStmt)>0 ){
-          iRet = sqlite3_column_int64(pStmt, 0);
-        }
-        first = 0;
-      }
-      if( SQLITE_OK!=sqlite3_reset(pStmt) ){
-        sqlite_error(pErr, pDb, "reset");
-      }
-    }
-    va_end(ap);
-  }
-  return iRet;
-}
-
-static char * execsql_text_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb,                    /* Database handle */
-  int iSlot,                      /* Db handle slot to store text in */
-  ...                             /* SQL and pointers to parameter values */
-){
-  char *zRet = 0;
-
-  if( iSlot>=pDb->nText ){
-    int nByte = sizeof(char *)*(iSlot+1);
-    pDb->aText = (char **)sqlite3_realloc(pDb->aText, nByte);
-    memset(&pDb->aText[pDb->nText], 0, sizeof(char*)*(iSlot+1-pDb->nText));
-    pDb->nText = iSlot+1;
-  }
-
-  if( pErr->rc==SQLITE_OK ){
-    sqlite3_stmt *pStmt;          /* SQL statement to execute */
-    va_list ap;                   /* ... arguments */
-    va_start(ap, iSlot);
-    pStmt = getAndBindSqlStatement(pErr, pDb, ap);
-    if( pStmt ){
-      int first = 1;
-      while( SQLITE_ROW==sqlite3_step(pStmt) ){
-        if( first && sqlite3_column_count(pStmt)>0 ){
-          zRet = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));
-          sqlite3_free(pDb->aText[iSlot]);
-          pDb->aText[iSlot] = zRet;
-        }
-        first = 0;
-      }
-      if( SQLITE_OK!=sqlite3_reset(pStmt) ){
-        sqlite_error(pErr, pDb, "reset");
-      }
-    }
-    va_end(ap);
-  }
-
-  return zRet;
-}
-
-static void integrity_check_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Sqlite *pDb                     /* Database handle */
-){
-  if( pErr->rc==SQLITE_OK ){
-    Statement *pStatement;        /* Statement to execute */
-    char *zErr = 0;               /* Integrity check error */
-
-    pStatement = getSqlStatement(pErr, pDb, "PRAGMA integrity_check");
-    if( pStatement ){
-      sqlite3_stmt *pStmt = pStatement->pStmt;
-      while( SQLITE_ROW==sqlite3_step(pStmt) ){
-        const char *z = (const char*)sqlite3_column_text(pStmt, 0);
-        if( strcmp(z, "ok") ){
-          if( zErr==0 ){
-            zErr = sqlite3_mprintf("%s", z);
-          }else{
-            zErr = sqlite3_mprintf("%z\n%s", zErr, z);
-          }
-        }
-      }
-      sqlite3_reset(pStmt);
-
-      if( zErr ){
-        pErr->zErr = zErr;
-        pErr->rc = 1;
-      }
-    }
-  }
-}
-
-static void *launch_thread_main(void *pArg){
-  Thread *p = (Thread *)pArg;
-  return (void *)p->xProc(p->iTid, p->pArg);
-}
-
-static void launch_thread_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Threadset *pThreads,            /* Thread set */
-  char *(*xProc)(int, void*),     /* Proc to run */
-  void *pArg                      /* Argument passed to thread proc */
-){
-  if( pErr->rc==SQLITE_OK ){
-    int iTid = ++pThreads->iMaxTid;
-    Thread *p;
-    int rc;
-
-    p = (Thread *)sqlite3_malloc(sizeof(Thread));
-    memset(p, 0, sizeof(Thread));
-    p->iTid = iTid;
-    p->pArg = pArg;
-    p->xProc = xProc;
-
-    rc = pthread_create(&p->tid, NULL, launch_thread_main, (void *)p);
-    if( rc!=0 ){
-      system_error(pErr, rc);
-      sqlite3_free(p);
-    }else{
-      p->pNext = pThreads->pThread;
-      pThreads->pThread = p;
-    }
-  }
-}
-
-static void join_all_threads_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  Threadset *pThreads             /* Thread set */
-){
-  Thread *p;
-  Thread *pNext;
-  for(p=pThreads->pThread; p; p=pNext){
-    void *ret;
-    pNext = p->pNext;
-    int rc;
-    rc = pthread_join(p->tid, &ret);
-    if( rc!=0 ){
-      if( pErr->rc==SQLITE_OK ) system_error(pErr, rc);
-    }else{
-      printf("Thread %d says: %s\n", p->iTid, (ret==0 ? "..." : (char *)ret));
-      fflush(stdout);
-    }
-    sqlite3_free(p);
-  }
-  pThreads->pThread = 0;
-}
-
-static i64 filesize_x(
-  Error *pErr,
-  const char *zFile
-){
-  i64 iRet = 0;
-  if( pErr->rc==SQLITE_OK ){
-    struct stat sStat;
-    if( stat(zFile, &sStat) ){
-      iRet = -1;
-    }else{
-      iRet = sStat.st_size;
-    }
-  }
-  return iRet;
-}
-
-static void filecopy_x(
-  Error *pErr,
-  const char *zFrom,
-  const char *zTo
-){
-  if( pErr->rc==SQLITE_OK ){
-    i64 nByte = filesize_x(pErr, zFrom);
-    if( nByte<0 ){
-      test_error_x(pErr, sqlite3_mprintf("no such file: %s", zFrom));
-    }else{
-      i64 iOff;
-      char aBuf[1024];
-      int fd1;
-      int fd2;
-      unlink(zTo);
-
-      fd1 = open(zFrom, O_RDONLY);
-      if( fd1<0 ){
-        system_error(pErr, errno);
-        return;
-      }
-      fd2 = open(zTo, O_RDWR|O_CREAT|O_EXCL, 0644);
-      if( fd2<0 ){
-        system_error(pErr, errno);
-        close(fd1);
-        return;
-      }
-
-      iOff = 0;
-      while( iOff<nByte ){
-        int nCopy = sizeof(aBuf);
-        if( nCopy+iOff>nByte ){
-          nCopy = nByte - iOff;
-        }
-        if( nCopy!=read(fd1, aBuf, nCopy) ){
-          system_error(pErr, errno);
-          break;
-        }
-        if( nCopy!=write(fd2, aBuf, nCopy) ){
-          system_error(pErr, errno);
-          break;
-        }
-        iOff += nCopy;
-      }
-
-      close(fd1);
-      close(fd2);
-    }
-  }
-}
-
-/* 
-** Used by setstoptime() and timetostop().
-*/
-static double timelimit = 0.0;
-
-static double currentTime(void){
-  double t;
-  static sqlite3_vfs *pTimelimitVfs = 0;
-  if( pTimelimitVfs==0 ) pTimelimitVfs = sqlite3_vfs_find(0);
-  if( pTimelimitVfs->iVersion>=2 && pTimelimitVfs->xCurrentTimeInt64!=0 ){
-    sqlite3_int64 tm;
-    pTimelimitVfs->xCurrentTimeInt64(pTimelimitVfs, &tm);
-    t = tm/86400000.0;
-  }else{
-    pTimelimitVfs->xCurrentTime(pTimelimitVfs, &t);
-  }
-  return t;
-}
-
-static void setstoptime_x(
-  Error *pErr,                    /* IN/OUT: Error code */
-  int nMs                         /* Milliseconds until "stop time" */
-){
-  if( pErr->rc==SQLITE_OK ){
-    double t = currentTime();
-    timelimit = t + ((double)nMs)/(1000.0*60.0*60.0*24.0);
-  }
-}
-
-static int timetostop_x(
-  Error *pErr                     /* IN/OUT: Error code */
-){
-  int ret = 1;
-  if( pErr->rc==SQLITE_OK ){
-    double t = currentTime();
-    ret = (t >= timelimit);
-  }
-  return ret;
-}
-
 
 /*************************************************************************
 **************************************************************************

test/wal2.test

@@ -35,43 +35,6 @@ proc cond_incr_sync_count {adj} {
   }
 }
 
-proc set_tvfs_hdr {file args} {
-
-  # Set $nHdr to the number of bytes in the wal-index header:
-  set nHdr 48
-  set nInt [expr {$nHdr/4}]
-
-  if {[llength $args]>2} {
-    error {wrong # args: should be "set_tvfs_hdr fileName ?val1? ?val2?"}
-  }
-
-  set blob [tvfs shm $file]
-  if {$::tcl_platform(byteOrder)=="bigEndian"} {set fmt I} {set fmt i}
-
-  if {[llength $args]} {
-    set ia [lindex $args 0]
-    set ib $ia
-    if {[llength $args]==2} {
-      set ib [lindex $args 1]
-    }
-    binary scan $blob a[expr $nHdr*2]a* dummy tail
-    set blob [binary format ${fmt}${nInt}${fmt}${nInt}a* $ia $ib $tail]
-    tvfs shm $file $blob
-  }
-
-  binary scan $blob ${fmt}${nInt} ints
-  return $ints
-}
-
-proc incr_tvfs_hdr {file idx incrval} {
-  set ints [set_tvfs_hdr $file]
-  set v [lindex $ints $idx]
-  incr v $incrval
-  lset ints $idx $v
-  set_tvfs_hdr $file $ints
-}
-
-
 #-------------------------------------------------------------------------
 # Test case wal2-1.*:
 #

test/wal_common.tcl

@@ -90,4 +90,44 @@ proc wal_fix_walindex_cksum {hdrvar} {
   lset hdr 11 $c2
 }
 
+# This command assumes that $file is the name of a database file opened
+# in wal mode using a [testvfs] VFS. It returns a list of the 12 32-bit
+# integers that make up the wal-index-header for the named file.
+#
+proc set_tvfs_hdr {file args} {
+
+  # Set $nHdr to the number of bytes in the wal-index header:
+  set nHdr 48
+  set nInt [expr {$nHdr/4}]
+
+  if {[llength $args]>2} {
+    error {wrong # args: should be "set_tvfs_hdr fileName ?val1? ?val2?"}
+  }
+
+  set blob [tvfs shm $file]
+  if {$::tcl_platform(byteOrder)=="bigEndian"} {set fmt I} {set fmt i}
+
+  if {[llength $args]} {
+    set ia [lindex $args 0]
+    set ib $ia
+    if {[llength $args]==2} {
+      set ib [lindex $args 1]
+    }
+    binary scan $blob a[expr $nHdr*2]a* dummy tail
+    set blob [binary format ${fmt}${nInt}${fmt}${nInt}a* $ia $ib $tail]
+    tvfs shm $file $blob
+  }
+
+  binary scan $blob ${fmt}${nInt} ints
+  return $ints
+}
+
+proc incr_tvfs_hdr {file idx incrval} {
+  set ints [set_tvfs_hdr $file]
+  set v [lindex $ints $idx]
+  incr v $incrval
+  lset ints $idx $v
+  set_tvfs_hdr $file $ints
+}
+
 

tool/addopcodes.tcl

@@ -32,6 +32,7 @@ set extras {
   UPLUS
   TRUTH
   REGISTER
+  CONCURRENT
   VECTOR
   SELECT_COLUMN
   IF_NULL_ROW

tool/mkkeywordhash.c

@@ -192,7 +192,6 @@ static Keyword aKeywordTable[] = {
   { "ELSE",             "TK_ELSE",         ALWAYS                 },
   { "ESCAPE",           "TK_ESCAPE",       ALWAYS                 },
   { "EXCEPT",           "TK_EXCEPT",       COMPOUND               },
-  { "EXCLUSIVE",        "TK_EXCLUSIVE",    ALWAYS                 },
   { "EXISTS",           "TK_EXISTS",       ALWAYS                 },
   { "EXPLAIN",          "TK_EXPLAIN",      EXPLAIN                },
   { "FAIL",             "TK_FAIL",         CONFLICT|TRIGGER       },

tool/mkpragmatab.tcl

@@ -112,6 +112,12 @@ set pragma_def {
   IF:   !defined(SQLITE_OMIT_FLAG_PRAGMAS)
   IF:   defined(SQLITE_DEBUG)
 
+  NAME: noop_update
+  TYPE: FLAG
+  ARG:  SQLITE_NoopUpdate
+  IF:   !defined(SQLITE_OMIT_FLAG_PRAGMAS)
+  IF:   defined(SQLITE_ENABLE_NOOP_UPDATE)
+
   NAME: ignore_check_constraints
   TYPE: FLAG
   ARG:  SQLITE_IgnoreChecks