Change the instrumentation messages on this branch so that they contain "v=7" instead of "v=6".

FossilOrigin-Name: acea12d90f180538161b9611ab8fb0e10a43262c15739a8c24d3732b99f968ed

Makefile.in

@@ -433,6 +433,7 @@ TESTSRC = \
   $(TOP)/ext/fts3/fts3_term.c \
   $(TOP)/ext/fts3/fts3_test.c  \
   $(TOP)/ext/session/test_session.c \
+  $(TOP)/ext/session/sqlite3changebatch.c \
   $(TOP)/ext/rbu/test_rbu.c 
 
 # Statically linked extensions
@@ -441,6 +442,7 @@ TESTSRC += \
   $(TOP)/ext/expert/sqlite3expert.c \
   $(TOP)/ext/expert/test_expert.c \
   $(TOP)/ext/misc/amatch.c \
+  $(TOP)/ext/misc/bgckpt.c \
   $(TOP)/ext/misc/appendvfs.c \
   $(TOP)/ext/misc/carray.c \
   $(TOP)/ext/misc/cksumvfs.c \

Makefile.msc

@@ -1560,6 +1560,7 @@ TESTEXT = \
   $(TOP)\ext\expert\test_expert.c \
   $(TOP)\ext\misc\amatch.c \
   $(TOP)\ext\misc\appendvfs.c \
+  $(TOP)\ext\misc\bgckpt.c \
   $(TOP)\ext\misc\carray.c \
   $(TOP)\ext\misc\cksumvfs.c \
   $(TOP)\ext\misc\closure.c \

doc/begin_concurrent.md

@@ -0,0 +1,107 @@
+
+Begin Concurrent
+================
+
+## Overview
+
+Usually, SQLite allows at most one writer to proceed concurrently. The
+BEGIN CONCURRENT enhancement allows multiple writers to process write
+transactions simultanously if the database is in "wal" or "wal2" mode,
+although the system still serializes COMMIT commands.
+
+When a write-transaction is opened with "BEGIN CONCURRENT", actually 
+locking the database is deferred until a COMMIT is executed. This means
+that any number of transactions started with BEGIN CONCURRENT may proceed
+concurrently. The system uses optimistic page-level-locking to prevent
+conflicting concurrent transactions from being committed.
+
+When a BEGIN CONCURRENT transaction is committed, the system checks whether 
+or not any of the database pages that the transaction has read have been
+modified since the BEGIN CONCURRENT was opened. In other words - it asks 
+if the transaction being committed operates on a different set of data than
+all other concurrently executing transactions. If the answer is "yes, this
+transaction did not read or modify any data modified by any concurrent
+transaction", then the transaction is committed as normal. Otherwise, if the
+transaction does conflict, it cannot be committed and an SQLITE_BUSY_SNAPSHOT
+error is returned. At this point, all the client can do is ROLLBACK the
+transaction.
+
+If SQLITE_BUSY_SNAPSHOT is returned, messages are output via the sqlite3_log
+mechanism indicating the page and table or index on which the conflict
+occurred. This can be useful when optimizing concurrency.
+
+## Application Programming Notes
+
+In order to serialize COMMIT processing, SQLite takes a lock on the database
+as part of each COMMIT command and releases it before returning. At most one
+writer may hold this lock at any one time. If a writer cannot obtain the lock,
+it uses SQLite's busy-handler to pause and retry for a while:
+
+  <a href=https://www.sqlite.org/c3ref/busy_handler.html>
+      https://www.sqlite.org/c3ref/busy_handler.html
+  </a>
+
+If there is significant contention for the writer lock, this mechanism can be
+inefficient. In this case it is better for the application to use a mutex or
+some other mechanism that supports blocking to ensure that at most one writer
+is attempting to COMMIT a BEGIN CONCURRENT transaction at a time. This is
+usually easier if all writers are part of the same operating system process.
+
+If all database clients (readers and writers) are located in the same OS
+process, and if that OS is a Unix variant, then it can be more efficient to
+the built-in VFS "unix-excl" instead of the default "unix". This is because it
+uses more efficient locking primitives.
+
+The key to maximizing concurrency using BEGIN CONCURRENT is to ensure that
+there are a large number of non-conflicting transactions. In SQLite, each
+table and each index is stored as a separate b-tree, each of which is
+distributed over a discrete set of database pages. This means that:
+
+  * Two transactions that write to different sets of tables never 
+    conflict, and that
+
+  * Two transactions that write to the same tables or indexes only 
+    conflict if the values of the keys (either primary keys or indexed 
+    rows) are fairly close together. For example, given a large 
+    table with the schema:
+
+      <pre>     CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB);</pre>
+
+    writing two rows with adjacent values for "a" probably will cause a
+    conflict (as the two keys are stored on the same page), but writing two
+    rows with vastly different values for "a" will not (as the keys will likly
+    be stored on different pages).
+
+Note that, in SQLite, if values are not explicitly supplied for an INTEGER
+PRIMARY KEY, as for example in:
+
+>
+     INSERT INTO t1(b) VALUES(&lt;blob-value>);
+
+then monotonically increasing values are assigned automatically. This is
+terrible for concurrency, as it all but ensures that all new rows are 
+added to the same database page. In such situations, it is better to
+explicitly assign random values to INTEGER PRIMARY KEY fields.
+
+This problem also comes up for non-WITHOUT ROWID tables that do not have an
+explicit INTEGER PRIMARY KEY column. In these cases each table has an implicit
+INTEGER PRIMARY KEY column that is assigned increasing values, leading to the
+same problem as omitting to assign a value to an explicit INTEGER PRIMARY KEY
+column.
+
+For both explicit and implicit INTEGER PRIMARY KEYs, it is possible to have
+SQLite assign values at random (instead of the monotonically increasing
+values) by writing a row with a rowid equal to the largest possible signed
+64-bit integer to the table. For example:
+
+     INSERT INTO t1(a) VALUES(9223372036854775807);
+
+Applications should take care not to malfunction due to the presence of such
+rows.
+
+The nature of some types of indexes, for example indexes on timestamp fields,
+can also cause problems (as concurrent transactions may assign similar
+timestamps that will be stored on the same db page to new records). In these
+cases the database schema may need to be rethought to increase the concurrency
+provided by page-level-locking.
+

doc/wal2.md

@@ -0,0 +1,98 @@
+
+Wal2 Mode Notes
+===============
+
+## Activating/Deactivating Wal2 Mode
+
+"Wal2" mode is very similar to "wal" mode. To change a database to wal2 mode,
+use the command:
+
+>
+     PRAGMA journal_mode = wal2;
+
+It is not possible to change a database directly from "wal" mode to "wal2"
+mode. Instead, it must first be changed to rollback mode. So, to change a wal
+mode database to wal2 mode, the following two commands may be used:
+
+>
+     PRAGMA journal_mode = delete;
+     PRAGMA journal_mode = wal2;
+
+A database in wal2 mode may only be accessed by versions of SQLite compiled
+from this branch. Attempting to use any other version of SQLite results in an
+SQLITE_NOTADB error. A wal2 mode database may be changed back to rollback mode
+(making it accessible by all versions of SQLite) using:
+
+>
+     PRAGMA journal_mode = delete;
+
+## The Advantage of Wal2 Mode
+
+In legacy wal mode, when a writer writes data to the database, it doesn't
+modify the database file directly. Instead, it appends new data to the
+"<database>-wal" file. Readers read data from both the original database
+file and the "<database>-wal" file. At some point, data is copied from the
+"<database>-wal" file into the database file, after which the wal file can
+be deleted or overwritten. Copying data from the wal file into the database
+file is called a "checkpoint", and may be done explictly (either by "PRAGMA
+wal_checkpoint" or sqlite3_wal_checkpoint_v2()), or
+automatically (by configuring "PRAGMA wal_autocheckpoint" - this is the
+default).
+
+Checkpointers do not block writers, and writers do not block checkpointers.
+However, if a writer writes to the database while a checkpoint is ongoing,
+then the new data is appended to the end of the wal file. This means that,
+even following the checkpoint, the wal file cannot be overwritten or deleted,
+and so all subsequent transactions must also be appended to the wal file. The
+work of the checkpointer is not wasted - SQLite remembers which parts of the
+wal file have already been copied into the db file so that the next checkpoint
+does not have to do so again - but it does mean that the wal file may grow
+indefinitely if the checkpointer never gets a chance to finish without a
+writer appending to the wal file. There are also circumstances in which
+long-running readers may prevent a checkpointer from checkpointing the entire
+wal file - also causing the wal file to grow indefinitely in a busy system.
+
+Wal2 mode does not have this problem. In wal2 mode, wal files do not grow
+indefinitely even if the checkpointer never has a chance to finish
+uninterrupted.
+
+In wal2 mode, the system uses two wal files instead of one. The files are named
+"<database>-wal" and "<database>-wal2", where "<database>" is of
+course the name of the database file. When data is written to the database, the
+writer begins by appending the new data to the first wal file. Once the first
+wal file has grown large enough, writers switch to appending data to the second
+wal file. At this point the first wal file can be checkpointed (after which it
+can be overwritten). Then, once the second wal file has grown large enough and
+the first wal file has been checkpointed, writers switch back to the first wal
+file. And so on.
+
+## Application Programming
+
+From the point of view of the user, the main differences between wal and 
+wal2 mode are to do with checkpointing:
+
+  * In wal mode, a checkpoint may be attempted at any time. In wal2 
+    mode, the checkpointer has to wait until writers have switched 
+    to the "other" wal file before a checkpoint can take place.
+
+  * In wal mode, the wal-hook (callback registered using
+    sqlite3_wal_hook()) is invoked after a transaction is committed
+    with the total number of pages in the wal file as an argument. In wal2
+    mode, the argument is either the total number of uncheckpointed pages in
+    both wal files, or - if the "other" wal file is empty or already
+    checkpointed - 0.
+
+Clients are recommended to use the same strategies for checkpointing wal2 mode
+databases as for wal databases - by registering a wal-hook using
+sqlite3_wal_hook() and attempting a checkpoint when the parameter
+exceeds a certain threshold.
+
+However, it should be noted that although the wal-hook is invoked after each
+transaction is committed to disk and database locks released, it is still
+invoked from within the sqlite3_step() call used to execute the "COMMIT"
+command. In BEGIN CONCURRENT systems, where the "COMMIT" is often protected by
+an application mutex, this may reduce concurrency. In such systems, instead of
+executing a checkpoint from within the wal-hook, a thread might defer this
+action until after the application mutex has been released.
+
+

ext/fts5/fts5_main.c

@@ -1732,6 +1732,7 @@ static int fts5UpdateMethod(
     }
   }
 
+  sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
   pTab->p.pConfig->pzErrmsg = 0;
   return rc;
 }

ext/fts5/test/fts5concurrent.test

@@ -0,0 +1,55 @@
+# 2022 May 09
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5concurrent
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(line, tokenize=trigram);
+}
+
+do_execsql_test 1.1 {
+  BEGIN CONCURRENT;
+    INSERT INTO ft VALUES( hex(randomblob(50)) );
+  COMMIT
+} {}
+
+do_execsql_test 1.2 {
+  BEGIN CONCURRENT;
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50
+    )
+    INSERT INTO ft SELECT hex(randomblob(50)) FROM s;
+
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50
+    )
+    INSERT INTO ft SELECT hex(randomblob(50)) FROM s;
+
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50
+    )
+    INSERT INTO ft SELECT hex(randomblob(50)) FROM s;
+  COMMIT;
+}
+
+
+finish_test
+

ext/misc/bgckpt.c

@@ -0,0 +1,244 @@
+/*
+** 2017-10-11
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+*/
+
+#if !defined(SQLITE_TEST) || defined(SQLITE_OS_UNIX)
+
+#include "sqlite3.h"
+#include <string.h>
+#include <pthread.h>
+
+/*
+** API declarations.
+*/
+typedef struct Checkpointer Checkpointer;
+int sqlite3_bgckpt_create(const char *zFilename, Checkpointer **pp);
+int sqlite3_bgckpt_checkpoint(Checkpointer *p, int bBlock);
+void sqlite3_bgckpt_destroy(Checkpointer *p);
+
+
+struct Checkpointer {
+  sqlite3 *db;                    /* Database handle */
+
+  pthread_t thread;               /* Background thread */
+  pthread_mutex_t mutex;
+  pthread_cond_t cond;
+
+  int rc;                         /* Error from "PRAGMA wal_checkpoint" */
+  int bCkpt;                      /* True if checkpoint requested */
+  int bExit;                      /* True if exit requested */
+};
+
+static void *bgckptThreadMain(void *pCtx){
+  int rc = SQLITE_OK;
+  Checkpointer *p = (Checkpointer*)pCtx;
+
+  while( rc==SQLITE_OK ){
+    int bExit;
+
+    pthread_mutex_lock(&p->mutex);
+    if( p->bCkpt==0 && p->bExit==0 ){
+      pthread_cond_wait(&p->cond, &p->mutex);
+    }
+    p->bCkpt = 0;
+    bExit = p->bExit;
+    pthread_mutex_unlock(&p->mutex);
+
+    if( bExit ) break;
+    rc = sqlite3_exec(p->db, "PRAGMA wal_checkpoint", 0, 0, 0);
+    if( rc==SQLITE_BUSY ){
+      rc = SQLITE_OK;
+    }
+  }
+
+  pthread_mutex_lock(&p->mutex);
+  p->rc = rc;
+  pthread_mutex_unlock(&p->mutex);
+  return 0;
+}
+
+void sqlite3_bgckpt_destroy(Checkpointer *p){
+  if( p ){
+    void *ret = 0;
+
+    /* Signal the background thread to exit */
+    pthread_mutex_lock(&p->mutex);
+    p->bExit = 1;
+    pthread_cond_broadcast(&p->cond);
+    pthread_mutex_unlock(&p->mutex);
+
+    pthread_join(p->thread, &ret);
+    sqlite3_close(p->db);
+    sqlite3_free(p);
+  }
+}
+
+
+int sqlite3_bgckpt_create(const char *zFilename, Checkpointer **pp){
+  Checkpointer *pNew = 0;
+  int rc;
+
+  pNew = (Checkpointer*)sqlite3_malloc(sizeof(Checkpointer));
+  if( pNew==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    memset(pNew, 0, sizeof(Checkpointer));
+    rc = sqlite3_open(zFilename, &pNew->db);
+  }
+
+  if( rc==SQLITE_OK ){
+    pthread_mutex_init(&pNew->mutex, 0);
+    pthread_cond_init(&pNew->cond, 0);
+    pthread_create(&pNew->thread, 0, bgckptThreadMain, (void*)pNew);
+  }
+
+  if( rc!=SQLITE_OK ){
+    sqlite3_bgckpt_destroy(pNew);
+    pNew = 0;
+  }
+  *pp = pNew;
+  return rc;
+}
+
+int sqlite3_bgckpt_checkpoint(Checkpointer *p, int bBlock){
+  int rc;
+  pthread_mutex_lock(&p->mutex);
+  rc = p->rc;
+  if( rc==SQLITE_OK ){
+    p->bCkpt = 1;
+    pthread_cond_broadcast(&p->cond);
+  }
+  pthread_mutex_unlock(&p->mutex);
+  return rc;
+}
+
+#ifdef SQLITE_TEST
+
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
+
+const char *sqlite3ErrName(int rc);
+
+static void SQLITE_TCLAPI bgckpt_del(void * clientData){
+  Checkpointer *pCkpt = (Checkpointer*)clientData;
+  sqlite3_bgckpt_destroy(pCkpt);
+}
+
+/*
+** Tclcmd: $ckpt SUBCMD ...
+*/
+static int SQLITE_TCLAPI bgckpt_obj_cmd(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  Checkpointer *pCkpt = (Checkpointer*)clientData;
+  const char *aCmd[] = { "checkpoint", "destroy", 0 };
+  int iCmd;
+
+  if( objc<2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "SUBCMD ...");
+    return TCL_ERROR;
+  }
+
+  if( Tcl_GetIndexFromObj(interp, objv[1], aCmd, "sub-command", 0, &iCmd) ){
+    return TCL_ERROR;
+  }
+
+  switch( iCmd ){
+    case 0: {
+      int rc;
+      int bBlock = 0;
+
+      if( objc>3 ){
+        Tcl_WrongNumArgs(interp, 2, objv, "?BLOCKING?");
+        return TCL_ERROR;
+      }
+      if( objc==3 && Tcl_GetBooleanFromObj(interp, objv[2], &bBlock) ){
+        return TCL_ERROR;
+      }
+
+      rc = sqlite3_bgckpt_checkpoint(pCkpt, bBlock);
+      if( rc!=SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+        return TCL_ERROR;
+      }
+      break;
+    }
+
+    case 1: {
+      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
+      break;
+    }
+  }
+
+  return TCL_OK;
+}
+
+/*
+** Tclcmd: bgckpt CMDNAME FILENAME
+*/
+static int SQLITE_TCLAPI bgckpt_cmd(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  const char *zCmd;
+  const char *zFilename;
+  int rc;
+  Checkpointer *pCkpt;
+
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "CMDNAME FILENAME");
+    return TCL_ERROR;
+  }
+  zCmd = Tcl_GetString(objv[1]);
+  zFilename = Tcl_GetString(objv[2]);
+
+  rc = sqlite3_bgckpt_create(zFilename, &pCkpt);
+  if( rc!=SQLITE_OK ){
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+    return TCL_ERROR;
+  }
+
+  Tcl_CreateObjCommand(interp, zCmd, bgckpt_obj_cmd, (void*)pCkpt, bgckpt_del);
+  Tcl_SetObjResult(interp, objv[1]);
+  return TCL_OK;
+}
+
+int Bgckpt_Init(Tcl_Interp *interp){
+  Tcl_CreateObjCommand(interp, "bgckpt", bgckpt_cmd, 0, 0);
+  return TCL_OK;
+}
+#endif   /* SQLITE_TEST */
+
+#else
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
+int Bgckpt_Init(Tcl_Interp *interp){ return TCL_OK; }
+#endif
+

ext/session/changebatch1.test

@@ -0,0 +1,222 @@
+# 2016 August 23
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source $testdir/tester.tcl
+ifcapable !session {finish_test; return}
+
+set testprefix changebatch1
+
+
+proc sql_to_changeset {method sql} {
+  sqlite3session S db main
+  S attach *
+  execsql $sql
+  set ret [S $method]
+  S delete
+  return $ret
+}
+
+proc do_changebatch_test {tn method args} {
+  set C [list]
+  foreach a $args {
+    lappend C [sql_to_changeset $method $a]
+  }
+
+  sqlite3changebatch cb db
+  set i 1
+  foreach ::cs [lrange $C 0 end-1] {
+    set rc [cb add $::cs]
+    if {$rc!="SQLITE_OK"} { error "expected SQLITE_OK, got $rc (i=$i)" }
+    incr i
+  }
+
+  set ::cs [lindex $C end]
+  do_test $tn { cb add [set ::cs] } SQLITE_CONSTRAINT
+  cb delete
+}
+
+proc do_changebatch_test1 {tn args} {
+  uplevel do_changebatch_test $tn changeset $args
+}
+proc do_changebatch_test2 {tn args} {
+  uplevel do_changebatch_test $tn fullchangeset $args
+}
+
+#-------------------------------------------------------------------------
+# The body of the following loop contains tests for database schemas
+# that do not feature multi-column UNIQUE constraints. In this case
+# it doesn't matter if the changesets are generated using
+# sqlite3session_changeset() or sqlite3session_fullchangeset().
+#
+foreach {tn testfunction} {
+  1 do_changebatch_test1
+  2 do_changebatch_test2
+} {
+  reset_db
+
+  #-------------------------------------------------------------------------
+  #
+  do_execsql_test $tn.1.0 {
+    CREATE TABLE t1(a PRIMARY KEY, b);
+  }
+  
+  $testfunction $tn.1.1 {
+    INSERT INTO t1 VALUES(1, 1);
+  } {
+    DELETE FROM t1 WHERE a=1;
+  }
+  
+  do_execsql_test $tn.1.2.0 {
+    INSERT INTO t1 VALUES(1, 1);
+    INSERT INTO t1 VALUES(2, 2);
+    INSERT INTO t1 VALUES(3, 3);
+  }
+  $testfunction $tn.1.2.1 {
+    DELETE FROM t1 WHERE a=2;
+  } {
+    INSERT INTO t1 VALUES(2, 2);
+  }
+  
+  #-------------------------------------------------------------------------
+  #
+  do_execsql_test $tn.2.0 {
+    CREATE TABLE x1(a, b PRIMARY KEY, c UNIQUE);
+    CREATE TABLE x2(a PRIMARY KEY, b UNIQUE, c UNIQUE);
+    CREATE INDEX x1a ON x1(a);
+  
+    INSERT INTO x1 VALUES(1, 1, 'a');
+    INSERT INTO x1 VALUES(1, 2, 'b');
+    INSERT INTO x1 VALUES(1, 3, 'c');
+  }
+  
+  $testfunction $tn.2.1 {
+    DELETE FROM x1 WHERE b=2;
+  } {
+    UPDATE x1 SET c='b' WHERE b=3;
+  }
+  
+  $testfunction $tn.2.2 {
+    DELETE FROM x1 WHERE b=1;
+  } {
+    INSERT INTO x1 VALUES(1, 5, 'a');
+  }
+
+  set L [list]
+  for {set i 1000} {$i < 10000} {incr i} {
+    lappend L "INSERT INTO x2 VALUES($i, $i, 'x' || $i)"
+  }
+  lappend L "DELETE FROM x2 WHERE b=1005"
+  $testfunction $tn.2.3 {*}$L
+
+  execsql { INSERT INTO x1 VALUES('f', 'f', 'f') }
+  $testfunction $tn.2.4 {
+    INSERT INTO x2 VALUES('f', 'f', 'f');
+  } {
+    INSERT INTO x1 VALUES('g', 'g', 'g');
+  } {
+    DELETE FROM x1 WHERE b='f';
+  } {
+    INSERT INTO x2 VALUES('g', 'g', 'g');
+  } {
+    INSERT INTO x1 VALUES('f', 'f', 'f');
+  }
+
+  execsql {
+    DELETE FROM x1;
+    INSERT INTO x1 VALUES(1.5, 1.5, 1.5);
+  }
+  $testfunction $tn.2.5 {
+    DELETE FROM x1 WHERE b BETWEEN 1 AND 2;
+  } {
+    INSERT INTO x1 VALUES(2.5, 2.5, 2.5);
+  } {
+    INSERT INTO x1 VALUES(1.5, 1.5, 1.5);
+  }
+
+  execsql {
+    DELETE FROM x2;
+    INSERT INTO x2 VALUES(X'abcd', X'1234', X'7890');
+    INSERT INTO x2 VALUES(X'0000', X'0000', X'0000');
+  }
+  breakpoint
+  $testfunction $tn.2.6 {
+    UPDATE x2 SET c = X'1234' WHERE a=X'abcd';
+    INSERT INTO x2 VALUES(X'1234', X'abcd', X'7890');
+  } {
+    DELETE FROM x2 WHERE b=X'0000';
+  } {
+    INSERT INTO x2 VALUES(1, X'0000', NULL);
+  }
+}
+
+#-------------------------------------------------------------------------
+# Test some multi-column UNIQUE constraints. First Using _changeset() to
+# demonstrate the problem, then using _fullchangeset() to show that it has
+# been fixed.
+#
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE y1(a PRIMARY KEY, b, c, UNIQUE(b, c));
+  INSERT INTO y1 VALUES(1, 1, 1);
+  INSERT INTO y1 VALUES(2, 2, 2);
+  INSERT INTO y1 VALUES(3, 3, 3);
+  INSERT INTO y1 VALUES(4, 3, 4);
+  BEGIN;
+}
+
+do_test 3.1.1 {
+  set c1 [sql_to_changeset changeset { DELETE FROM y1 WHERE a=4    }]
+  set c2 [sql_to_changeset changeset { UPDATE y1 SET c=4 WHERE a=3 }]
+  sqlite3changebatch cb db
+  cb add $c1
+  cb add $c2
+} {SQLITE_OK}
+do_test 3.1.2 {
+  cb delete
+  execsql ROLLBACK
+} {}
+
+do_test 3.1.1 {
+  set c1 [sql_to_changeset fullchangeset { DELETE FROM y1 WHERE a=4    }]
+  set c2 [sql_to_changeset fullchangeset { UPDATE y1 SET c=4 WHERE a=3 }]
+  sqlite3changebatch cb db
+  cb add $c1
+  cb add $c2
+} {SQLITE_CONSTRAINT}
+do_test 3.1.2 {
+  cb delete
+} {}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 4.0 {
+  CREATE TABLE t1(x, y, z, PRIMARY KEY(x, y), UNIQUE(z));
+}
+
+do_test 4.1 {
+  set c1 [sql_to_changeset fullchangeset { INSERT INTO t1 VALUES(1, 2, 3) }]
+  execsql {
+    DROP TABLE t1;
+    CREATE TABLE t1(w, x, y, z, PRIMARY KEY(x, y), UNIQUE(z));
+  }
+  sqlite3changebatch cb db
+  list [catch { cb add $c1 } msg] $msg
+} {1 SQLITE_RANGE}
+
+
+
+finish_test

ext/session/changebatchfault.test

@@ -0,0 +1,42 @@
+# 2011 Mar 21
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the session module.
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source [file join [file dirname [info script]] session_common.tcl]
+source $testdir/tester.tcl
+ifcapable !session {finish_test; return}
+set testprefix changebatchfault
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c PRIMARY KEY, UNIQUE(a, b));
+  INSERT INTO t1 VALUES('a', 'a', 'a');
+  INSERT INTO t1 VALUES('b', 'b', 'b');
+}
+
+set ::c1 [changeset_from_sql { delete from t1 where c='a' }]
+set ::c2 [changeset_from_sql { insert into t1 values('c', 'c', 'c') }]
+
+do_faultsim_test 1 -faults oom-* -body {
+  sqlite3changebatch cb db
+  cb add $::c1
+  cb add $::c2
+} -test {
+  faultsim_test_result {0 SQLITE_OK} {1 SQLITE_NOMEM}
+  catch { cb delete }
+}
+
+
+finish_test

ext/session/sessionH.test

@@ -29,7 +29,7 @@ do_test 1.0 {
     WITH s(i) AS (
       VALUES(1) UNION ALL SELECT i+1 FROM s WHERe i<10000
     )
-    INSERT INTO t1 SELECT 'abcde', randomblob(16), i FROM s;
+    INSERT INTO t1 SELECT 'abcde', randomblob(18), i FROM s;
   }
   compare_db db db2
 } {}

ext/session/sqlite3changebatch.c

@@ -0,0 +1,485 @@
+
+#if !defined(SQLITE_TEST) || (defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK))
+
+#include "sqlite3session.h"
+#include "sqlite3changebatch.h"
+
+#include <assert.h>
+#include <string.h>
+
+typedef struct BatchTable BatchTable;
+typedef struct BatchIndex BatchIndex;
+typedef struct BatchIndexEntry BatchIndexEntry;
+typedef struct BatchHash BatchHash;
+
+struct sqlite3_changebatch {
+  sqlite3 *db;                    /* Database handle used to read schema */
+  BatchTable *pTab;               /* First in linked list of tables */
+  int iChangesetId;               /* Current changeset id */
+  int iNextIdxId;                 /* Next available index id */
+  int nEntry;                     /* Number of entries in hash table */
+  int nHash;                      /* Number of hash buckets */
+  BatchIndexEntry **apHash;       /* Array of hash buckets */
+};
+
+struct BatchTable {
+  BatchIndex *pIdx;               /* First in linked list of UNIQUE indexes */
+  BatchTable *pNext;              /* Next table */
+  char zTab[1];                   /* Table name */
+};
+
+struct BatchIndex {
+  BatchIndex *pNext;              /* Next index on same table */
+  int iId;                        /* Index id (assigned internally) */
+  int bPk;                        /* True for PK index */
+  int nCol;                       /* Size of aiCol[] array */
+  int *aiCol;                     /* Array of columns that make up index */
+};
+
+struct BatchIndexEntry {
+  BatchIndexEntry *pNext;         /* Next colliding hash table entry */
+  int iChangesetId;               /* Id of associated changeset */
+  int iIdxId;                     /* Id of index this key is from */
+  int szRecord;
+  char aRecord[1];
+};
+
+/*
+** Allocate and zero a block of nByte bytes. Must be freed using cbFree().
+*/
+static void *cbMalloc(int *pRc, int nByte){
+  void *pRet;
+
+  if( *pRc ){
+    pRet = 0;
+  }else{
+    pRet = sqlite3_malloc(nByte);
+    if( pRet ){
+      memset(pRet, 0, nByte);
+    }else{
+      *pRc = SQLITE_NOMEM;
+    }
+  }
+
+  return pRet;
+}
+
+/*
+** Free an allocation made by cbMalloc().
+*/
+static void cbFree(void *p){
+  sqlite3_free(p);
+}
+
+/*
+** Return the hash bucket that pEntry belongs in.
+*/
+static int cbHash(sqlite3_changebatch *p, BatchIndexEntry *pEntry){
+  unsigned int iHash = (unsigned int)pEntry->iIdxId;
+  unsigned char *pEnd = (unsigned char*)&pEntry->aRecord[pEntry->szRecord];
+  unsigned char *pIter;
+
+  for(pIter=(unsigned char*)pEntry->aRecord; pIter<pEnd; pIter++){
+    iHash += (iHash << 7) + *pIter;
+  }
+
+  return (int)(iHash % p->nHash);
+}
+
+/*
+** Resize the hash table.
+*/
+static int cbHashResize(sqlite3_changebatch *p){
+  int rc = SQLITE_OK;
+  BatchIndexEntry **apNew;
+  int nNew = (p->nHash ? p->nHash*2 : 512);
+  int i;
+
+  apNew = cbMalloc(&rc, sizeof(BatchIndexEntry*) * nNew);
+  if( rc==SQLITE_OK ){
+    int nHash = p->nHash;
+    p->nHash = nNew;
+    for(i=0; i<nHash; i++){
+      BatchIndexEntry *pEntry;
+      while( (pEntry=p->apHash[i])!=0 ){
+        int iHash = cbHash(p, pEntry);
+        p->apHash[i] = pEntry->pNext;
+        pEntry->pNext = apNew[iHash];
+        apNew[iHash] = pEntry;
+      }
+    }
+
+    cbFree(p->apHash);
+    p->apHash = apNew;
+  }
+
+  return rc;
+}
+
+
+/*
+** Allocate a new sqlite3_changebatch object.
+*/
+int sqlite3changebatch_new(sqlite3 *db, sqlite3_changebatch **pp){
+  sqlite3_changebatch *pRet;
+  int rc = SQLITE_OK;
+  *pp = pRet = (sqlite3_changebatch*)cbMalloc(&rc, sizeof(sqlite3_changebatch));
+  if( pRet ){
+    pRet->db = db;
+  }
+  return rc;
+}
+
+/*
+** Add a BatchIndex entry for index zIdx to table pTab.
+*/
+static int cbAddIndex(
+  sqlite3_changebatch *p, 
+  BatchTable *pTab, 
+  const char *zIdx, 
+  int bPk
+){
+  int nCol = 0;
+  sqlite3_stmt *pIndexInfo = 0;
+  BatchIndex *pNew = 0;
+  int rc;
+  char *zIndexInfo;
+
+  zIndexInfo = (char*)sqlite3_mprintf("PRAGMA main.index_info = %Q", zIdx);
+  if( zIndexInfo ){
+    rc = sqlite3_prepare_v2(p->db, zIndexInfo, -1, &pIndexInfo, 0);
+    sqlite3_free(zIndexInfo);
+  }else{
+    rc = SQLITE_NOMEM;
+  }
+
+  if( rc==SQLITE_OK ){
+    while( SQLITE_ROW==sqlite3_step(pIndexInfo) ){ nCol++; }
+    rc = sqlite3_reset(pIndexInfo);
+  }
+
+  pNew = (BatchIndex*)cbMalloc(&rc, sizeof(BatchIndex) + sizeof(int) * nCol);
+  if( rc==SQLITE_OK ){
+    pNew->nCol = nCol;
+    pNew->bPk = bPk;
+    pNew->aiCol = (int*)&pNew[1];
+    pNew->iId = p->iNextIdxId++;
+    while( SQLITE_ROW==sqlite3_step(pIndexInfo) ){ 
+      int i = sqlite3_column_int(pIndexInfo, 0);
+      int j = sqlite3_column_int(pIndexInfo, 1);
+      pNew->aiCol[i] = j;
+    }
+    rc = sqlite3_reset(pIndexInfo);
+  }
+
+  if( rc==SQLITE_OK ){
+    pNew->pNext = pTab->pIdx;
+    pTab->pIdx = pNew;
+  }else{
+    cbFree(pNew);
+  }
+  sqlite3_finalize(pIndexInfo);
+
+  return rc;
+}
+
+/*
+** Free the object passed as the first argument.
+*/
+static void cbFreeTable(BatchTable *pTab){
+  BatchIndex *pIdx;
+  BatchIndex *pIdxNext;
+  for(pIdx=pTab->pIdx; pIdx; pIdx=pIdxNext){
+    pIdxNext = pIdx->pNext;
+    cbFree(pIdx);
+  }
+  cbFree(pTab);
+}
+
+/*
+** Find or create the BatchTable object named zTab.
+*/
+static int cbFindTable(
+  sqlite3_changebatch *p, 
+  const char *zTab, 
+  BatchTable **ppTab
+){
+  BatchTable *pRet = 0;
+  int rc = SQLITE_OK;
+
+  for(pRet=p->pTab; pRet; pRet=pRet->pNext){
+    if( 0==sqlite3_stricmp(zTab, pRet->zTab) ) break;
+  }
+
+  if( pRet==0 ){
+    int nTab = strlen(zTab);
+    pRet = (BatchTable*)cbMalloc(&rc, nTab + sizeof(BatchTable));
+    if( pRet ){
+      sqlite3_stmt *pIndexList = 0;
+      char *zIndexList = 0;
+      int rc2;
+      memcpy(pRet->zTab, zTab, nTab);
+
+      zIndexList = sqlite3_mprintf("PRAGMA main.index_list = %Q", zTab);
+      if( zIndexList==0 ){
+        rc = SQLITE_NOMEM;
+      }else{
+        rc = sqlite3_prepare_v2(p->db, zIndexList, -1, &pIndexList, 0);
+        sqlite3_free(zIndexList);
+      }
+
+      while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pIndexList) ){
+        if( sqlite3_column_int(pIndexList, 2) ){
+          const char *zIdx = (const char*)sqlite3_column_text(pIndexList, 1);
+          const char *zTyp = (const char*)sqlite3_column_text(pIndexList, 3);
+          rc = cbAddIndex(p, pRet, zIdx, (zTyp[0]=='p'));
+        }
+      }
+      rc2 = sqlite3_finalize(pIndexList);
+      if( rc==SQLITE_OK ) rc = rc2;
+
+      if( rc==SQLITE_OK ){
+        pRet->pNext = p->pTab;
+        p->pTab = pRet;
+      }else{
+        cbFreeTable(pRet);
+        pRet = 0;
+      }
+    }
+  }
+
+  *ppTab = pRet;
+  return rc;
+}
+
+/*
+** Extract value iVal from the changeset iterator passed as the first
+** argument. Set *ppVal to point to the value before returning.
+**
+** This function attempts to extract the value using function xVal
+** (which is always either sqlite3changeset_new or sqlite3changeset_old).
+** If the call returns SQLITE_OK but does not supply an sqlite3_value*
+** pointer, an attempt to extract the value is made using the xFallback 
+** function.
+*/
+static int cbGetChangesetValue(
+  sqlite3_changeset_iter *pIter, 
+  int (*xVal)(sqlite3_changeset_iter*,int,sqlite3_value**),
+  int (*xFallback)(sqlite3_changeset_iter*,int,sqlite3_value**),
+  int iVal,
+  sqlite3_value **ppVal
+){
+  int rc = xVal(pIter, iVal, ppVal);
+  if( rc==SQLITE_OK && *ppVal==0 && xFallback ){
+    rc = xFallback(pIter, iVal, ppVal);
+  }
+  return rc;
+}
+
+static int cbAddToHash(
+  sqlite3_changebatch *p, 
+  sqlite3_changeset_iter *pIter, 
+  BatchIndex *pIdx, 
+  int (*xVal)(sqlite3_changeset_iter*,int,sqlite3_value**),
+  int (*xFallback)(sqlite3_changeset_iter*,int,sqlite3_value**),
+  int *pbConf
+){
+  BatchIndexEntry *pNew;
+  int sz = pIdx->nCol;
+  int i;
+  int iOut = 0;
+  int rc = SQLITE_OK;
+
+  for(i=0; rc==SQLITE_OK && i<pIdx->nCol; i++){
+    sqlite3_value *pVal;
+    rc = cbGetChangesetValue(pIter, xVal, xFallback, pIdx->aiCol[i], &pVal);
+    if( rc==SQLITE_OK ){
+      int eType = 0;
+      if( pVal ) eType = sqlite3_value_type(pVal);
+      switch( eType ){
+        case 0:
+        case SQLITE_NULL:
+          return SQLITE_OK;
+
+        case SQLITE_INTEGER:
+          sz += 8;
+          break;
+        case SQLITE_FLOAT:
+          sz += 8;
+          break;
+
+        default:
+          assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB );
+          sz += sqlite3_value_bytes(pVal);
+          break;
+      }
+    }
+  }
+
+  pNew = cbMalloc(&rc, sizeof(BatchIndexEntry) + sz);
+  if( pNew ){
+    pNew->iChangesetId = p->iChangesetId;
+    pNew->iIdxId = pIdx->iId;
+    pNew->szRecord = sz;
+
+    for(i=0; i<pIdx->nCol; i++){
+      int eType;
+      sqlite3_value *pVal;
+      rc = cbGetChangesetValue(pIter, xVal, xFallback, pIdx->aiCol[i], &pVal);
+      if( rc!=SQLITE_OK ) break;  /* coverage: condition is never true */
+      eType = sqlite3_value_type(pVal);
+      pNew->aRecord[iOut++] = eType;
+      switch( eType ){
+        case SQLITE_INTEGER: {
+          sqlite3_int64 i64 = sqlite3_value_int64(pVal);
+          memcpy(&pNew->aRecord[iOut], &i64, 8);
+          iOut += 8;
+          break;
+        }
+        case SQLITE_FLOAT: {
+          double d64 = sqlite3_value_double(pVal);
+          memcpy(&pNew->aRecord[iOut], &d64, sizeof(double));
+          iOut += sizeof(double);
+          break;
+        }
+
+        default: {
+          int nByte = sqlite3_value_bytes(pVal);
+          const char *z = (const char*)sqlite3_value_blob(pVal);
+          memcpy(&pNew->aRecord[iOut], z, nByte);
+          iOut += nByte;
+          break;
+        }
+      }
+    }
+  }
+
+  if( rc==SQLITE_OK && p->nEntry>=(p->nHash/2) ){
+    rc = cbHashResize(p);
+  }
+
+  if( rc==SQLITE_OK ){
+    BatchIndexEntry *pIter;
+    int iHash = cbHash(p, pNew);
+
+    assert( iHash>=0 && iHash<p->nHash );
+    for(pIter=p->apHash[iHash]; pIter; pIter=pIter->pNext){
+      if( pNew->szRecord==pIter->szRecord 
+       && 0==memcmp(pNew->aRecord, pIter->aRecord, pNew->szRecord)
+      ){
+        if( pNew->iChangesetId!=pIter->iChangesetId ){
+          *pbConf = 1;
+        }
+        cbFree(pNew);
+        pNew = 0;
+        break;
+      }
+    }
+
+    if( pNew ){
+      pNew->pNext = p->apHash[iHash];
+      p->apHash[iHash] = pNew;
+      p->nEntry++;
+    }
+  }else{
+    cbFree(pNew);
+  }
+
+  return rc;
+}
+
+
+/*
+** Add a changeset to the current batch.
+*/
+int sqlite3changebatch_add(sqlite3_changebatch *p, void *pBuf, int nBuf){
+  sqlite3_changeset_iter *pIter;  /* Iterator opened on pBuf/nBuf */
+  int rc;                         /* Return code */
+  int bConf = 0;                  /* Conflict was detected */
+
+  rc = sqlite3changeset_start(&pIter, nBuf, pBuf);
+  if( rc==SQLITE_OK ){
+    int rc2;
+    for(rc2 = sqlite3changeset_next(pIter);
+        rc2==SQLITE_ROW;
+        rc2 = sqlite3changeset_next(pIter)
+    ){
+      BatchTable *pTab;
+      BatchIndex *pIdx;
+      const char *zTab;           /* Table this change applies to */
+      int nCol;                   /* Number of columns in table */
+      int op;                     /* UPDATE, INSERT or DELETE */
+
+      sqlite3changeset_op(pIter, &zTab, &nCol, &op, 0);
+      assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE );
+
+      rc = cbFindTable(p, zTab, &pTab);
+      assert( pTab || rc!=SQLITE_OK );
+      if( pTab ){
+        for(pIdx=pTab->pIdx; pIdx && rc==SQLITE_OK; pIdx=pIdx->pNext){
+          if( op==SQLITE_UPDATE && pIdx->bPk ) continue;
+          if( op==SQLITE_UPDATE || op==SQLITE_DELETE ){
+            rc = cbAddToHash(p, pIter, pIdx, sqlite3changeset_old, 0, &bConf);
+          }
+          if( op==SQLITE_UPDATE || op==SQLITE_INSERT ){
+            rc = cbAddToHash(p, pIter, pIdx, 
+                sqlite3changeset_new, sqlite3changeset_old, &bConf
+            );
+          }
+        }
+      }
+      if( rc!=SQLITE_OK ) break;
+    }
+
+    rc2 = sqlite3changeset_finalize(pIter);
+    if( rc==SQLITE_OK ) rc = rc2;
+  }
+
+  if( rc==SQLITE_OK && bConf ){
+    rc = SQLITE_CONSTRAINT;
+  }
+  p->iChangesetId++;
+  return rc;
+}
+
+/*
+** Zero an existing changebatch object.
+*/
+void sqlite3changebatch_zero(sqlite3_changebatch *p){
+  int i;
+  for(i=0; i<p->nHash; i++){
+    BatchIndexEntry *pEntry;
+    BatchIndexEntry *pNext;
+    for(pEntry=p->apHash[i]; pEntry; pEntry=pNext){
+      pNext = pEntry->pNext;
+      cbFree(pEntry);
+    }
+  }
+  cbFree(p->apHash);
+  p->nHash = 0;
+  p->apHash = 0;
+}
+
+/*
+** Delete a changebatch object.
+*/
+void sqlite3changebatch_delete(sqlite3_changebatch *p){
+  BatchTable *pTab;
+  BatchTable *pTabNext;
+
+  sqlite3changebatch_zero(p);
+  for(pTab=p->pTab; pTab; pTab=pTabNext){
+    pTabNext = pTab->pNext;
+    cbFreeTable(pTab);
+  }
+  cbFree(p);
+}
+
+/*
+** Return the db handle.
+*/
+sqlite3 *sqlite3changebatch_db(sqlite3_changebatch *p){
+  return p->db;
+}
+
+#endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */

ext/session/sqlite3changebatch.h

@@ -0,0 +1,82 @@
+
+#if !defined(SQLITECHANGEBATCH_H_) 
+#define SQLITECHANGEBATCH_H_ 1
+
+typedef struct sqlite3_changebatch sqlite3_changebatch;
+
+/*
+** Create a new changebatch object for detecting conflicts between
+** changesets associated with a schema equivalent to that of the "main"
+** database of the open database handle db passed as the first
+** parameter. It is the responsibility of the caller to ensure that
+** the database handle is not closed until after the changebatch
+** object has been deleted.
+**
+** A changebatch object is used to detect batches of non-conflicting
+** changesets. Changesets that do not conflict may be applied to the 
+** target database in any order without affecting the final state of 
+** the database.
+**
+** The changebatch object only works reliably if PRIMARY KEY and UNIQUE
+** constraints on tables affected by the changesets use collation
+** sequences that are equivalent to built-in collation sequence 
+** BINARY for the == operation.
+**
+** If successful, SQLITE_OK is returned and (*pp) set to point to
+** the new changebatch object. If an error occurs, an SQLite error
+** code is returned and the final value of (*pp) is undefined.
+*/
+int sqlite3changebatch_new(sqlite3 *db, sqlite3_changebatch **pp);
+
+/*
+** Argument p points to a buffer containing a changeset n bytes in
+** size. Assuming no error occurs, this function returns SQLITE_OK
+** if the changeset does not conflict with any changeset passed 
+** to an sqlite3changebatch_add() call made on the same 
+** sqlite3_changebatch* handle since the most recent call to
+** sqlite3changebatch_zero(). If the changeset does conflict with 
+** an earlier such changeset, SQLITE_CONSTRAINT is returned. Or, 
+** if an error occurs, some other SQLite error code may be returned.
+**
+** One changeset is said to conflict with another if
+** either:
+**
+**   * the two changesets contain operations (INSERT, UPDATE or 
+**     DELETE) on the same row, identified by primary key, or
+**
+**   * the two changesets contain operations (INSERT, UPDATE or 
+**     DELETE) on rows with identical values in any combination 
+**     of fields constrained by a UNIQUE constraint.
+**
+** Even if this function returns SQLITE_CONFLICT, the current
+** changeset is added to the internal data structures - so future
+** calls to this function may conflict with it. If this function
+** returns any result code other than SQLITE_OK or SQLITE_CONFLICT,
+** the result of any future call to sqlite3changebatch_add() is
+** undefined.
+**
+** Only changesets may be passed to this function. Passing a 
+** patchset to this function results in an SQLITE_MISUSE error.
+*/
+int sqlite3changebatch_add(sqlite3_changebatch*, void *p, int n);
+
+/*
+** Zero a changebatch object. This causes the records of all earlier 
+** calls to sqlite3changebatch_add() to be discarded.
+*/
+void sqlite3changebatch_zero(sqlite3_changebatch*);
+
+/*
+** Return a copy of the first argument passed to the sqlite3changebatch_new()
+** call used to create the changebatch object passed as the only argument
+** to this function.
+*/
+sqlite3 *sqlite3changebatch_db(sqlite3_changebatch*);
+
+/*
+** Delete a changebatch object.
+*/
+void sqlite3changebatch_delete(sqlite3_changebatch*);
+
+#endif /* !defined(SQLITECHANGEBATCH_H_) */
+

ext/session/sqlite3session.c

@@ -25,6 +25,13 @@ typedef struct SessionInput SessionInput;
 # endif
 #endif
 
+/*
+** The three different types of changesets generated.
+*/
+#define SESSIONS_PATCHSET      0
+#define SESSIONS_CHANGESET     1
+#define SESSIONS_FULLCHANGESET 2
+
 static int sessions_strm_chunk_size = SESSIONS_STRM_CHUNK_SIZE;
 
 typedef struct SessionHook SessionHook;
@@ -2233,7 +2240,7 @@ static void sessionAppendCol(
 */ 
 static int sessionAppendUpdate(
   SessionBuffer *pBuf,            /* Buffer to append to */
-  int bPatchset,                  /* True for "patchset", 0 for "changeset" */
+  int ePatchset,                  /* True for "patchset", 0 for "changeset" */
   sqlite3_stmt *pStmt,            /* Statement handle pointing at new row */
   SessionChange *p,               /* Object containing old values */
   u8 *abPK                        /* Boolean array - true for PK columns */
@@ -2297,8 +2304,8 @@ static int sessionAppendUpdate(
 
     /* Add a field to the old.* record. This is omitted if this modules is
     ** currently generating a patchset. */
-    if( bPatchset==0 ){
-      if( bChanged || abPK[i] ){
+    if( ePatchset!=SESSIONS_PATCHSET ){
+      if( ePatchset==SESSIONS_FULLCHANGESET || bChanged || abPK[i] ){
         sessionAppendBlob(pBuf, pCsr, nAdvance, &rc);
       }else{
         sessionAppendByte(pBuf, 0, &rc);
@@ -2307,7 +2314,7 @@ static int sessionAppendUpdate(
 
     /* Add a field to the new.* record. Or the only record if currently
     ** generating a patchset.  */
-    if( bChanged || (bPatchset && abPK[i]) ){
+    if( bChanged || (ePatchset==SESSIONS_PATCHSET && abPK[i]) ){
       sessionAppendCol(&buf2, pStmt, i, &rc);
     }else{
       sessionAppendByte(&buf2, 0, &rc);
@@ -2333,7 +2340,7 @@ static int sessionAppendUpdate(
 */
 static int sessionAppendDelete(
   SessionBuffer *pBuf,            /* Buffer to append to */
-  int bPatchset,                  /* True for "patchset", 0 for "changeset" */
+  int eChangeset,                 /* One of SESSIONS_CHANGESET etc. */
   SessionChange *p,               /* Object containing old values */
   int nCol,                       /* Number of columns in table */
   u8 *abPK                        /* Boolean array - true for PK columns */
@@ -2343,7 +2350,7 @@ static int sessionAppendDelete(
   sessionAppendByte(pBuf, SQLITE_DELETE, &rc);
   sessionAppendByte(pBuf, p->bIndirect, &rc);
 
-  if( bPatchset==0 ){
+  if( eChangeset!=SESSIONS_PATCHSET ){
     sessionAppendBlob(pBuf, p->aRecord, p->nRecord, &rc);
   }else{
     int i;
@@ -2516,12 +2523,12 @@ static int sessionSelectBind(
 */
 static void sessionAppendTableHdr(
   SessionBuffer *pBuf,            /* Append header to this buffer */
-  int bPatchset,                  /* Use the patchset format if true */
+  int ePatchset,                  /* Use the patchset format if true */
   SessionTable *pTab,             /* Table object to append header for */
   int *pRc                        /* IN/OUT: Error code */
 ){
   /* Write a table header */
-  sessionAppendByte(pBuf, (bPatchset ? 'P' : 'T'), pRc);
+  sessionAppendByte(pBuf, (ePatchset==SESSIONS_PATCHSET) ? 'P' : 'T', pRc);
   sessionAppendVarint(pBuf, pTab->nCol, pRc);
   sessionAppendBlob(pBuf, pTab->abPK, pTab->nCol, pRc);
   sessionAppendBlob(pBuf, (u8 *)pTab->zName, (int)strlen(pTab->zName)+1, pRc);
@@ -2539,7 +2546,7 @@ static void sessionAppendTableHdr(
 */
 static int sessionGenerateChangeset(
   sqlite3_session *pSession,      /* Session object */
-  int bPatchset,                  /* True for patchset, false for changeset */
+  int ePatchset,                  /* One of SESSIONS_CHANGESET etc. */
   int (*xOutput)(void *pOut, const void *pData, int nData),
   void *pOut,                     /* First argument for xOutput */
   int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
@@ -2586,7 +2593,7 @@ static int sessionGenerateChangeset(
       }
 
       /* Write a table header */
-      sessionAppendTableHdr(&buf, bPatchset, pTab, &rc);
+      sessionAppendTableHdr(&buf, ePatchset, pTab, &rc);
 
       /* Build and compile a statement to execute: */
       if( rc==SQLITE_OK ){
@@ -2611,10 +2618,10 @@ static int sessionGenerateChangeset(
               }
             }else{
               assert( abPK!=0 );  /* Because sessionSelectStmt() returned ok */
-              rc = sessionAppendUpdate(&buf, bPatchset, pSel, p, abPK);
+              rc = sessionAppendUpdate(&buf, ePatchset, pSel, p, abPK);
             }
           }else if( p->op!=SQLITE_INSERT ){
-            rc = sessionAppendDelete(&buf, bPatchset, p, nCol, abPK);
+            rc = sessionAppendDelete(&buf, ePatchset, p, nCol, abPK);
           }
           if( rc==SQLITE_OK ){
             rc = sqlite3_reset(pSel);
@@ -2674,7 +2681,8 @@ int sqlite3session_changeset(
   int rc;
 
   if( pnChangeset==0 || ppChangeset==0 ) return SQLITE_MISUSE;
-  rc = sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset,ppChangeset);
+  rc = sessionGenerateChangeset(
+      pSession, SESSIONS_CHANGESET, 0, 0, pnChangeset, ppChangeset);
   assert( rc || pnChangeset==0 
        || pSession->bEnableSize==0 || *pnChangeset<=pSession->nMaxChangesetSize 
   );
@@ -2690,7 +2698,8 @@ int sqlite3session_changeset_strm(
   void *pOut
 ){
   if( xOutput==0 ) return SQLITE_MISUSE;
-  return sessionGenerateChangeset(pSession, 0, xOutput, pOut, 0, 0);
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_CHANGESET, xOutput, pOut, 0, 0);
 }
 
 /*
@@ -2702,7 +2711,8 @@ int sqlite3session_patchset_strm(
   void *pOut
 ){
   if( xOutput==0 ) return SQLITE_MISUSE;
-  return sessionGenerateChangeset(pSession, 1, xOutput, pOut, 0, 0);
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_PATCHSET, xOutput, pOut, 0, 0);
 }
 
 /*
@@ -2718,9 +2728,20 @@ int sqlite3session_patchset(
   void **ppPatchset               /* OUT: Buffer containing changeset */
 ){
   if( pnPatchset==0 || ppPatchset==0 ) return SQLITE_MISUSE;
-  return sessionGenerateChangeset(pSession, 1, 0, 0, pnPatchset, ppPatchset);
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_PATCHSET, 0, 0, pnPatchset, ppPatchset);
 }
 
+int sqlite3session_fullchangeset(
+  sqlite3_session *pSession,      /* Session object */
+  int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
+  void **ppChangeset              /* OUT: Buffer containing changeset */
+){
+  return sessionGenerateChangeset(
+      pSession, SESSIONS_FULLCHANGESET, 0, 0, pnChangeset, ppChangeset);
+}
+
+
 /*
 ** Enable or disable the session object passed as the first argument.
 */
@@ -5263,10 +5284,11 @@ static int sessionChangegroupOutput(
   ** hash tables attached to the SessionTable objects in list p->pList. 
   */
   for(pTab=pGrp->pList; rc==SQLITE_OK && pTab; pTab=pTab->pNext){
+    int eChangeset = pGrp->bPatch ? SESSIONS_PATCHSET : SESSIONS_CHANGESET;
     int i;
     if( pTab->nEntry==0 ) continue;
 
-    sessionAppendTableHdr(&buf, pGrp->bPatch, pTab, &rc);
+    sessionAppendTableHdr(&buf, eChangeset, pTab, &rc);
     for(i=0; i<pTab->nChange; i++){
       SessionChange *p;
       for(p=pTab->apChange[i]; p; p=p->pNext){

ext/session/sqlite3session.h

@@ -355,6 +355,32 @@ int sqlite3session_changeset(
   void **ppChangeset              /* OUT: Buffer containing changeset */
 );
 
+/*
+** CAPI3REF: Generate A Full Changeset From A Session Object
+**
+** This function is similar to sqlite3session_changeset(), except that for
+** each row affected by an UPDATE statement, all old.* values are recorded
+** as part of the changeset, not just those modified.
+*/
+int sqlite3session_fullchangeset(
+  sqlite3_session *pSession,      /* Session object */
+  int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
+  void **ppChangeset              /* OUT: Buffer containing changeset */
+);
+
+/*
+** CAPI3REF: Generate A Full Changeset From A Session Object
+**
+** This function is similar to sqlite3session_changeset(), except that for
+** each row affected by an UPDATE statement, all old.* values are recorded
+** as part of the changeset, not just those modified.
+*/
+int sqlite3session_fullchangeset(
+  sqlite3_session *pSession,      /* Session object */
+  int *pnChangeset,               /* OUT: Size of buffer at *ppChangeset */
+  void **ppChangeset              /* OUT: Buffer containing changeset */
+);
+
 /*
 ** CAPI3REF: Return An Upper-limit For The Size Of The Changeset
 ** METHOD: sqlite3_session

ext/session/test_session.c

@@ -221,6 +221,7 @@ static int testStreamOutput(
 **          $session indirect INTEGER
 **          $session patchset
 **          $session table_filter SCRIPT
+**          $session fullchangeset
 */
 static int SQLITE_TCLAPI test_session_cmd(
   void *clientData,
@@ -234,20 +235,20 @@ static int SQLITE_TCLAPI test_session_cmd(
     const char *zSub;
     int nArg;
     const char *zMsg;
-    int iSub;
   } aSub[] = {
-    { "attach",       1, "TABLE",      }, /* 0 */
-    { "changeset",    0, "",           }, /* 1 */
-    { "delete",       0, "",           }, /* 2 */
-    { "enable",       1, "BOOL",       }, /* 3 */
-    { "indirect",     1, "BOOL",       }, /* 4 */
-    { "isempty",      0, "",           }, /* 5 */
-    { "table_filter", 1, "SCRIPT",     }, /* 6 */
+    { "attach",       1, "TABLE"       }, /* 0 */
+    { "changeset",    0, ""            }, /* 1 */
+    { "delete",       0, ""            }, /* 2 */
+    { "enable",       1, "BOOL"        }, /* 3 */
+    { "indirect",     1, "BOOL"        }, /* 4 */
+    { "isempty",      0, ""            }, /* 5 */
+    { "table_filter", 1, "SCRIPT"      }, /* 6 */
     { "patchset",     0, "",           }, /* 7 */
-    { "diff",         2, "FROMDB TBL", }, /* 8 */
-    { "memory_used",  0, "",           }, /* 9 */
-    { "changeset_size", 0, "",         }, /* 10 */
-    { "object_config_size", 1, "INTEGER", }, /* 11 */
+    { "diff",         2, "FROMDB TBL"  }, /* 8 */
+    { "fullchangeset",0, ""            }, /* 9 */
+    { "memory_used",  0, "",           }, /* 10 */
+    { "changeset_size", 0, "",         }, /* 11 */
+    { "object_config_size", 1, "INTEGER", }, /* 12 */
     { 0 }
   };
   int iSub;
@@ -277,10 +278,11 @@ static int SQLITE_TCLAPI test_session_cmd(
       break;
     }
 
+    case 9:        /* fullchangeset */
     case 7:        /* patchset */
     case 1: {      /* changeset */
       TestSessionsBlob o = {0, 0};
-      if( test_tcl_integer(interp, SESSION_STREAM_TCL_VAR) ){
+      if( iSub!=9 && test_tcl_integer(interp, SESSION_STREAM_TCL_VAR) ){
         void *pCtx = (void*)&o;
         if( iSub==7 ){
           rc = sqlite3session_patchset_strm(pSession, testStreamOutput, pCtx);
@@ -290,6 +292,8 @@ static int SQLITE_TCLAPI test_session_cmd(
       }else{
         if( iSub==7 ){
           rc = sqlite3session_patchset(pSession, &o.n, &o.p);
+        }else if( iSub==9 ){
+          rc = sqlite3session_fullchangeset(pSession, &o.n, &o.p);
         }else{
           rc = sqlite3session_changeset(pSession, &o.n, &o.p);
         }
@@ -304,6 +308,7 @@ static int SQLITE_TCLAPI test_session_cmd(
       break;
     }
 
+
     case 2:        /* delete */
       Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
       break;
@@ -354,18 +359,18 @@ static int SQLITE_TCLAPI test_session_cmd(
       break;
     }
 
-    case 9: {      /* memory_used */
+    case 10: {      /* memory_used */
       sqlite3_int64 nMalloc = sqlite3session_memory_used(pSession);
       Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nMalloc));
       break;
     }
 
-    case 10: {
+    case 11: {
       sqlite3_int64 nSize = sqlite3session_changeset_size(pSession);
       Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nSize));
       break;
     }
-    case 11: {
+    case 12: {
       int rc;
       int iArg;
       if( Tcl_GetIntFromObj(interp, objv[2], &iArg) ){
@@ -521,7 +526,7 @@ static int test_filter_handler(
 
   Tcl_DecrRefCount(pEval);
   return res;
-}  
+}
 
 static int test_conflict_handler(
   void *pCtx,                     /* Pointer to TestConflictHandler structure */
@@ -1163,6 +1168,127 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach(
   return TCL_OK;
 }
 
+#include "sqlite3changebatch.h"
+
+typedef struct TestChangebatch TestChangebatch;
+struct TestChangebatch {
+  sqlite3_changebatch *pChangebatch;
+};
+
+/*
+** Tclcmd:  $changebatch add BLOB
+**          $changebatch zero
+**          $changebatch delete
+*/
+static int SQLITE_TCLAPI test_changebatch_cmd(
+  void *clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  TestChangebatch *p = (TestChangebatch*)clientData;
+  sqlite3_changebatch *pChangebatch = p->pChangebatch;
+  struct SessionSubcmd {
+    const char *zSub;
+    int nArg;
+    const char *zMsg;
+    int iSub;
+  } aSub[] = {
+    { "add",          1, "CHANGESET",  }, /* 0 */
+    { "zero",         0, "",           }, /* 1 */
+    { "delete",       0, "",           }, /* 2 */
+    { 0 }
+  };
+  int iSub;
+  int rc;
+
+  if( objc<2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ...");
+    return TCL_ERROR;
+  }
+  rc = Tcl_GetIndexFromObjStruct(interp, 
+      objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub
+  );
+  if( rc!=TCL_OK ) return rc;
+  if( objc!=2+aSub[iSub].nArg ){
+    Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg);
+    return TCL_ERROR;
+  }
+
+  switch( iSub ){
+    case 0: {      /* add */
+      int nArg;
+      unsigned char *pArg = Tcl_GetByteArrayFromObj(objv[2], &nArg);
+      rc = sqlite3changebatch_add(pChangebatch, pArg, nArg);
+      if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){
+        return test_session_error(interp, rc, 0);
+      }else{
+        extern const char *sqlite3ErrName(int);
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+      }
+      break;
+    }
+
+    case 1: {      /* zero */
+      sqlite3changebatch_zero(pChangebatch);
+      break;
+    }
+
+    case 2:        /* delete */
+      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
+      break;
+  }
+
+  return TCL_OK;
+}
+
+static void SQLITE_TCLAPI test_changebatch_del(void *clientData){
+  TestChangebatch *p = (TestChangebatch*)clientData;
+  sqlite3changebatch_delete(p->pChangebatch);
+  ckfree((char*)p);
+}
+
+/*
+** Tclcmd:  sqlite3changebatch CMD DB-HANDLE
+*/
+static int SQLITE_TCLAPI test_sqlite3changebatch(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3 *db;
+  Tcl_CmdInfo info;
+  int rc;                         /* sqlite3session_create() return code */
+  TestChangebatch *p;             /* New wrapper object */
+
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "CMD DB-HANDLE");
+    return TCL_ERROR;
+  }
+
+  if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[2]), &info) ){
+    Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[2]), 0);
+    return TCL_ERROR;
+  }
+  db = *(sqlite3 **)info.objClientData;
+
+  p = (TestChangebatch*)ckalloc(sizeof(TestChangebatch));
+  memset(p, 0, sizeof(TestChangebatch));
+  rc = sqlite3changebatch_new(db, &p->pChangebatch);
+  if( rc!=SQLITE_OK ){
+    ckfree((char*)p);
+    return test_session_error(interp, rc, 0);
+  }
+
+  Tcl_CreateObjCommand(
+      interp, Tcl_GetString(objv[1]), test_changebatch_cmd, (ClientData)p,
+      test_changebatch_del
+  );
+  Tcl_SetObjResult(interp, objv[1]);
+  return TCL_OK;
+}
+
 /*
 ** tclcmd: CMD configure REBASE-BLOB
 ** tclcmd: CMD rebase CHANGESET
@@ -1345,6 +1471,10 @@ int TestSession_Init(Tcl_Interp *interp){
     Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, 0, 0);
   }
 
+
+  Tcl_CreateObjCommand(
+      interp, "sqlite3changebatch", test_sqlite3changebatch, 0, 0
+  );
   return TCL_OK;
 }
 

main.mk

@@ -362,6 +362,7 @@ TESTSRC = \
 TESTSRC += \
   $(TOP)/ext/misc/amatch.c \
   $(TOP)/ext/misc/appendvfs.c \
+  $(TOP)/ext/misc/bgckpt.c \
   $(TOP)/ext/misc/carray.c \
   $(TOP)/ext/misc/cksumvfs.c \
   $(TOP)/ext/misc/closure.c \
@@ -443,6 +444,7 @@ TESTSRC2 = \
   $(TOP)/ext/async/sqlite3async.c \
   $(TOP)/ext/misc/stmt.c \
   $(TOP)/ext/session/sqlite3session.c \
+  $(TOP)/ext/session/sqlite3changebatch.c \
   $(TOP)/ext/session/test_session.c \
   fts5.c
 
@@ -1001,11 +1003,15 @@ THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
                   $(TOP)/test/tt3_index.c      \
                   $(TOP)/test/tt3_vacuum.c      \
                   $(TOP)/test/tt3_stress.c      \
+                  $(TOP)/test/tt3_bcwal2.c      \
                   $(TOP)/test/tt3_lookaside1.c
 
 threadtest3$(EXE): sqlite3.o $(THREADTEST3_SRC) $(TOP)/src/test_multiplex.c
 	$(TCCX) $(TOP)/test/threadtest3.c $(TOP)/src/test_multiplex.c sqlite3.o -o $@ $(THREADLIB)
 
+bc_test1$(EXE): sqlite3.o $(TOP)/test/bc_test1.c $(TOP)/test/tt3_core.c
+	$(TCCX) $(TOP)/test/bc_test1.c sqlite3.o -o $@ $(THREADLIB)
+
 threadtest: threadtest3$(EXE)
 	./threadtest3$(EXE)
 

manifest

@@ -1,11 +1,11 @@
-C Fix\smisuse\sof\sthe\ssqlite3_set_auxdata()\sinterface\sin\sthe\sICU\sextension.
-D 2022-09-28T19:14:01.801
+C Change\sthe\sinstrumentation\smessages\son\sthis\sbranch\sso\sthat\sthey\scontain\s"v=7"\sinstead\sof\s"v=6".
+D 2023-01-25T17:24:34.562
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
-F Makefile.in 525bccb89e36a927f9312a231f054a2dc029f6af75901c7fc1a781d51b260323
+F Makefile.in 6b71e6e41d5a56742309afdb69f2408d9827744123112f2649420b4beb055dc6
 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241
-F Makefile.msc d547a2fdba38a1c6cd1954977d0b0cc017f5f8fbfbc65287bf8d335808938016
+F Makefile.msc e27b351d669b070d1c217df18118dcb3a41f7ca6bf18fa3b47e36f93792823cc
 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e
 F VERSION 8868ddfa6e1eee218286021a94b3e22d13e550c76c72d878857547ca001de24a
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -38,6 +38,7 @@ F configure f959db96f314b3b91b3d658eebbc0a96b9542f1265c4de97e885aedb6bdcead5 x
 F configure.ac 3ef6eeff4387585bfcab76b0c3f6e15a0618587bb90245dd5d44e4378141bb35
 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
 F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd
+F doc/begin_concurrent.md 4bee2c3990d1eb800f1ce3726a911292a8e4b889300b2ffd4b08d357370db299
 F doc/json-enhancements.md e356fc834781f1f1aa22ee300027a270b2c960122468499bf347bb123ce1ea4f
 F doc/lemon.html efc0cd2345d66905505d98f862e1c571512def0ceb5b016cb658fd4918eb76a3
 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710
@@ -45,6 +46,7 @@ F doc/trusted-schema.md 33625008620e879c7bcfbbfa079587612c434fa094d338b08242288d
 F doc/vdbesort-memory.md 4da2639c14cd24a31e0af694b1a8dd37eaf277aff3867e9a8cc14046bc49df56
 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a
 F doc/wal-lock.md 781726aaba20bafeceb7ba9f91d5c98c6731691b30c954e37cf0b49a053d461d
+F doc/wal2.md a807405a05e19a4945c5905a9ffa0fe45b8560dd7572461192501f565c19cdb5
 F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 6f247666b495c477628dd19364d279c78ea48cd90c72d9f9b98ad1aff3294f94
@@ -120,7 +122,7 @@ F ext/fts5/fts5_config.c 501e7d3566bc92766b0e11c0109a7c5a6146bc41144195459af5422
 F ext/fts5/fts5_expr.c 40174a64829d30cc86e8266306ad24980f6911edd5ca0b8c1ce7821ea1341b88
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
 F ext/fts5/fts5_index.c 4b1ac44c665667be970df780bd8e734748047bd30a971d0bb7e884af8ac6e62c
-F ext/fts5/fts5_main.c 6078ae86d3b813753a4f1201054550aff21a3f660e97b30f200d2b1472874151
+F ext/fts5/fts5_main.c 82687a852cb4229c24d5525c9c435592719149af73dc955490adc88a95e9b45e
 F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
@@ -154,6 +156,7 @@ F ext/fts5/test/fts5cat.test daba0b80659460b0cb60bd1f40b402478a761fe7ea414c3c94c
 F ext/fts5/test/fts5circref.test f880dfd0d99f6fb73b88ccacb0927d18e833672fd906cc47d6b4e529419eaa62
 F ext/fts5/test/fts5colset.test 7031ce84fb4d312df5a99fc4e7b324e660ccb513c97eccdef469bfd52d3d0f8f
 F ext/fts5/test/fts5columnsize.test 45459ce4dd9fd853b6044cdc9674921bff89e3d840f348ca8c1630f9edbf5482
+F ext/fts5/test/fts5concurrent.test eaaba0037090138bc86d581ca95f3be7ef8eff60155b5b8fa3014acee4a595c2
 F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825ce437d26afe0817f
 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
@@ -290,6 +293,7 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23
 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
 F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824
+F ext/misc/bgckpt.c 49ae19aa03e6da393db5d17da256374d4c4d36889fdd89d6e4bc93aca2b752e6
 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a
 F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9
 F ext/misc/carray.c b752f46411e4e47e34dce6f0c88bc8e51bb821ba9e49bfcd882506451c928f69
@@ -433,6 +437,8 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de
 F ext/rtree/util/randomshape.tcl 54ee03d0d4a1c621806f7f44d5b78d2db8fac26e0e8687c36c4bd0203b27dbff
 F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
 F ext/rtree/visual01.txt e9c2564083bcd30ec51b07f881bffbf0e12b50a3f6fced0c222c5c1d2f94ac66
+F ext/session/changebatch1.test 9ceaa8f7b2a505933e250fbe6cbc550e4ce1e59d
+F ext/session/changebatchfault.test be49c793219bf387ad692a60856b921f0854ad6d
 F ext/session/changeset.c 7a1e6a14c7e92d36ca177e92e88b5281acd709f3b726298dc34ec0fb58869cb5
 F ext/session/changesetfuzz.c 227076ab0ae4447d742c01ee88a564da6478bbf26b65108bf8fac9cd8b0b24aa
 F ext/session/changesetfuzz1.test 2e1b90d888fbf0eea5e1bd2f1e527a48cc85f8e0ff75df1ec4e320b21f580b3a
@@ -451,7 +457,7 @@ F ext/session/sessionD.test 4f91d0ca8afc4c3969c72c9f0b5ea9527e21de29039937d0d973
 F ext/session/sessionE.test b2010949c9d7415306f64e3c2072ddabc4b8250c98478d3c0c4d064bce83111d
 F ext/session/sessionF.test d37ed800881e742c208df443537bf29aa49fd56eac520d0f0c6df3e6320f3401
 F ext/session/sessionG.test 3828b944cd1285f4379340fd36f8b64c464fc84df6ff3ccbc95578fd87140b9c
-F ext/session/sessionH.test b17afdbd3b8f17e9bab91e235acf167cf35485db2ab2df0ea8893fbb914741a4
+F ext/session/sessionH.test ecabc041e04e48671a94cd7c02993bc73f75dced943280a026e89cdf64374eef
 F ext/session/session_common.tcl f613174665456b2d916ae8df3e5735092a1c1712f36f46840172e9a01e8cc53e
 F ext/session/session_speed_test.c dcf0ef58d76b70c8fbd9eab3be77cf9deb8bc1638fed8be518b62d6cbdef88b3
 F ext/session/sessionat.test 52993535f1230a42f70886643574ba7ae60ef854f8add9c8e3fcc3eb5c564bd2
@@ -466,9 +472,11 @@ F ext/session/sessionrebase.test ccfa716b23bd1d3b03217ee58cfd90c78d4b99f53e6a9a2
 F ext/session/sessionsize.test 6f644aff31c7f1e4871e9ff3542766e18da68fc7e587b83a347ea9820a002dd8
 F ext/session/sessionstat1.test 218d351cf9fcd6648f125a26b607b140310160184723c2666091b54450a68fb5
 F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc
-F ext/session/sqlite3session.c 1d019c5caf51936ef24c761db63552b06e0e0d951c8740bba9639b17fa0cb107
-F ext/session/sqlite3session.h 0907de79bc13a2e3af30a6dc29acc60792a3eaf7d33d44cf52500d0f3c2b2171
-F ext/session/test_session.c f433f68a8a8c64b0f5bc74dc725078f12483301ad4ae8375205eef790274a787
+F ext/session/sqlite3changebatch.c d5553b79e012ee2cb06c0a96bdf9dfe19e66354390ea0036cc46c4953142d517
+F ext/session/sqlite3changebatch.h e72016998c9a22d439ddfd547b69e1ebac810c24
+F ext/session/sqlite3session.c 421854336ece362552508426d89091c1832148066f0641e8d72e22f58d9241f3
+F ext/session/sqlite3session.h 01807bb8c8f531c23052032e9d531eec49af79b7394f8213cf5a49784325b6a8
+F ext/session/test_session.c 226461035031632255c8098f3bcfa4ea27f0b051efd88857a2a36bcea5c294b9
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb
@@ -509,7 +517,7 @@ F ext/wasm/testing2.js d37433c601f88ed275712c1cfc92d3fb36c7c22e1ed8c7396fb2359e4
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 20801eed419dc58936ff9449b04041edbbbc0488a9fc683e72471dded050e0bb
+F main.mk c7406b33f2184b5678adb9e5582761d013ef14989aa7cb7bf5425120bcf1d892
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -526,12 +534,12 @@ F src/analyze.c aabdf3769c7fd9954a8ec508eb7041ae174b66f88d12c47199fabbea9a646467
 F src/attach.c 4431f82f0247bf3aaf91589acafdff77d1882235c95407b36da1585c765fbbc8
 F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf
 F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7
-F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d
+F src/bitvec.c 3907fcbe8a0c8c2db58d97087d15cdabbf2842adb9125df9ab9ff87d3db16775
 F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca
-F src/btree.c aad3381b25b0aa56838b35f86fbae9fb10ab8670ba130b413e6e3652cb732d74
-F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22
-F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e
-F src/build.c 898884afd67d953808cb687babc15b66a10213f99fe2ce7db98960e959881f98
+F src/btree.c 8e8fb7a5824476b5435d7b884351f15e9a5cef0e70dfbf082fdce47cd50e3132
+F src/btree.h bef8ed2baec32f6e1c56dd17a18e1149af0e9c69ea3d91548a1664306630591b
+F src/btreeInt.h 1594454c0440db232fc88d827c649808e5f266b43e1fd70e3a216a435cd7fb79
+F src/build.c c3cfa409c354a291acdb9d40199fa30e71020c30fdc6dee2d936ca15deb9046a
 F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484a
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 93e4b5f4faf6d3f688988a116773259a4fbfb4ddac0e9bf9d0ae0429390c2543
@@ -542,7 +550,7 @@ F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e
 F src/expr.c 1cbdd76eeedb729ea9060df03e3e6b74a302784a13bfa38794a8194f894641ea
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002
-F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1
+F src/func.c c4b2229aac24b3d7989ef51abae0e5e87e58e8bda3c7b6c55f76d231dbedfb8d
 F src/global.c e83ee571b79ee3adc32e380cf554cf1254bc43763d23786c71721fbcdfbbb965
 F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19
 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
@@ -552,7 +560,7 @@ F src/insert.c aea5361767817f917b0f0f647a1f0b1621bd858938ae6ae545c3b6b9814b798f
 F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 853385cc7a604157e137585097949252d5d0c731768e16b044608e5c95c3614b
-F src/main.c 8983b4a316d7e09946dd731913aa41712f02e2b55cb5c6c92126ccfe2473244a
+F src/main.c db487c05a732709ae4c94f1592ceaa49e05d0f56257223f2ae7e0dec54a858c3
 F src/malloc.c dfddca1e163496c0a10250cedeafaf56dff47673e0f15888fb0925340a8e3f90
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -572,33 +580,33 @@ F src/os.c 0eb831ba3575af5277e47f4edd14fdfc90025c67eb25ce5cda634518d308d4e9
 F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 102f7e5c5b59c18ea3dbc929dc3be8acb3afc0e0b6ad572e032335c9c27f44f1
+F src/os_unix.c 52ac6823d3895c6a3a5cf4b07c05052f7ee8bd100a076bf0cd672ec83c996246
 F src/os_win.c 8d129ae3e59e0fa900e20d0ad789e96f2e08177f0b00b53cdda65c40331e0902
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c 6176d9752eb580419e8fef4592dc417a6b00ddfd43ee22f818819bf8840ceee8
-F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3
-F src/parse.y 8e67d820030d2655b9942ffe61c1e7e6b96cea2f2f72183533299393907d0564
+F src/pager.c a0dfc443300e2af6e25b742f5fc6b8366261e168bdd72102d982cee4bb5b0b62
+F src/pager.h be2b4b375a535cbca8a8c37223dddd0cac05f2d3a79b92d831c5bdcbefdef31a
+F src/parse.y 17c50d262d92083badeb60b3ebe4725e19c76548f90aea898ab07d4f2940a7d8
 F src/pcache.c f4268f7f73c6a3db12ce22fd25bc68dc42315d19599414ab1207d7cf32f79197
 F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586
-F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc
-F src/pragma.c 9bf7d8a2a9ad3bc36df3ec0d61817a44c38a1da527d59c26c203047f906e334a
-F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
+F src/pcache1.c 6b05f457044703eb6da771dc98f025a7753d2c5a352c6eaf0ff551139d835650
+F src/pragma.c c471a8752cc37919a213d860a1550c3e4b5bc1a416dff2aa72212e73c6982230
+F src/pragma.h 1f421360eed1a7721e8c521463df8519a7c8d0d5893ebd9dbfe0dba8de996f8c
 F src/prepare.c 971d5819a4bda88038c2283d71fc0a2974ffc3dd480f9bd941341017abacfd1b
 F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764
-F src/random.c 546d6feb15ec69c1aafe9bb351a277cbb498fd5410e646add673acb805714960
+F src/random.c f767e3c0048b408aa14bcdc084fdb9520b88bfdb7cd6d7e356f70c7ee26bcb45
 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633
 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
-F src/select.c bb18acf4eded647fef88d4d543c673874dbebff516fbeba90a85e6c13f2a58cd
+F src/select.c 321a529476188dbd8ff1e795e1f78287c6ea0e895523081d3c3954f416b9766f
 F src/shell.c.in e7e7c2c69ae86c5ee9e8ad66227203d46ff6dce8700a1b1dababff01c71d33df
-F src/sqlite.h.in b9b7fd73239d94db20332bb6e504688001e5564b655e1318a4427a1caef4b99e
+F src/sqlite.h.in 8134578d7027812e7c3cfdff69117b08f5a69592c5168dfc2f545368a284c9d8
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d
-F src/sqliteInt.h 94e7fc2a5f0fa5d1f0af84513fd2d1c70a9f6e772556b9dfef16feee63291eae
+F src/sqliteInt.h c9b07e3a5542b935ac160c71705c345d3e113438bdb67986ef09ed36efd16b8f
 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
 F src/tclsqlite.c 4e64ba300a5a26e0f1170e09032429faeb65e45e8f3d1a7833e8edb69fc2979e
-F src/test1.c 1356984e97bff07e4a8cc3863e892f05b3348678a74783bb6f350b76316736f1
+F src/test1.c 95c688261edf59ff77676aca426b343beb9f90d6548ecb16189ce614e5a63620
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644
 F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664
@@ -613,13 +621,13 @@ F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0
 F src/test_bestindex.c 8294d8223b7f18a3ddb7f9a0e30815dcca4e61681f78b538c870f7d934f88b81
 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce
 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274
-F src/test_config.c 8264637b06a3c1f0727c88d1ea32dcf7986b9e7e358a970cae87cdac8a5b2708
+F src/test_config.c c7a93ef3c0c881e3c46ff14163af886a602317206d82a7dc4ebbf400e661a6ff
 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f
 F src/test_demovfs.c 86142ba864d4297d54c5b2e972e74f3141ae4b30f05b3a95824184ed2d3d7f91
 F src/test_devsym.c aff2255ea290d7718da08af30cdf18e470ff7325a5eff63e0057b1496ed66593
 F src/test_fs.c ba1e1dc18fd3159fdba0b9c4256f14032159785320dfbd6776eb9973cb75d480
 F src/test_func.c 24df3a346c012b1fc9e1001d346db6054deb426db0a7437e92490630e71c9b0a
-F src/test_hexio.c 9478e56a0f08e07841a014a93b20e4ba2709ab56d039d1ca8020e26846aa19bd
+F src/test_hexio.c a605a100e628d39330044ae5f34cb19d50843061ed3178c3f83b37aef65f7e0a
 F src/test_init.c 4413c211a94b62157ca4c145b3f27c497f03c664
 F src/test_intarray.c 39b4181662a0f33a427748d87218e7578d913e683dc27eab7098bb41617cac71
 F src/test_intarray.h d57ae92f420cda25e22790dac474d60961bd0c500cbaa3338a05152d4a669ef7
@@ -639,9 +647,9 @@ F src/test_rtree.c 671f3fae50ff116ef2e32a3bf1fe21b5615b4b7b
 F src/test_schema.c f5d6067dfc2f2845c4dd56df63e66ee826fb23877855c785f75cc2ca83fd0c1b
 F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe
 F src/test_sqllog.c 540feaea7280cd5f926168aee9deb1065ae136d0bbbe7361e2ef3541783e187a
-F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e
+F src/test_superlock.c f4d4cc7319a608a54b7608158e8c7135fac19b88d6179e5bf17080e89d1f0278
 F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939
-F src/test_tclsh.c c4065ced25126e25c40122c5ff62dc89902ea617d72cdd27765151cdd7fcc477
+F src/test_tclsh.c dec6a760161492d51580e3b3702c6d527ae03aff238cc52fd15234085157a154
 F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc
 F src/test_thread.c 269ea9e1fa5828dba550eb26f619aa18aedbc29fd92f8a5f6b93521fbb74a61c
 F src/test_vdbecov.c f60c6f135ec42c0de013a1d5136777aa328a776d33277f92abac648930453d43
@@ -655,16 +663,16 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b
 F src/treeview.c 07787f67cd297a6d09d04b8d70c06769c60c9c1d9080378f93929c16f8fd3298
 F src/trigger.c bc70c58e713dcfb6cabe5cc0bed71aedb02c3e9e128c6089a78aca945ba4d720
-F src/update.c c52a7991bece0453d22c77c08469512ee2f1391c12503fd347d1c939220c5877
+F src/update.c c6fbfa86c7dbff057bb62b29e00846dec1468f12c62e1a575f745f87c5a55818
 F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937
 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
 F src/util.c 0be191521ff6d2805995f4910f0b6231b42843678b2efdc1abecaf39929a673f
-F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8
-F src/vdbe.c 0d1e3c658d98a7bb7201532ea7a3e4d59bf9165421c780d5f84c361e372f1179
-F src/vdbe.h 64619af62603dc3c4f5ff6ff6d2c8f389abd667a29ce6007ed44bd22b3211cd0
-F src/vdbeInt.h 17b7461ffcf9ee760d1341731715a419f6b8c763089a7ece25c2e8098d702b3f
+F src/vacuum.c 5b7888f917936dda09f98b1fda164cff1dad44fb5e143436bdbb9dc3191ce2df
+F src/vdbe.c 574598c7688e16c6b49bb588edfebb23c4eb8bfbce781b3b9f973b0c1779d7c2
+F src/vdbe.h 923ce223c6b64bf38e174069ce9ac7b729aea82c6b278da346f5812f29da90dd
+F src/vdbeInt.h 202638f8eea5cf2600e033c6862c2872dbbde626582c1a2c2e1ff0760fcacfcd
 F src/vdbeapi.c fc3183daf72808b4311b228989120fdbc2dc44972fb0d77d5c453460cc0e5b2c
-F src/vdbeaux.c c719cebaffa75e166f16a405b3dee96e7150d60fc563ab2dea716b60c6a51312
+F src/vdbeaux.c 6ee6ea20eaf22c72b5f28a2b0b77931aa648a59b366c7fb22f97b01755a40e1a
 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd
 F src/vdbemem.c c3ce80af15e2ff5c2824a8db881681cbf511376f13613da020bac6d320c535b1
 F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35
@@ -672,8 +680,8 @@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf8
 F src/vdbevtab.c f99b275366c5fc5e2d99f734729880994ab9500bdafde7fae3b02d562b9d323c
 F src/vtab.c bb53f9e2eaeecca07158643dd3d5039cf13b525fe2d267e113b39a36f374556c
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d
-F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
+F src/wal.c 08ab1092ca27ffeb97515482139c04b46c69bfd8f26251913409f1590c1f7038
+F src/wal.h 7a733af13b966ecb81872ce397e862116b3575ea53245b90b139a2873ee87825
 F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
 F src/where.c 63e712bcad47f70e94c2150976cd7da5040933699e3938d4189d064acbe40891
 F src/whereInt.h 70cd30de9ed784aa33fa6bd1245f060617de7a00d992469b6d8e419eed915743
@@ -761,6 +769,7 @@ F test/backup_malloc.test 0c9abdf74c51e7bedb66d504cd684f28d4bd4027
 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f
 F test/badutf2.test f310fd3b24a491b6b77bccdf14923b85d6ebcce751068c180d93a6b8ff854399
 F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c
+F test/bc_test1.c e0a092579552e066ed4ce7bcdaecfa69c4aacc8d
 F test/bestindex1.test 856a453dff8c68b4568601eed5a8b5e20b4763af9229f3947c215729ed878db0
 F test/bestindex2.test 394ff8fbf34703391247116d6a44e1c50ee7282236ee77909044573cefc37bc0
 F test/bestindex3.test 34bea272b0e0f835651b16a3931dbe7ac927039be6b2e1cb617bbe1d584b492b
@@ -828,6 +837,16 @@ F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151eca
 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1
 F test/colname.test 87ad5458bb8709312dac0d6755fd30e8e4ca83298d0a9ef6e5c24277a3c3390e
 F test/columncount.test 6fe99c2f35738b0129357a1cf3fa483f76140f4cd8a89014c88c33c876d2638f
+F test/concfault.test 500f17c3fcfe7705114422bcc6ddd3c740001a43
+F test/concfault2.test 34b3fd258836aa305475d00e804c7450ade92f0de0bf9fa620e701446669bb12
+F test/concurrent.test a0248ec6e3e79a5948453649cf86b5b359175cba55ea636b15426d6f0fa6c3da
+F test/concurrent2.test 847cd11edc82229a95e1eaf88b55c974430b0f669cefd67a453d274e3480981c
+F test/concurrent3.test 530671ac706f6a1d0f4992dbdd33a86408330d03cd90fb9e82ecb1b27f5fd081
+F test/concurrent4.test e0b12cd467137e50259df3b4f837507e82aaa07c35941c88664dc8ed1d089c44
+F test/concurrent5.test 0c16cbf7446af162a14e6def30445e94016064eb994e5aa4ebb2bebc59554176
+F test/concurrent6.test a7860e9ca13bb5fb76bcf41c5524fbfa9c37e6e258ecf84ffb5748a272488c67
+F test/concurrent7.test b96fa5c4cfdf8d5c0bc66b6934214500bad0260884a736f054ccc76e81aae85d
+F test/concurrent8.test b93937e74a8efb8b84f2fea7595b53418c5f29777bbe9cbdb5dc219b3dd72a7d
 F test/conflict.test ac0667090f66130ac77d5fb764655558ca6600dd6d88f670ca9123b61c448337
 F test/conflict2.test 5557909ce683b1073982f5d1b61dfb1d41e369533bfdaf003180c5bc87282dd1
 F test/conflict3.test 81865d9599609aca394fb3b9cd5f561d4729ea5b176bece3644f6ecb540f88ac
@@ -841,7 +860,7 @@ F test/corrupt6.test fc6a891716139665dae0073b6945e3670bf92568
 F test/corrupt7.test b036f94bda4b0b23a2919bf717046ce9ecca4543
 F test/corrupt8.test 2399dfe40d2c0c63af86706e30f3e6302a8d0516
 F test/corrupt9.test 730a3db08d4ab9aa43392ea30d9c2b4879cbff85
-F test/corruptA.test 112f4b2ae0b95ebf3ea63718642fb969a93acea557ace3a307234d19c245989b
+F test/corruptA.test 56e8f321adaf3411960e9d2c7136669d8e1a91cbde6cf401ea84e6d6c7ccbe10
 F test/corruptB.test 73a8d6c0b9833697ecf16b63e3c5c05c945b5dec
 F test/corruptC.test 74d4498fd25759618b393f1e9cde111de828b88c1848ab320f6c179fd52b5a60
 F test/corruptD.test 33a37ce3ed56a20093ceee778cd2d7109c7085a59f3213d2baede11d952e8e50
@@ -854,7 +873,7 @@ F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4
 F test/corruptK.test 5b4212fe346699831c5ad559a62c54e11c0611bdde1ea8423a091f9c01aa32af
 F test/corruptL.test ecce40d7b9b909a670a42a45d86e30d927735d7e7f09041af438b19529d35532
 F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c3e067
-F test/corruptN.test 7c099d153a554001b4fb829c799b01f2ea6276cbc32479131e0db0da4efd9cc4
+F test/corruptN.test 57985a0737f5e008283a91c24630cd3c7003d3c7b62824edaa21258e46da9455
 F test/cost.test b11cdbf9f11ffe8ef99c9881bf390e61fe92baf2182bad1dbe6de59a7295c576
 F test/count.test cd4bd531066e8d77ef8fe1e3fc8253d042072e117ccab214b290cf83f1602249
 F test/countofview.test e17d6e6688cf74f22783c9ec6e788c0790ee4fbbaee713affd00b1ac0bb39b86
@@ -1332,11 +1351,10 @@ F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f
 F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8
 F test/pageropt.test 84e4cc5cbca285357f7906e99b21be4f2bf5abc0
 F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305
-F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035ce4b3
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test 909c84575ac50f6d30fa125a109a01986e08c26b9ea38d28501a0711b50b5627
+F test/permutations.test d8874f89c3ae67055a2fa1666bc94229ffde68d47304cae11839c10af3f7328a
 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f
 F test/pragma.test cae534c12a033a5c319ccc94f50b32811acdef9f67bf19a82ff42697caccd69f
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
@@ -1359,7 +1377,7 @@ F test/quote.test ffb40f0eb7a25c1d8cfe11ee2fe67f8e85fbf3fed348810834114be1fdada1
 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459
 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df
 F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736
-F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8
+F test/rdonly.test 21e99ee237265d0cf95a0c84b50c784e834acaa4ef05d92a27b262626a656682
 F test/recover.test ccb8c2623902a92ebb76770edd075cb4f75a4760bb7afde38026572c6e79070d
 F test/regexp1.test 83c631617357150f8054ca1d1fed40a552b0d0f8eb7a7f090c3be02cee9f9913
 F test/regexp2.test 55ed41da802b0e284ac7e2fe944be3948f93ff25abbca0361a609acfed1368b5
@@ -1390,11 +1408,11 @@ F test/rowvaluefault.test 963ae9cdaed30a85a29668dd514e639f3556cae903ee9f172ea972
 F test/rowvaluevtab.test cd9747bb3f308086944c07968f547ad6b05022e698d80b9ffbdfe09ce0b8da6f
 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798
 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09
-F test/savepoint.test 1f8a6b1aea9a0d05837adc463d4bf47bd9d0f1c842f1c2a9caccd639baf34bf9
+F test/savepoint.test ea1a6c08454e1a4edd973589bcf1fca83928ed09c48858fde1e8653b6daab278
 F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7
 F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0
 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd
-F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7
+F test/savepoint6.test 48a645a7bb3a59a6fcf06a7364cfe5b655c336760de39068f7c241b0fc80d963
 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa
 F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2
 F test/scanstatus.test 9a0ed37ab6d57b50567282788fffdf832d9b16739ecc41bff9d77a8d767cf317
@@ -1524,7 +1542,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16
 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc
-F test/tester.tcl d759ac44a501fb832f2ea966429ca18acfba0f9a8d34ad5c499332b079b37023
+F test/tester.tcl c2c5095594f8b6d9bcb30a835f872241faf9adf8c26f613b7c5cb672202ab2fd
 F test/testrunner.tcl 86b57135754ab2160aeb04b4829d321fb285a5cfa7a505fe61d69aed605854cc
 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899
 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502
@@ -1536,7 +1554,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46
 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd
 F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b
 F test/threadtest2.c a70a8e94bef23339d34226eb9521015ef99f4df8
-F test/threadtest3.c 655bff6c0895ec03f014126aa65e808fac9aae8c5a7a7da58a510cbe8b43b781
+F test/threadtest3.c e947b396444f7992a942cd8db0f01589ede90dd250ec802fe800cc90bbee21e3
 F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925
 F test/threadtest5.c 9b4d782c58d8915d7e955ff8051f3d03628bda0d33b82971ea8c0f2f2808c421
 F test/time-wordcount.sh 8e0b0f8109367827ad5d58f5cc849705731e4b90
@@ -1710,7 +1728,9 @@ F test/triggerF.test 5d76f0a8c428ff87a4d5ed52da06f6096a2c787a1e21b846111dfac4123
 F test/triggerG.test 2b816093c91ba73c733cfa8aedcc210ad819d72a98b1da30768a3c56505233e9
 F test/triggerupfrom.test d1f9e56090408115c522bee626cc33a2f3370f627a5e341d832589d72e3aa271
 F test/trustschema1.test 4e970aef0bfe0cee139703cc7209d0e0f07725d999b180ba50770f49edef1494
+F test/tt3_bcwal2.c 8351577477ce58c3b21a1772c28e73ec58538c44be4a183ff7485d6814bd5385
 F test/tt3_checkpoint.c ac7ca661d739280c89d9c253897df64a59a49369bd1247207ac0f655b622579d
+F test/tt3_core.c 8cd89ead95410f70e7fb02c79f1e040f9c5ad5cf
 F test/tt3_index.c 39eec10a35f57672225be4d182862152896dee4a
 F test/tt3_lookaside1.c 0377e202c3c2a50d688cb65ba203afeda6fafeb9
 F test/tt3_shared.c b37d22defc944a2ac4c91c927fd06c1d48cd51e2ce9d004fe868625bd2399f93
@@ -1742,7 +1762,7 @@ F test/upsert3.test 88d7d590a1948a9cb6eac1b54b0642f67a9f35a1fc0f19b200e97d5d39e3
 F test/upsert4.test 25d2a1da92f149331ae0c51ca6e3eee78189577585eab92de149900d62994fa5
 F test/upsert5.test fff0dcfce73c649204543088d8e5bde01172676063ec9b8f8fc7f195abc386fe
 F test/upsertfault.test f21ca47740841fdb4d61acfa7b17646d773e67724fe8c185b71c018db8a94b35
-F test/uri.test c1abaaaa28e9422d61e5f3f9cbc8ef993ec49fe802f581520731708561d49384
+F test/uri.test 8f27eaa41804099fca15101e30fd1b29aebbebf32d4e3c24614fa6319216936d
 F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7
 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9
 F test/utf16align.test 9fde0bb5d3a821594aa68c6829ab9c5453a084384137ebb9f6153e2d678039da
@@ -1784,8 +1804,21 @@ F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad
 F test/vtabdistinct.test 7688f0889358f849fd60bbfde1ded38b014b18066076d4bfbb75395804dfe072
 F test/vtabdrop.test 65d4cf6722972e5499bdaf0c0d70ee3b8133944a4e4bc31862563f32a7edca12
 F test/vtabrhs1.test 9b5ecbc74a689500c33a4b2b36761f9bcc22fcc4e3f9d21066ee0c9c74cf5f6c
-F test/wal.test b7cc6984709f54afbf8441747ced1f646af120bf0c1b1d847bfa39306fbea089
-F test/wal2.test 31f6e2c404b9f2cdf9ca19b105a1742fdc19653c2c936da39e3658c617524046
+F test/wal.test 07d61ee522bc203235a7979645d6c5d84f86882afad5b678532ee37b5b85b133
+F test/wal2.test 88d34cc62bf5460ad6b1f837cd56ab55731bb0637ff99e113b790873d9b60b14
+F test/wal2big.test 0b4ec526f9ca4bbabc355042c38045ae2e253fb46eb327bb7693d0122bc6968b
+F test/wal2concurrent.test 7fc3e570073683a2a28f42bda46ecf516f5bc82afd703c1fbf4aa38e18fb3361
+F test/wal2fault.test 2e8e60cacd5bcd451618aeffd05f676894d17202d3e2986e288d36e2c5993249
+F test/wal2lock.test 0ef98d72dc6bcf7711dedd684760488400d9a9a6eec0dc5d3822060437793552
+F test/wal2openclose.test 2b26be723ea7f4263c8d5d70b37efd1c359561a0526e39466c45fe8e6478daee
+F test/wal2recover.test ba8f4bc9397c838734619f9e759bd98b00e355347b3cf80a2e677610d231d5d8
+F test/wal2recover2.test 698bd0da28b84f470dc5dd50340401fdb50cf8d7894de27d55b61d22130bb1e1
+F test/wal2recover3.test 4a91689e165a38bc401736e6518188c2b0ff4fa1566d1810b8867536db128177
+F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c
+F test/wal2rollback.test 23adc4a099b23f6aaea8b04fdca1c35861d887dd80f8be7da2d5273eb777e428
+F test/wal2savepoint.test 3793a0ae97011fca358f79775f5d7d9f85da75c8e67686e2e19713da0cb0d99c
+F test/wal2simple.test 320a08927f307d0ead26fa3fcef4e3f64279f49be9504f918cb33294f75aeec8
+F test/wal2snapshot.test f6c3945bea572fd8745774e95b8dca1e5832c696e2251bb0db33391ee567fe60
 F test/wal3.test 5de023bb862fd1eb9d2ad26fa8d9c43abb5370582e5b08b2ae0d6f93661bc310
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
 F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9
@@ -1794,7 +1827,7 @@ F test/wal64k.test 2a525c0f45d709bae3765c71045ccec5df7d100ccbd3a7860fdba46c9addb
 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd
 F test/wal8.test d9df3fba4caad5854ed69ed673c68482514203c8
 F test/wal9.test 378e76a9ad09cd9bee06c172ad3547b0129a6750
-F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe
+F test/wal_common.tcl 37902864b63794e9bf59393b8887e21a9cbd4db8
 F test/walbak.test 018d4e5a3d45c6298d11b99f09a8ef6876527946
 F test/walbig.test f437473a16cfb314867c6b5d1dbcd519e73e3434
 F test/walblock.test be48f3a75eff0b4456209f26b3ce186c2015497d
@@ -1811,10 +1844,10 @@ F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496
 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test 1b3f922125e341703f6e946d77fdc564d38fb3e07a9385cfdc6c99cac1ecf878
-F test/walprotocol2.test 7d3b6b4bf0b12f8007121b1e6ef714bc99101fb3b48e46371df1db868eebc131
+F test/walprotocol2.test 7e4bedd5ee83607e2928ac438bf7332a396b980d3e02aa0746509ce11ad1f13c
 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20
 F test/walro2.test 33955a6fd874dd9724005e17f77fef89d334b3171454a1256fe4941a96766cdc
-F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68
+F test/walrofault.test 5a25f91c16a68bae65edec7cdef4495e5c6494c8408743fe9b29045fa6665cd0
 F test/walsetlk.test 34c901443b31ab720afc463f5b236c86ca5c4134402573dce91aa0761de8db5a
 F test/walshared.test 42e3808582504878af237ea02c42ca793e8a0efaa19df7df26ac573370dbc7a3
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
@@ -1932,7 +1965,7 @@ F tool/mkmsvcmin.tcl 6ecab9fe22c2c8de4d82d4c46797bda3d2deac8e763885f5a38d0c44a89
 F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef
 F tool/mkopcodeh.tcl bcb2bd5affb545fd219ef0304c7978e2a356407ab723f45ec8569235892c1c3f
 F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa
-F tool/mkpragmatab.tcl bd07bd59d45d0f3448e123d6937e9811195f9908a51e09d774609883055bfd3d
+F tool/mkpragmatab.tcl 8dcba40365eceba24f059d82bbc8b61bb647d5a0b8076f24a1876872a1b58832
 F tool/mkshellc.tcl df5d249617f9cc94d5c48eb0401673eb3f31f383ecbc54e8a13ca3dd97e89450
 F tool/mksourceid.c 36aa8020014aed0836fd13c51d6dc9219b0df1761d6b5f58ff5b616211b079b9
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
@@ -1973,6 +2006,8 @@ F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
 F tool/symbols.sh 1612bd947750e21e7b47befad5f6b3825b06cce0705441f903bf35ced65ae9b9
+F tool/tserver.c 17b7f0b06f4e776e26220889941a86936b3c56ad18608baadc8faa00b7bd46ee
+F tool/tserver_test.tcl 64415a134749ac3f38c9abd2bb95c7387a9b44e5116419487fd008cff8a459db
 F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003
 F tool/vdbe-compress.tcl 1dcb7632e57cf57105248029e6e162fddaf6c0fccb3bb9e6215603752c5a2d4a
 F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
@@ -2000,8 +2035,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 584de6a996c78b8e41bdfcd05a8e2a3844664c6b4efedb5883c8b8af388462b5
-R d79b3651cd76ebe372f6bfce59942639
-U drh
-Z 79f3fdc2db5301812c92ae8ca64cb084
+P 1e4cfd2f30c36172eb513b82db322f5c38d3686137a063f5c2a9dff957068e23
+R f51b267673d00e48e6358cb3156b2489
+U dan
+Z 30b0adacbf5833d4a322fe01bc07b041
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-f25cf63471cbed1edb27591e57fead62550d4046dbdcb61312288f0f6f24c646
+acea12d90f180538161b9611ab8fb0e10a43262c15739a8c24d3732b99f968ed

src/bitvec.c

@@ -171,6 +171,12 @@ int sqlite3BitvecSet(Bitvec *p, u32 i){
   if( p==0 ) return SQLITE_OK;
   assert( i>0 );
   assert( i<=p->iSize );
+  if( i>p->iSize || i==0 ){
+    sqlite3_log(SQLITE_ERROR, 
+        "Bitvec: setting bit %d of bitvec size %d\n", (int)i, (int)p->iSize
+    );
+    abort();
+  }
   i--;
   while((p->iSize > BITVEC_NBIT) && p->iDivisor) {
     u32 bin = i/p->iDivisor;

src/btree.c

@@ -14,6 +14,7 @@
 ** Including a description of file format and an overview of operation.
 */
 #include "btreeInt.h"
+#include "vdbeInt.h"
 
 /*
 ** The header string that appears at the beginning of every
@@ -480,7 +481,240 @@ static void downgradeAllSharedCacheTableLocks(Btree *p){
 
 #endif /* SQLITE_OMIT_SHARED_CACHE */
 
-static void releasePage(MemPage *pPage);         /* Forward reference */
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** The following structure - BtreePtrmap - stores the in-memory pointer map
+** used for newly allocated pages in CONCURRENT transactions. Such pages are
+** always allocated in a contiguous block (from the end of the file) starting
+** with page BtreePtrmap.iFirst.
+*/
+typedef struct RollbackEntry RollbackEntry;
+typedef struct PtrmapEntry PtrmapEntry;
+struct PtrmapEntry {
+  Pgno parent;
+  u8 eType;
+};
+struct RollbackEntry {
+  Pgno pgno;
+  Pgno parent;
+  u8 eType;
+};
+struct BtreePtrmap {
+  Pgno iFirst;                    /* First new page number aPtr[0] */
+
+  int nPtrAlloc;                  /* Allocated size of aPtr[] array */
+  PtrmapEntry *aPtr;              /* Array of parent page numbers */
+
+  int nSvpt;                      /* Used size of aSvpt[] array */
+  int nSvptAlloc;                 /* Allocated size of aSvpt[] */
+  int *aSvpt;                     /* First aRollback[] entry for savepoint i */
+
+  int nRollback;                  /* Used size of aRollback[] array */
+  int nRollbackAlloc;             /* Allocated size of aRollback[] array */
+  RollbackEntry *aRollback;       /* Array of rollback entries */
+};
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** If page number pgno is greater than or equal to BtreePtrmap.iFirst, 
+** store an entry for it in the pointer-map structure.
+*/
+static int btreePtrmapStore(
+  BtShared *pBt,
+  Pgno pgno,
+  u8 eType, 
+  Pgno parent
+){
+  BtreePtrmap *pMap = pBt->pMap;
+  if( pgno>=pMap->iFirst ){
+    int iEntry = pgno - pMap->iFirst;
+
+    /* Grow the aPtr[] array as required */
+    while( iEntry>=pMap->nPtrAlloc ){
+      int nNew = pMap->nPtrAlloc ? pMap->nPtrAlloc*2 : 16;
+      PtrmapEntry *aNew = (PtrmapEntry*)sqlite3_realloc(
+          pMap->aPtr, nNew*sizeof(PtrmapEntry)
+      );
+      if( aNew==0 ){
+        return SQLITE_NOMEM;
+      }else{
+        int nByte = (nNew-pMap->nPtrAlloc)*sizeof(PtrmapEntry);
+        memset(&aNew[pMap->nPtrAlloc], 0, nByte);
+        pMap->aPtr = aNew;
+        pMap->nPtrAlloc = nNew;
+      }
+    }
+
+    /* Add an entry to the rollback log if required */
+    if( pMap->nSvpt>0 && pMap->aPtr[iEntry].parent ){
+      if( pMap->nRollback>=pMap->nRollbackAlloc ){
+        int nNew = pMap->nRollback ? pMap->nRollback*2 : 16;
+        RollbackEntry *aNew = (RollbackEntry*)sqlite3_realloc(
+            pMap->aRollback, nNew*sizeof(RollbackEntry)
+        );
+        if( aNew==0 ){
+          return SQLITE_NOMEM;
+        }else{
+          pMap->aRollback = aNew;
+          pMap->nRollbackAlloc = nNew;
+        }
+      }
+
+      pMap->aRollback[pMap->nRollback].pgno = pgno;
+      pMap->aRollback[pMap->nRollback].parent = pMap->aPtr[iEntry].parent;
+      pMap->aRollback[pMap->nRollback].eType = pMap->aPtr[iEntry].eType;
+      pMap->nRollback++;
+    }
+
+    /* Update the aPtr[] array */
+    pMap->aPtr[iEntry].parent = parent;
+    pMap->aPtr[iEntry].eType = eType;
+  }
+
+  return SQLITE_OK;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Open savepoint iSavepoint, if it is not already open.
+*/
+static int btreePtrmapBegin(BtShared *pBt, int nSvpt){
+  BtreePtrmap *pMap = pBt->pMap;
+  if( pMap && nSvpt>pMap->nSvpt ){
+    int i;
+    if( nSvpt>=pMap->nSvptAlloc ){
+      int nNew = pMap->nSvptAlloc ? pMap->nSvptAlloc*2 : 16;
+      int *aNew = sqlite3_realloc(pMap->aSvpt, sizeof(int) * nNew);
+      if( aNew==0 ){
+        return SQLITE_NOMEM;
+      }else{
+        pMap->aSvpt = aNew;
+        pMap->nSvptAlloc = nNew;
+      }
+    }
+
+    for(i=pMap->nSvpt; i<nSvpt; i++){
+      pMap->aSvpt[i] = pMap->nRollback;
+    }
+    pMap->nSvpt = nSvpt;
+  }
+
+  return SQLITE_OK;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Rollback (if op==SAVEPOINT_ROLLBACK) or release (if op==SAVEPOINT_RELEASE)
+** savepoint iSvpt.
+*/
+static void btreePtrmapEnd(BtShared *pBt, int op, int iSvpt){
+  BtreePtrmap *pMap = pBt->pMap;
+  if( pMap ){
+    assert( op==SAVEPOINT_ROLLBACK || op==SAVEPOINT_RELEASE );
+    assert( iSvpt>=0 || (iSvpt==-1 && op==SAVEPOINT_ROLLBACK) );
+    if( iSvpt<0 ){
+      pMap->nSvpt = 0;
+      pMap->nRollback = 0;
+      memset(pMap->aPtr, 0, sizeof(Pgno) * pMap->nPtrAlloc);
+    }else if( iSvpt<pMap->nSvpt ){
+      if( op==SAVEPOINT_ROLLBACK ){
+        int ii;
+        for(ii=pMap->nRollback-1; ii>=pMap->aSvpt[iSvpt]; ii--){
+          RollbackEntry *p = &pMap->aRollback[ii];
+          PtrmapEntry *pEntry = &pMap->aPtr[p->pgno - pMap->iFirst];
+          pEntry->parent = p->parent;
+          pEntry->eType = p->eType;
+        }
+      }
+      pMap->nSvpt = iSvpt + (op==SAVEPOINT_ROLLBACK);
+      pMap->nRollback = pMap->aSvpt[iSvpt];
+    }
+  }
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** This function is called after an CONCURRENT transaction is opened on the
+** database. It allocates the BtreePtrmap structure used to track pointers
+** to allocated pages and zeroes the nFree/iTrunk fields in the database 
+** header on page 1.
+*/
+static int btreePtrmapAllocate(BtShared *pBt){
+  int rc = SQLITE_OK;
+  if( pBt->pMap==0 ){
+    BtreePtrmap *pMap = sqlite3_malloc(sizeof(BtreePtrmap));
+    if( pMap==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      memset(&pBt->pPage1->aData[32], 0, sizeof(u32)*2);
+      memset(pMap, 0, sizeof(BtreePtrmap));
+      pMap->iFirst = pBt->nPage + 1;
+      pBt->pMap = pMap;
+    }
+  }
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Free any BtreePtrmap structure allocated by an earlier call to
+** btreePtrmapAllocate().
+*/
+static void btreePtrmapDelete(BtShared *pBt){
+  BtreePtrmap *pMap = pBt->pMap;
+  if( pMap ){
+    sqlite3_free(pMap->aRollback);
+    sqlite3_free(pMap->aPtr);
+    sqlite3_free(pMap->aSvpt);
+    sqlite3_free(pMap);
+    pBt->pMap = 0;
+  }
+}
+
+/*
+** Check that the pointer-map does not contain any entries with a parent
+** page of 0. Call sqlite3_log() multiple times to output the entire
+** data structure if it does.
+*/
+static void btreePtrmapCheck(BtShared *pBt, Pgno nPage){
+  Pgno i;
+  int bProblem = 0;
+  BtreePtrmap *p = pBt->pMap;
+
+  for(i=p->iFirst; i<=nPage; i++){
+    PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst];
+    if( pEntry->eType==PTRMAP_OVERFLOW1
+     || pEntry->eType==PTRMAP_OVERFLOW2
+     || pEntry->eType==PTRMAP_BTREE
+    ){
+      if( pEntry->parent==0 ){
+        bProblem = 1;
+        break;
+      }
+    }
+  }
+
+  if( bProblem ){
+    for(i=p->iFirst; i<=nPage; i++){
+      PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst];
+      sqlite3_log(SQLITE_CORRUPT, 
+          "btreePtrmapCheck: pgno=%d eType=%d parent=%d", 
+          (int)i, (int)pEntry->eType, (int)pEntry->parent
+      );
+    }
+    abort();
+  }
+}
+
+#else  /* SQLITE_OMIT_CONCURRENT */
+# define btreePtrmapAllocate(x) SQLITE_OK
+# define btreePtrmapDelete(x) 
+# define btreePtrmapBegin(x,y)  SQLITE_OK
+# define btreePtrmapEnd(x,y,z) 
+# define btreePtrmapCheck(y,z) 
+#endif /* SQLITE_OMIT_CONCURRENT */
+
+static void releasePage(MemPage *pPage);  /* Forward reference */
 static void releasePageOne(MemPage *pPage);      /* Forward reference */
 static void releasePageNotNull(MemPage *pPage);  /* Forward reference */
 
@@ -1005,6 +1239,13 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){
   /* The super-journal page number must never be used as a pointer map page */
   assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) );
 
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( pBt->pMap ){
+    *pRC = btreePtrmapStore(pBt, key, eType, parent);
+    return;
+  }
+#endif
+
   assert( pBt->autoVacuum );
   if( key==0 ){
     *pRC = SQLITE_CORRUPT_BKPT;
@@ -2311,6 +2552,17 @@ getAndInitPage_error1:
   return rc;
 }
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/* 
+** Set the value of the MemPage.pgnoRoot variable, if it exists.
+*/
+static void setMempageRoot(MemPage *pPg, u32 pgnoRoot){
+  pPg->pgnoRoot = pgnoRoot;
+}
+#else
+# define setMempageRoot(x,y)
+#endif
+
 /*
 ** Release a MemPage.  This should be called once for each prior
 ** call to btreeGetPage.
@@ -3214,10 +3466,10 @@ static int lockBtree(BtShared *pBt){
       goto page1_init_failed;
     }
 #else
-    if( page1[18]>2 ){
+    if( page1[18]>3 ){
       pBt->btsFlags |= BTS_READ_ONLY;
     }
-    if( page1[19]>2 ){
+    if( page1[19]>3 ){
       goto page1_init_failed;
     }
 
@@ -3229,9 +3481,9 @@ static int lockBtree(BtShared *pBt){
     ** may not be the latest version - there may be a newer one in the log
     ** file.
     */
-    if( page1[19]==2 && (pBt->btsFlags & BTS_NO_WAL)==0 ){
+    if( page1[19]>=2 && (pBt->btsFlags & BTS_NO_WAL)==0 ){
       int isOpen = 0;
-      rc = sqlite3PagerOpenWal(pBt->pPager, &isOpen);
+      rc = sqlite3PagerOpenWal(pBt->pPager, (page1[19]==3), &isOpen);
       if( rc!=SQLITE_OK ){
         goto page1_init_failed;
       }else{
@@ -3489,6 +3741,7 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){
   BtShared *pBt = p->pBt;
   Pager *pPager = pBt->pPager;
   int rc = SQLITE_OK;
+  int bConcurrent = (p->db->eConcurrent && !ISAUTOVACUUM);
 
   sqlite3BtreeEnter(p);
   btreeIntegrity(p);
@@ -3502,6 +3755,10 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){
   }
   assert( pBt->inTransaction==TRANS_WRITE || IfNotOmitAV(pBt->bDoTruncate)==0 );
 
+  if( p->aOpenTransTm ){
+    p->aOpenTransTm[OPEN_TRANS_START] = sqlite3STimeNow();
+  }
+
   if( (p->db->flags & SQLITE_ResetDatabase) 
    && sqlite3PagerIsreadonly(pPager)==0 
   ){
@@ -3576,7 +3833,8 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){
       if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){
         rc = SQLITE_READONLY;
       }else{
-        rc = sqlite3PagerBegin(pPager, wrflag>1, sqlite3TempInMemory(p->db));
+        int exFlag = bConcurrent ? -1 : (wrflag>1);
+        rc = sqlite3PagerBegin(pPager, exFlag, sqlite3TempInMemory(p->db));
         if( rc==SQLITE_OK ){
           rc = newDatabase(pBt);
         }else if( rc==SQLITE_BUSY_SNAPSHOT && pBt->inTransaction==TRANS_NONE ){
@@ -3640,6 +3898,15 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){
   }
 
 trans_begun:
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( bConcurrent && rc==SQLITE_OK && sqlite3PagerIsWal(pBt->pPager) ){
+    rc = sqlite3PagerBeginConcurrent(pBt->pPager);
+    if( rc==SQLITE_OK && wrflag ){
+      rc = btreePtrmapAllocate(pBt);
+    }
+  }
+#endif
+
   if( rc==SQLITE_OK ){
     if( pSchemaVersion ){
       *pSchemaVersion = get4byte(&pBt->pPage1->aData[40]);
@@ -3649,7 +3916,11 @@ trans_begun:
       ** open savepoints. If the second parameter is greater than 0 and
       ** the sub-journal is not already open, then it will be opened here.
       */
-      rc = sqlite3PagerOpenSavepoint(pPager, p->db->nSavepoint);
+      int nSavepoint = p->db->nSavepoint;
+      rc = sqlite3PagerOpenSavepoint(pPager, nSavepoint);
+      if( rc==SQLITE_OK && nSavepoint ){
+        rc = btreePtrmapBegin(pBt, nSavepoint);
+      }
     }
   }
 
@@ -4114,6 +4385,175 @@ static int autoVacuumCommit(Btree *p){
 # define setChildPtrmaps(x) SQLITE_OK
 #endif
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** This function is called as part of merging an CONCURRENT transaction with
+** the snapshot at the head of the wal file. It relocates all pages in the
+** range iFirst..iLast, inclusive. It is assumed that the BtreePtrmap 
+** structure at BtShared.pMap contains the location of the pointers to each
+** page in the range.
+**
+** If pnCurrent is NULL, then all pages in the range are moved to currently
+** free locations (i.e. free-list entries) within the database file before page
+** iFirst.
+**
+** Or, if pnCurrent is not NULL, then it points to a value containing the
+** current size of the database file in pages. In this case, all pages are
+** relocated to the end of the database file - page iFirst is relocated to
+** page (*pnCurrent+1), page iFirst+1 to page (*pnCurrent+2), and so on.
+** Value *pnCurrent is set to the new size of the database before this 
+** function returns.
+**
+** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error code.
+*/
+static int btreeRelocateRange(
+  BtShared *pBt,                  /* B-tree handle */
+  Pgno iFirst,                    /* First page to relocate */
+  Pgno iLast,                     /* Last page to relocate */
+  Pgno *pnCurrent                 /* If not NULL, IN/OUT: Database size */
+){
+  int rc = SQLITE_OK;
+  BtreePtrmap *pMap = pBt->pMap;
+  Pgno iPg;
+
+  for(iPg=iFirst; iPg<=iLast && rc==SQLITE_OK; iPg++){
+    MemPage *pFree = 0;     /* Page allocated from free-list */
+    MemPage *pPg = 0;
+    Pgno iNew;              /* New page number for pPg */
+    PtrmapEntry *pEntry;    /* Pointer map entry for page iPg */
+
+    if( iPg==PENDING_BYTE_PAGE(pBt) ) continue;
+    pEntry = &pMap->aPtr[iPg - pMap->iFirst];
+
+    if( pEntry->eType==PTRMAP_FREEPAGE ){
+      Pgno dummy;
+      rc = allocateBtreePage(pBt, &pFree, &dummy, iPg, BTALLOC_EXACT);
+      if( pFree ){
+        assert( sqlite3PagerPageRefcount(pFree->pDbPage)==1 );
+        sqlite3PcacheDrop(pFree->pDbPage);
+      }
+      assert( rc!=SQLITE_OK || dummy==iPg );
+    }else if( pnCurrent ){
+      btreeGetPage(pBt, iPg, &pPg, 0);
+      assert( sqlite3PagerIswriteable(pPg->pDbPage) );
+      assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 );
+      iNew = ++(*pnCurrent);
+      if( iNew==PENDING_BYTE_PAGE(pBt) ) iNew = ++(*pnCurrent);
+      rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent, iNew, 1);
+      releasePageNotNull(pPg);
+    }else{
+      rc = allocateBtreePage(pBt, &pFree, &iNew, iFirst-1, BTALLOC_LE);
+      assert( rc!=SQLITE_OK || iNew<iFirst );
+      if( rc==SQLITE_OK ){
+        releasePage(pFree);
+        btreeGetPage(pBt, iPg, &pPg, 0);
+        rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent,iNew,1);
+        releasePage(pPg);
+      }
+    }
+  }
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** The b-tree handle passed as the only argument is about to commit an
+** CONCURRENT transaction. At this point it is guaranteed that this is 
+** possible - the wal WRITER lock is held and it is known that there are 
+** no conflicts with committed transactions.
+*/
+static int btreeFixUnlocked(Btree *p){
+  BtShared *pBt = p->pBt;
+  MemPage *pPage1 = pBt->pPage1;
+  u8 *p1 = pPage1->aData;
+  Pager *pPager = pBt->pPager;
+  int rc = SQLITE_OK;
+
+  /* If page 1 of the database is not writable, then no pages were allocated
+  ** or freed by this transaction. In this case no special handling is 
+  ** required. Otherwise, if page 1 is dirty, proceed.  */
+  BtreePtrmap *pMap = pBt->pMap;
+  Pgno iTrunk = get4byte(&p1[32]);
+  Pgno nPage = btreePagecount(pBt);
+  u32 nFree = get4byte(&p1[36]);
+
+  assert( pBt->pMap );
+  rc = sqlite3PagerUpgradeSnapshot(pPager, pPage1->pDbPage);
+  assert( p1==pPage1->aData );
+
+  if( rc==SQLITE_OK ){
+    Pgno nHPage = get4byte(&p1[28]);
+    Pgno nFin = nHPage;         /* Size of db after transaction merge */
+
+    if( sqlite3PagerIswriteable(pPage1->pDbPage) ){
+      Pgno iHTrunk = get4byte(&p1[32]);
+      u32 nHFree = get4byte(&p1[36]);
+
+      btreePtrmapCheck(pBt, nPage);
+
+      /* Attach the head database free list to the end of the current
+      ** transactions free-list (if any).  */
+      if( iTrunk!=0 ){
+        put4byte(&p1[36], nHFree + nFree);
+        put4byte(&p1[32], iTrunk);
+        while( iTrunk ){
+          DbPage *pTrunk = sqlite3PagerLookup(pPager, iTrunk);
+          iTrunk = get4byte((u8*)pTrunk->pData);
+          if( iTrunk==0 ){
+            put4byte((u8*)pTrunk->pData, iHTrunk);
+          }
+          sqlite3PagerUnref(pTrunk);
+        };
+      }
+
+      if( nHPage<(pMap->iFirst-1) ){
+        /* The database consisted of (pMap->iFirst-1) pages when the current
+        ** concurrent transaction was opened. And an concurrent transaction may
+        ** not be executed on an auto-vacuum database - so the db should 
+        ** not have shrunk since the transaction was opened. Therefore nHPage
+        ** should be set to (pMap->iFirst-1) or greater. */
+        rc = SQLITE_CORRUPT_BKPT;
+      }else{
+        /* The current transaction allocated pages pMap->iFirst through
+        ** nPage (inclusive) at the end of the database file. Meanwhile,
+        ** other transactions have allocated (iFirst..nHPage). So move
+        ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1). */
+        Pgno iLast = MIN(nPage, nHPage);    /* Last page to move */
+        Pgno nCurrent;                      /* Current size of db */
+
+        nCurrent = MAX(nPage, nHPage);
+        pBt->nPage = nCurrent;
+        rc = btreeRelocateRange(pBt, pMap->iFirst, iLast, &nCurrent);
+
+        /* There are now no collisions with the snapshot at the head of the
+        ** database file. So at this point it would be possible to write
+        ** the transaction out to disk. Before doing so though, attempt to
+        ** relocate some of the new pages to free locations within the body
+        ** of the database file (i.e. free-list entries). */
+        if( rc==SQLITE_OK ){
+          assert( nCurrent!=PENDING_BYTE_PAGE(pBt) );
+          sqlite3PagerSetDbsize(pBt->pPager, nCurrent);
+          nFree = get4byte(&p1[36]);
+          nFin = nCurrent-nFree;
+          if( nCurrent>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){
+            nFin--;
+          }
+          nFin = MAX(nFin, nHPage);
+          rc = btreeRelocateRange(pBt, nFin+1, nCurrent, 0);
+        }
+
+        put4byte(&p1[28], nFin);
+      }
+    }
+    sqlite3PagerSetDbsize(pPager, nFin);
+  }
+
+  return rc;
+}
+#else
+# define btreeFixUnlocked(X)  SQLITE_OK
+#endif /* SQLITE_OMIT_CONCURRENT */
+
 /*
 ** This routine does the first phase of a two-phase commit.  This routine
 ** causes a rollback journal to be created (if it does not already exist)
@@ -4145,8 +4585,10 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zSuperJrnl){
   if( p->inTrans==TRANS_WRITE ){
     BtShared *pBt = p->pBt;
     sqlite3BtreeEnter(p);
+
 #ifndef SQLITE_OMIT_AUTOVACUUM
     if( pBt->autoVacuum ){
+      assert( ISCONCURRENT==0 );
       rc = autoVacuumCommit(p);
       if( rc!=SQLITE_OK ){
         sqlite3BtreeLeave(p);
@@ -4157,7 +4599,12 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zSuperJrnl){
       sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage);
     }
 #endif
-    rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zSuperJrnl, 0);
+    if( rc==SQLITE_OK && ISCONCURRENT && p->db->eConcurrent==CONCURRENT_OPEN ){
+      rc = btreeFixUnlocked(p);
+    }
+    if( rc==SQLITE_OK ){
+      rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zSuperJrnl, 0);
+    }
     sqlite3BtreeLeave(p);
   }
   return rc;
@@ -4200,6 +4647,11 @@ static void btreeEndTransaction(Btree *p){
     unlockBtreeIfUnused(pBt);
   }
 
+  /* If this was an CONCURRENT transaction, delete the pBt->pMap object.
+  ** Also call PagerEndConcurrent() to ensure that the pager has discarded
+  ** the record of all pages read within the transaction.  */
+  btreePtrmapDelete(pBt);
+  sqlite3PagerEndConcurrent(pBt->pPager);
   btreeIntegrity(p);
 }
 
@@ -4429,6 +4881,9 @@ int sqlite3BtreeBeginStmt(Btree *p, int iStatement){
   ** such savepoints while the statement transaction savepoint is active.
   */
   rc = sqlite3PagerOpenSavepoint(pBt->pPager, iStatement);
+  if( rc==SQLITE_OK ){
+    rc = btreePtrmapBegin(pBt, iStatement);
+  }
   sqlite3BtreeLeave(p);
   return rc;
 }
@@ -4452,6 +4907,7 @@ int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){
     assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK );
     assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) );
     sqlite3BtreeEnter(p);
+    btreePtrmapEnd(pBt, op, iSavepoint);
     if( op==SAVEPOINT_ROLLBACK ){
       rc = saveAllCursors(pBt, 0, 0);
     }
@@ -5249,6 +5705,9 @@ const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){
 ** vice-versa).
 */
 static int moveToChild(BtCursor *pCur, u32 newPgno){
+  BtShared *pBt = pCur->pBt;
+  int rc;
+
   assert( cursorOwnsBtShared(pCur) );
   assert( pCur->eState==CURSOR_VALID );
   assert( pCur->iPage<BTCURSOR_MAX_DEPTH );
@@ -5262,8 +5721,12 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){
   pCur->apPage[pCur->iPage] = pCur->pPage;
   pCur->ix = 0;
   pCur->iPage++;
-  return getAndInitPage(pCur->pBt, newPgno, &pCur->pPage, pCur,
-                        pCur->curPagerFlags);
+  rc = getAndInitPage(pBt, newPgno, &pCur->pPage,
+                        pCur, pCur->curPagerFlags);
+  if( rc==SQLITE_OK ){
+    setMempageRoot(pCur->pPage, pCur->pgnoRoot);
+  }
+  return rc;
 }
 
 #ifdef SQLITE_DEBUG
@@ -5375,6 +5838,7 @@ static int moveToRoot(BtCursor *pCur){
       pCur->eState = CURSOR_INVALID;
       return rc;
     }
+    setMempageRoot(pCur->pPage, pCur->pgnoRoot);
     pCur->iPage = 0;
     pCur->curIntKey = pCur->pPage->intKey;
   }
@@ -6257,7 +6721,7 @@ static int allocateBtreePage(
   Pgno mxPage;     /* Total size of the database file */
 
   assert( sqlite3_mutex_held(pBt->mutex) );
-  assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) );
+  assert( eMode==BTALLOC_ANY || (nearby>0 && REQUIRE_PTRMAP ) );
   pPage1 = pBt->pPage1;
   mxPage = btreePagecount(pBt);
   /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36
@@ -6267,6 +6731,15 @@ static int allocateBtreePage(
   if( n>=mxPage ){
     return SQLITE_CORRUPT_BKPT;
   }
+
+  /* Ensure page 1 is writable. This function will either change the number
+  ** of pages in the free-list or the size of the database file. Since both
+  ** of these operations involve modifying page 1 header fields, page 1
+  ** will definitely be written by this transaction. If this is an CONCURRENT
+  ** transaction, ensure the BtreePtrmap structure has been allocated.  */
+  rc = sqlite3PagerWrite(pPage1->pDbPage);
+  if( rc ) return rc;
+
   if( n>0 ){
     /* There are pages on the freelist.  Reuse one of those pages. */
     Pgno iTrunk;
@@ -6277,28 +6750,29 @@ static int allocateBtreePage(
     ** shows that the page 'nearby' is somewhere on the free-list, then
     ** the entire-list will be searched for that page.
     */
-#ifndef SQLITE_OMIT_AUTOVACUUM
     if( eMode==BTALLOC_EXACT ){
-      if( nearby<=mxPage ){
-        u8 eType;
-        assert( nearby>0 );
-        assert( pBt->autoVacuum );
-        rc = ptrmapGet(pBt, nearby, &eType, 0);
-        if( rc ) return rc;
-        if( eType==PTRMAP_FREEPAGE ){
-          searchList = 1;
+      assert( ISAUTOVACUUM!=ISCONCURRENT );
+      if( ISAUTOVACUUM ){
+        if( nearby<=mxPage ){
+          u8 eType;
+          assert( nearby>0 );
+          assert( pBt->autoVacuum );
+          rc = ptrmapGet(pBt, nearby, &eType, 0);
+          if( rc ) return rc;
+          if( eType==PTRMAP_FREEPAGE ){
+            searchList = 1;
+          }
         }
+      }else{
+        searchList = 1;
       }
     }else if( eMode==BTALLOC_LE ){
       searchList = 1;
     }
-#endif
 
     /* Decrement the free-list count by 1. Set iTrunk to the index of the
     ** first free-list trunk page. iPrevTrunk is initially 1.
     */
-    rc = sqlite3PagerWrite(pPage1->pDbPage);
-    if( rc ) return rc;
     put4byte(&pPage1->aData[36], n-1);
 
     /* The code within this loop is run only once if the 'searchList' variable
@@ -6606,7 +7080,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){
   /* If the database supports auto-vacuum, write an entry in the pointer-map
   ** to indicate that the page is free.
   */
-  if( ISAUTOVACUUM ){
+  if( REQUIRE_PTRMAP ){
     ptrmapPut(pBt, iPage, PTRMAP_FREEPAGE, 0, &rc);
     if( rc ) goto freepage_out;
   }
@@ -6937,7 +7411,7 @@ static int fillInCell(
       }
 #endif
       rc = allocateBtreePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl, 0);
-#ifndef SQLITE_OMIT_AUTOVACUUM
+
       /* If the database supports auto-vacuum, and the second or subsequent
       ** overflow page is being allocated, add an entry to the pointer-map
       ** for that page now. 
@@ -6948,14 +7422,13 @@ static int fillInCell(
       ** may misinterpret the uninitialized values and delete the
       ** wrong pages from the database.
       */
-      if( pBt->autoVacuum && rc==SQLITE_OK ){
+      if( REQUIRE_PTRMAP && rc==SQLITE_OK ){
         u8 eType = (pgnoPtrmap?PTRMAP_OVERFLOW2:PTRMAP_OVERFLOW1);
         ptrmapPut(pBt, pgnoOvfl, eType, pgnoPtrmap, &rc);
         if( rc ){
           releasePage(pOvfl);
         }
       }
-#endif
       if( rc ){
         releasePage(pToRelease);
         return rc;
@@ -7096,6 +7569,7 @@ static void insertCell(
     assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */
     assert( j==0 || i==pPage->aiOvfl[j-1]+1 );   /* Overflows are sequential */
   }else{
+    BtShared *pBt = pPage->pBt;
     int rc = sqlite3PagerWrite(pPage->pDbPage);
     if( rc!=SQLITE_OK ){
       *pRC = rc;
@@ -7110,7 +7584,7 @@ static void insertCell(
     ** if it returns successfully */
     assert( idx >= 0 );
     assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB );
-    assert( idx+sz <= (int)pPage->pBt->usableSize );
+    assert( idx+sz <= (int)pBt->usableSize );
     pPage->nFree -= (u16)(2 + sz);
     if( iChild ){
       /* In a corrupt database where an entry in the cell index section of
@@ -7130,14 +7604,12 @@ static void insertCell(
     /* increment the cell count */
     if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++;
     assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB );
-#ifndef SQLITE_OMIT_AUTOVACUUM
-    if( pPage->pBt->autoVacuum ){
+    if( REQUIRE_PTRMAP ){
       /* The cell may contain a pointer to an overflow page. If so, write
       ** the entry for the overflow page into the pointer map.
       */
       ptrmapPutOvflPtr(pPage, pPage, pCell, pRC);
     }
-#endif
   }
 }
 
@@ -7677,7 +8149,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){
     ** be marked as dirty. Returning an error code will cause a
     ** rollback, undoing any changes made to the parent page.
     */
-    if( ISAUTOVACUUM ){
+    if( REQUIRE_PTRMAP ){
       ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc);
       if( szCell>pNew->minLocal ){
         ptrmapPutOvflPtr(pNew, pNew, pCell, &rc);
@@ -7815,7 +8287,7 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){
     /* If this is an auto-vacuum database, update the pointer-map entries
     ** for any b-tree or overflow pages that pTo now contains the pointers to.
     */
-    if( ISAUTOVACUUM ){
+    if( REQUIRE_PTRMAP ){
       *pRC = setChildPtrmaps(pTo);
     }
   }
@@ -7866,7 +8338,8 @@ static int balance_nonroot(
   int iParentIdx,                 /* Index of "the page" in pParent */
   u8 *aOvflSpace,                 /* page-size bytes of space for parent ovfl */
   int isRoot,                     /* True if pParent is a root-page */
-  int bBulk                       /* True if this call is part of a bulk load */
+  int bBulk,                      /* True if this call is part of a bulk load */
+  Pgno pgnoRoot                   /* Root page of b-tree being balanced */
 ){
   BtShared *pBt;               /* The whole database */
   int nMaxCells = 0;           /* Allocated size of apCell, szCell, aFrom. */
@@ -7961,6 +8434,7 @@ static int balance_nonroot(
         goto balance_cleanup;
       }
     }
+    setMempageRoot(apOld[i], pgnoRoot);
     nMaxCells += apOld[i]->nCell + ArraySize(pParent->apOvfl);
     if( (i--)==0 ) break;
 
@@ -8301,7 +8775,7 @@ static int balance_nonroot(
       cntOld[i] = b.nCell;
 
       /* Set the pointer-map entry for the new sibling page. */
-      if( ISAUTOVACUUM ){
+      if( REQUIRE_PTRMAP ){
         ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc);
         if( rc!=SQLITE_OK ){
           goto balance_cleanup;
@@ -8394,7 +8868,7 @@ static int balance_nonroot(
   ** updated. This happens below, after the sibling pages have been 
   ** populated, not here.
   */
-  if( ISAUTOVACUUM ){
+  if( REQUIRE_PTRMAP ){
     MemPage *pOld;
     MemPage *pNew = pOld = apNew[0];
     int cntOldNext = pNew->nCell + pNew->nOverflow;
@@ -8587,7 +9061,7 @@ static int balance_nonroot(
     );
     copyNodeContent(apNew[0], pParent, &rc);
     freePage(apNew[0], &rc);
-  }else if( ISAUTOVACUUM && !leafCorrection ){
+  }else if( REQUIRE_PTRMAP && !leafCorrection ){
     /* Fix the pointer map entries associated with the right-child of each
     ** sibling page. All other pointer map entries have already been taken
     ** care of.  */
@@ -8670,7 +9144,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){
   if( rc==SQLITE_OK ){
     rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0);
     copyNodeContent(pRoot, pChild, &rc);
-    if( ISAUTOVACUUM ){
+    if( REQUIRE_PTRMAP ){
       ptrmapPut(pBt, pgnoChild, PTRMAP_BTREE, pRoot->pgno, &rc);
     }
   }
@@ -8828,7 +9302,7 @@ static int balance(BtCursor *pCur){
           */
           u8 *pSpace = sqlite3PageMalloc(pCur->pBt->pageSize);
           rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1,
-                               pCur->hints&BTREE_BULKLOAD);
+                               pCur->hints&BTREE_BULKLOAD, pCur->pgnoRoot);
           if( pFree ){
             /* If pFree is not NULL, it points to the pSpace buffer used 
             ** by a previous call to balance_nonroot(). Its contents are
@@ -9197,7 +9671,7 @@ int sqlite3BtreeInsert(
     testcase( pCur->curFlags & BTCF_ValidOvfl );
     invalidateOverflowCache(pCur);
     if( info.nSize==szNew && info.nLocal==info.nPayload 
-     && (!ISAUTOVACUUM || szNew<pPage->minLocal)
+     && (!REQUIRE_PTRMAP || szNew<pPage->minLocal)
     ){
       /* Overwrite the old cell with the new if they are the same size.
       ** We could also try to do this if the old cell is smaller, then add
@@ -9784,7 +10258,8 @@ static int clearDatabasePage(
   BtShared *pBt,           /* The BTree that contains the table */
   Pgno pgno,               /* Page number to clear */
   int freePageFlag,        /* Deallocate page if true */
-  i64 *pnChange            /* Add number of Cells freed to this counter */
+  i64 *pnChange,           /* Add number of Cells freed to this counter */
+  Pgno pgnoRoot
 ){
   MemPage *pPage;
   int rc;
@@ -9799,6 +10274,7 @@ static int clearDatabasePage(
   }
   rc = getAndInitPage(pBt, pgno, &pPage, 0, 0);
   if( rc ) return rc;
+  setMempageRoot(pPage, pgnoRoot);
   if( (pBt->openFlags & BTREE_SINGLE)==0 
    && sqlite3PagerPageRefcount(pPage->pDbPage) != (1 + (pgno==1))
   ){
@@ -9809,14 +10285,16 @@ static int clearDatabasePage(
   for(i=0; i<pPage->nCell; i++){
     pCell = findCell(pPage, i);
     if( !pPage->leaf ){
-      rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange);
+      rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange, pgnoRoot);
       if( rc ) goto cleardatabasepage_out;
     }
     BTREE_CLEAR_CELL(rc, pPage, pCell, info);
     if( rc ) goto cleardatabasepage_out;
   }
   if( !pPage->leaf ){
-    rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange);
+    rc = clearDatabasePage(
+        pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange, pgnoRoot
+    );
     if( rc ) goto cleardatabasepage_out;
     if( pPage->intKey ) pnChange = 0;
   }
@@ -9862,7 +10340,7 @@ int sqlite3BtreeClearTable(Btree *p, int iTable, i64 *pnChange){
     if( p->hasIncrblobCur ){
       invalidateIncrblobCursors(p, (Pgno)iTable, 0, 1);
     }
-    rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange);
+    rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange, (Pgno)iTable);
   }
   sqlite3BtreeLeave(p);
   return rc;
@@ -10769,10 +11247,12 @@ char *sqlite3BtreeIntegrityCheck(
   }
   pBt->db->flags = savedDbFlags;
 
-  /* Make sure every page in the file is referenced
-  */
+  /* Make sure every page in the file is referenced. Skip this if the
+  ** database is currently being written by a CONCURRENT transaction (it 
+  ** may fail as pages that were part of the free-list when the transaction
+  ** was opened cannot be counted).  */
   if( !bPartial ){
-    for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){
+    for(i=1; ISCONCURRENT==0 && i<=sCheck.nPage && sCheck.mxErr; i++){
 #ifdef SQLITE_OMIT_AUTOVACUUM
       if( getPageReferenced(&sCheck, i)==0 ){
         checkAppendMsg(&sCheck, "Page %d is never used", i);
@@ -11025,7 +11505,7 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){
   BtShared *pBt = pBtree->pBt;
   int rc;                         /* Return code */
  
-  assert( iVersion==1 || iVersion==2 );
+  assert( iVersion==1 || iVersion==2 || iVersion==3 );
 
   /* If setting the version fields to 1, do not automatically open the
   ** WAL connection, even if the version fields are currently set to 2.
@@ -11072,6 +11552,102 @@ int sqlite3BtreeIsReadonly(Btree *p){
 */
 int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); }
 
+/*
+** This function is called to ensure that all locks required to commit the
+** current write-transaction to the database file are held. If the db is
+** in rollback mode, this means the EXCLUSIVE lock on the database file.
+**
+** Or, if this is an CONCURRENT transaction on a wal-mode database, the WRITER
+** lock on the wal file. In this case this function also checks that the
+** CONCURRENT transaction can be safely committed (does not commit with any
+** other transaction committed since it was opened).
+**
+** SQLITE_OK is returned if successful. SQLITE_BUSY if the required locks
+** cannot be obtained due to a conflicting lock. If the locks cannot be
+** obtained for an CONCURRENT transaction due to a conflict with an already
+** committed transaction, SQLITE_BUSY_SNAPSHOT is returned. Otherwise, if
+** some other error (OOM, IO, etc.) occurs, the relevant SQLite error code
+** is returned.
+*/
+int sqlite3BtreeExclusiveLock(Btree *p){
+  int rc;
+  Pgno pgno = 0;
+  BtShared *pBt = p->pBt;
+  assert( p->inTrans==TRANS_WRITE && pBt->pPage1 );
+  sqlite3BtreeEnter(p);
+  rc = sqlite3PagerExclusiveLock(pBt->pPager, 
+    (p->db->eConcurrent==CONCURRENT_SCHEMA) ? 0 : pBt->pPage1->pDbPage,
+    &pgno
+  );
+#ifdef SQLITE_OMIT_CONCURRENT
+  assert( pgno==0 );
+#else
+  if( rc==SQLITE_BUSY_SNAPSHOT && pgno ){
+    PgHdr *pPg = 0;
+    int rc2 = sqlite3PagerGet(pBt->pPager, pgno, &pPg, 0);
+    if( rc2==SQLITE_OK ){
+      int bWrite = -1;
+      const char *zObj = 0;
+      const char *zTab = 0;
+      char zContent[17];
+
+      if( pPg ){
+        Pgno pgnoRoot = 0;
+        HashElem *pE;
+        Schema *pSchema;
+        u8 *aData = (u8*)sqlite3PagerGetData(pPg);
+        int i;
+        for(i=0; i<8; i++){
+          static const char hexdigits[] = {
+            '0', '1', '2', '3', '4', '5', '6', '7',
+            '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' 
+          };
+          zContent[i*2] = hexdigits[(aData[i] >> 4)];
+          zContent[i*2+1] = hexdigits[(aData[i] & 0xF)];
+        }
+        zContent[16] = '\0';
+
+        pgnoRoot = ((MemPage*)sqlite3PagerGetExtra(pPg))->pgnoRoot;
+        bWrite = sqlite3PagerIswriteable(pPg);
+        sqlite3PagerUnref(pPg);
+
+        pSchema = sqlite3SchemaGet(p->db, p);
+        if( pSchema ){
+          for(pE=sqliteHashFirst(&pSchema->tblHash); pE; pE=sqliteHashNext(pE)){
+            Table *pTab = (Table *)sqliteHashData(pE);
+            if( pTab->tnum==(int)pgnoRoot ){
+              zObj = pTab->zName;
+              zTab = 0;
+            }else{
+              Index *pIdx;
+              for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
+                if( pIdx->tnum==(int)pgnoRoot ){
+                  zObj = pIdx->zName;
+                  zTab = pTab->zName;
+                }
+              }
+            }
+          }
+        }
+      }
+
+      sqlite3_log(SQLITE_OK,
+          "cannot commit CONCURRENT transaction "
+          "- conflict at page %d "
+          "(%s page; part of db %s %s%s%s; content=%s...)",
+          (int)pgno,
+          (bWrite==0?"read-only":(bWrite>0?"read/write":"unknown")),
+          (zTab ? "index" : "table"),
+          (zTab ? zTab : ""), (zTab ? "." : ""), (zObj ? zObj : "UNKNOWN"),
+          zContent
+      );
+    }
+  }
+#endif
+  sqlite3BtreeLeave(p);
+  return rc;
+}
+
 #if !defined(SQLITE_OMIT_SHARED_CACHE)
 /*
 ** Return true if the Btree passed as the only argument is sharable.
@@ -11090,3 +11666,8 @@ int sqlite3BtreeConnectionCount(Btree *p){
   return p->pBt->nRef;
 }
 #endif
+
+void sqlite3BtreeOpenTransTm(Btree *p, u64 *a){
+  p->aOpenTransTm = a;
+  sqlite3PagerOpenTransTm(p->pBt->pPager, a);
+}

src/btree.h

@@ -350,6 +350,8 @@ sqlite3_uint64 sqlite3BtreeSeekCount(Btree*);
 # define sqlite3BtreeSeekCount(X) 0
 #endif
 
+int sqlite3BtreeExclusiveLock(Btree *pBt);
+
 #ifndef NDEBUG
 int sqlite3BtreeCursorIsValid(BtCursor*);
 #endif
@@ -408,5 +410,6 @@ int sqlite3BtreeTransferRow(BtCursor*, BtCursor*, i64);
 # define sqlite3SchemaMutexHeld(X,Y,Z) 1
 #endif
 
+void sqlite3BtreeOpenTransTm(Btree *p, u64 *a);
 
 #endif /* SQLITE_BTREE_H */

src/btreeInt.h

@@ -232,6 +232,7 @@
 typedef struct MemPage MemPage;
 typedef struct BtLock BtLock;
 typedef struct CellInfo CellInfo;
+typedef struct BtreePtrmap BtreePtrmap;
 
 /*
 ** This is a magic string that appears at the beginning of every
@@ -275,6 +276,9 @@ struct MemPage {
   u8 intKey;           /* True if table b-trees.  False for index b-trees */
   u8 intKeyLeaf;       /* True if the leaf of an intKey table */
   Pgno pgno;           /* Page number for this page */
+#ifndef SQLITE_OMIT_CONCURRENT
+  Pgno pgnoRoot;       /* Root page of b-tree that this page belongs to */
+#endif
   /* Only the first 8 bytes (above) are zeroed by pager.c when a new page
   ** is allocated. All fields that follow must be initialized before use */
   u8 leaf;             /* True if a leaf page */
@@ -360,6 +364,7 @@ struct Btree {
 #ifndef SQLITE_OMIT_SHARED_CACHE
   BtLock lock;       /* Object used to lock page 1 */
 #endif
+  u64 *aOpenTransTm;
 };
 
 /*
@@ -456,6 +461,9 @@ struct BtShared {
   Btree *pWriter;       /* Btree with currently open write transaction */
 #endif
   u8 *pTmpSpace;        /* Temp space sufficient to hold a single cell */
+#ifndef SQLITE_OMIT_CONCURRENT
+  BtreePtrmap *pMap;
+#endif
   int nPreformatSize;   /* Size of last cell written by TransferRow() */
 };
 
@@ -673,12 +681,19 @@ struct BtCursor {
 ** (sqliteMallocRaw), it is not possible to use conditional compilation.
 ** So, this macro is defined instead.
 */
-#ifndef SQLITE_OMIT_AUTOVACUUM
-#define ISAUTOVACUUM (pBt->autoVacuum)
-#else
+#ifdef SQLITE_OMIT_AUTOVACUUM
 #define ISAUTOVACUUM 0
+#else
+#define ISAUTOVACUUM (pBt->autoVacuum)
 #endif
 
+#ifdef SQLITE_OMIT_CONCURRENT
+# define ISCONCURRENT 0
+#else
+# define ISCONCURRENT (pBt->pMap!=0)
+#endif
+
+#define REQUIRE_PTRMAP (ISAUTOVACUUM || ISCONCURRENT)
 
 /*
 ** This structure is passed around through all the sanity checking routines

src/build.c

@@ -5118,7 +5118,7 @@ void sqlite3BeginTransaction(Parse *pParse, int type){
   }
   v = sqlite3GetVdbe(pParse);
   if( !v ) return;
-  if( type!=TK_DEFERRED ){
+  if( type==TK_IMMEDIATE || type==TK_EXCLUSIVE ){
     for(i=0; i<db->nDb; i++){
       int eTxnType;
       Btree *pBt = db->aDb[i].pBt;
@@ -5133,7 +5133,7 @@ void sqlite3BeginTransaction(Parse *pParse, int type){
       sqlite3VdbeUsesBtree(v, i);
     }
   }
-  sqlite3VdbeAddOp0(v, OP_AutoCommit);
+  sqlite3VdbeAddOp3(v, OP_AutoCommit, 0, 0, (type==TK_CONCURRENT));
 }
 
 /*

src/func.c

@@ -523,8 +523,9 @@ static void randomFunc(
   sqlite3_value **NotUsed2
 ){
   sqlite_int64 r;
+  sqlite3 *db = sqlite3_context_db_handle(context);
   UNUSED_PARAMETER2(NotUsed, NotUsed2);
-  sqlite3_randomness(sizeof(r), &r);
+  sqlite3FastRandomness(&db->sPrng, sizeof(r), &r);
   if( r<0 ){
     /* We need to prevent a random number of 0x8000000000000000 
     ** (or -9223372036854775808) since when you do abs() of that
@@ -550,6 +551,7 @@ static void randomBlob(
 ){
   sqlite3_int64 n;
   unsigned char *p;
+  sqlite3 *db = sqlite3_context_db_handle(context);
   assert( argc==1 );
   UNUSED_PARAMETER(argc);
   n = sqlite3_value_int64(argv[0]);
@@ -558,7 +560,7 @@ static void randomBlob(
   }
   p = contextMalloc(context, n);
   if( p ){
-    sqlite3_randomness(n, p);
+    sqlite3FastRandomness(&db->sPrng, n, p);
     sqlite3_result_blob(context, (char*)p, n, sqlite3_free);
   }
 }

src/main.c

@@ -3245,7 +3245,7 @@ static int openDatabase(
   db->aDb = db->aDbStatic;
   db->lookaside.bDisable = 1;
   db->lookaside.sz = 0;
-
+  sqlite3FastPrngInit(&db->sPrng);
   assert( sizeof(db->aLimit)==sizeof(aHardLimit) );
   memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit));
   db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS;
@@ -4815,6 +4815,35 @@ void sqlite3_snapshot_free(sqlite3_snapshot *pSnapshot){
 }
 #endif /* SQLITE_ENABLE_SNAPSHOT */
 
+SQLITE_EXPERIMENTAL int sqlite3_wal_info(
+  sqlite3 *db, const char *zDb, 
+  unsigned int *pnPrior, unsigned int *pnFrame
+){
+  int rc = SQLITE_OK;
+
+#ifndef SQLITE_OMIT_WAL
+  Btree *pBt;
+  int iDb;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( !sqlite3SafetyCheckOk(db) ){
+    return SQLITE_MISUSE_BKPT;
+  }
+#endif
+
+  sqlite3_mutex_enter(db->mutex);
+  iDb = sqlite3FindDbName(db, zDb);
+  if( iDb<0 ){
+    return SQLITE_ERROR;
+  }
+  pBt = db->aDb[iDb].pBt;
+  rc = sqlite3PagerWalInfo(sqlite3BtreePager(pBt), pnPrior, pnFrame);
+  sqlite3_mutex_leave(db->mutex);
+#endif   /* SQLITE_OMIT_WAL */
+
+  return rc;
+}
+
 #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS
 /*
 ** Given the name of a compile-time option, return true if that option

src/os_unix.c

@@ -46,6 +46,16 @@
 #include "sqliteInt.h"
 #if SQLITE_OS_UNIX              /* This file is used on unix only */
 
+/* Turn this feature on in all builds for now */
+#define SQLITE_MUTEXFREE_SHMLOCK 1
+#define SQLITE_MFS_EXCLUSIVE     255
+#ifndef SQLITE_MFS_NSHARD
+# define SQLITE_MFS_NSHARD       8
+#endif
+#if SQLITE_MFS_NSHARD<1
+# error "SQLITE_MFS_NSHARD must be greater than 0"
+#endif
+
 /*
 ** There are various methods for file locking used for concurrency
 ** control:
@@ -1177,6 +1187,10 @@ struct unixInodeInfo {
   sem_t *pSem;                    /* Named POSIX semaphore */
   char aSemName[MAX_PATHNAME+2];  /* Name of that semaphore */
 #endif
+#ifdef SQLITE_SHARED_MAPPING
+  sqlite3_int64 nSharedMapping;   /* Size of mapped region in bytes */
+  void *pSharedMapping;           /* Memory mapped region */
+#endif
 };
 
 /*
@@ -1329,6 +1343,13 @@ static void releaseInodeInfo(unixFile *pFile){
     pInode->nRef--;
     if( pInode->nRef==0 ){
       assert( pInode->pShmNode==0 );
+#ifdef SQLITE_SHARED_MAPPING
+      if( pInode->pSharedMapping ){
+        osMunmap(pInode->pSharedMapping, pInode->nSharedMapping);
+        pInode->pSharedMapping = 0;
+        pInode->nSharedMapping = 0;
+      }
+#endif
       sqlite3_mutex_enter(pInode->pLockMutex);
       closePendingFds(pFile);
       sqlite3_mutex_leave(pInode->pLockMutex);
@@ -2192,6 +2213,14 @@ static int nolockUnlock(sqlite3_file *NotUsed, int NotUsed2){
 ** Close the file.
 */
 static int nolockClose(sqlite3_file *id) {
+#ifdef SQLITE_SHARED_MAPPING
+  unixFile *pFd = (unixFile*)id;
+  if( pFd->pInode ){
+    unixEnterMutex();
+    releaseInodeInfo(pFd);
+    unixLeaveMutex();
+  }
+#endif
   return closeUnixFile(id);
 }
 
@@ -4045,6 +4074,9 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){
       *(i64*)pArg = pFile->mmapSizeMax;
       if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){
         pFile->mmapSizeMax = newLimit;
+#ifdef SQLITE_SHARED_MAPPING
+        if( pFile->pInode==0 )
+#endif
         if( pFile->mmapSize>0 ){
           unixUnmapfile(pFile);
           rc = unixMapfile(pFile, -1);
@@ -4294,8 +4326,41 @@ struct unixShmNode {
   u8 sharedMask;             /* Mask of shared locks held */
   u8 nextShmId;              /* Next available unixShm.id value */
 #endif
+
+#ifdef SQLITE_MUTEXFREE_SHMLOCK
+  /* In unix-excl mode, if SQLITE_MUTEXFREE_SHMLOCK is defined, all locks
+  ** are stored in the following 64-bit value. There are in total 8 
+  ** shm-locking slots, each of which are assigned 8-bits from the 64-bit
+  ** value. The least-significant 8 bits correspond to shm-locking slot
+  ** 0, and so on.
+  **
+  ** If the 8-bits corresponding to a shm-locking locking slot are set to
+  ** 0xFF, then a write-lock is held on the slot. Or, if they are set to
+  ** a non-zero value smaller than 0xFF, then they represent the total 
+  ** number of read-locks held on the slot. There is no way to distinguish
+  ** between a write-lock and 255 read-locks.  */
+  struct LockingSlot {
+    u32 nLock;
+    u64 aPadding[7];
+  } aMFSlot[3 + SQLITE_MFS_NSHARD*5];
+#endif
 };
 
+/*
+** Atomic CAS primitive used in multi-process mode. Equivalent to:
+** 
+**   int unixCompareAndSwap(u32 *ptr, u32 oldval, u32 newval){
+**     if( *ptr==oldval ){
+**       *ptr = newval;
+**       return 1;
+**     }
+**     return 0;
+**   }
+*/
+#define unixCompareAndSwap(ptr,oldval,newval) \
+    __sync_bool_compare_and_swap(ptr,oldval,newval)
+
+
 /*
 ** Structure used internally by this VFS to record the state of an
 ** open shared memory connection.
@@ -4316,6 +4381,9 @@ struct unixShm {
   u8 id;                     /* Id of this connection within its unixShmNode */
   u16 sharedMask;            /* Mask of shared locks held */
   u16 exclMask;              /* Mask of exclusive locks held */
+#ifdef SQLITE_MUTEXFREE_SHMLOCK
+  u8 aMFCurrent[8];          /* Current slot used for each shared lock */
+#endif
 };
 
 /*
@@ -4863,6 +4931,87 @@ shmpage_out:
   return rc;
 }
 
+#ifdef SQLITE_MUTEXFREE_SHMLOCK
+static int unixMutexFreeShmlock(
+  unixFile *pFd,             /* Database file holding the shared memory */
+  int ofst,                  /* First lock to acquire or release */
+  int n,                     /* Number of locks to acquire or release */
+  int flags                  /* What to do with the lock */
+){
+  struct LockMapEntry {
+    int iFirst;
+    int nSlot;
+  } aMap[9] = {
+    { 0, 1 },
+    { 1, 1 },
+    { 2, 1 },
+    { 3+0*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+1*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+2*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+3*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+4*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD },
+    { 3+5*SQLITE_MFS_NSHARD, 0 },
+  };
+
+  unixShm *p = pFd->pShm;               /* The shared memory being locked */
+  unixShmNode *pShmNode = p->pShmNode;  /* The underlying file iNode */
+
+  if( flags & SQLITE_SHM_SHARED ){
+    /* SHARED locks */
+    u32 iOld, iNew, *ptr;
+    int iIncr = -1;
+    if( (flags & SQLITE_SHM_UNLOCK)==0 ){
+      p->aMFCurrent[ofst] = (p->aMFCurrent[ofst] + 1) % aMap[ofst].nSlot;
+      iIncr = 1;
+    }
+    ptr = &pShmNode->aMFSlot[aMap[ofst].iFirst + p->aMFCurrent[ofst]].nLock;
+    do {
+      iOld = *ptr;
+      iNew = iOld + iIncr;
+      if( iNew>SQLITE_MFS_EXCLUSIVE ){
+        return SQLITE_BUSY;
+      }
+    }while( 0==unixCompareAndSwap(ptr, iOld, iNew) );
+  }else{
+    /* EXCLUSIVE locks */
+    u16 mask = (1<<(ofst+n)) - (1<<ofst);
+    if( (flags & SQLITE_SHM_LOCK) || (mask & p->exclMask) ){
+      int iFirst = aMap[ofst].iFirst;
+      int iLast = aMap[ofst+n].iFirst;
+      int i;
+      for(i=iFirst; i<iLast; i++){
+        u32 *ptr = &pShmNode->aMFSlot[i].nLock;
+        if( flags & SQLITE_SHM_UNLOCK ){
+          assert( (*ptr)==SQLITE_MFS_EXCLUSIVE );
+          *ptr = 0;
+        }else{
+          u32 iOld;
+          do {
+            iOld = *ptr;
+            if( iOld>0 ){
+              while( i>iFirst ){
+                i--;
+                pShmNode->aMFSlot[i].nLock = 0;
+              }
+              return SQLITE_BUSY;
+            }
+          }while( 0==unixCompareAndSwap(ptr, iOld, SQLITE_MFS_EXCLUSIVE) );
+        }
+      }
+      if( flags & SQLITE_SHM_UNLOCK ){
+        p->exclMask &= ~mask;
+      }else{
+        p->exclMask |= mask;
+      }
+    }
+  }
+
+  return SQLITE_OK;
+}
+#else
+# define unixMutexFreeShmlock(a,b,c,d) SQLITE_OK
+#endif
+
 /*
 ** Check that the pShmNode->aLock[] array comports with the locking bitmasks
 ** held by each client. Return true if it does, or false otherwise. This
@@ -4934,6 +5083,11 @@ static int unixShmLock(
   assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 );
   assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 );
 
+  if( pDbFd->pInode->bProcessLock ){
+    return unixMutexFreeShmlock(pDbFd, ofst, n, flags);
+  }
+
+
   /* Check that, if this to be a blocking lock, no locks that occur later
   ** in the following list than the lock being obtained are already held:
   **
@@ -5045,12 +5199,16 @@ static void unixShmBarrier(
   sqlite3_file *fd                /* Database file holding the shared memory */
 ){
   UNUSED_PARAMETER(fd);
+#ifdef SQLITE_MUTEXFREE_SHMLOCK
+  __sync_synchronize();
+#else
   sqlite3MemoryBarrier();         /* compiler-defined memory barrier */
   assert( fd->pMethods->xLock==nolockLock 
        || unixFileMutexNotheld((unixFile*)fd) 
   );
   unixEnterMutex();               /* Also mutex, for redundancy */
   unixLeaveMutex();
+#endif
 }
 
 /*
@@ -5119,6 +5277,9 @@ static int unixShmUnmap(
 */
 static void unixUnmapfile(unixFile *pFd){
   assert( pFd->nFetchOut==0 );
+#ifdef SQLITE_SHARED_MAPPING
+  if( pFd->pInode ) return;
+#endif
   if( pFd->pMapRegion ){
     osMunmap(pFd->pMapRegion, pFd->mmapSizeActual);
     pFd->pMapRegion = 0;
@@ -5250,6 +5411,28 @@ static int unixMapfile(unixFile *pFd, i64 nMap){
     nMap = pFd->mmapSizeMax;
   }
 
+#ifdef SQLITE_SHARED_MAPPING
+  if( pFd->pInode ){
+    unixInodeInfo *pInode = pFd->pInode;
+    if( pFd->pMapRegion ) return SQLITE_OK;
+    unixEnterMutex();
+    if( pInode->pSharedMapping==0 ){
+      u8 *pNew = osMmap(0, nMap, PROT_READ, MAP_SHARED, pFd->h, 0);
+      if( pNew==MAP_FAILED ){
+        unixLogError(SQLITE_OK, "mmap", pFd->zPath);
+        pFd->mmapSizeMax = 0;
+      }else{
+        pInode->pSharedMapping = pNew;
+        pInode->nSharedMapping = nMap;
+      }
+    }
+    pFd->pMapRegion = pInode->pSharedMapping;
+    pFd->mmapSizeActual = pFd->mmapSize = pInode->nSharedMapping;
+    unixLeaveMutex();
+    return SQLITE_OK;
+  }
+#endif
+
   assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) );
   if( nMap!=pFd->mmapSize ){
     unixRemapfile(pFd, nMap);
@@ -5687,6 +5870,9 @@ static int fillInUnixFile(
   if( pLockingStyle == &posixIoMethods
 #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE
     || pLockingStyle == &nfsIoMethods
+#endif
+#ifdef SQLITE_SHARED_MAPPING
+    || pLockingStyle == &nolockIoMethods
 #endif
   ){
     unixEnterMutex();

src/pager.c

@@ -21,6 +21,7 @@
 #ifndef SQLITE_OMIT_DISKIO
 #include "sqliteInt.h"
 #include "wal.h"
+#include "vdbeInt.h"
 
 
 /******************* NOTES ON THE DESIGN OF THE PAGER ************************
@@ -658,6 +659,9 @@ struct Pager {
   u32 cksumInit;              /* Quasi-random value added to every checksum */
   u32 nSubRec;                /* Number of records written to sub-journal */
   Bitvec *pInJournal;         /* One bit for each page in the database file */
+#ifndef SQLITE_OMIT_CONCURRENT
+  Bitvec *pAllRead;           /* Pages read within current CONCURRENT trans. */
+#endif
   sqlite3_file *fd;           /* File descriptor for database */
   sqlite3_file *jfd;          /* File descriptor for main journal */
   sqlite3_file *sjfd;         /* File descriptor for sub-journal */
@@ -700,6 +704,7 @@ struct Pager {
   Wal *pWal;                  /* Write-ahead log used by "journal_mode=wal" */
   char *zWal;                 /* File name for write-ahead log */
 #endif
+  u64 *aOpenTransTm;
 };
 
 /*
@@ -789,20 +794,6 @@ static const unsigned char aJournalMagic[] = {
 # define USEFETCH(x) 0
 #endif
 
-/*
-** The argument to this macro is a file descriptor (type sqlite3_file*).
-** Return 0 if it is not open, or non-zero (but not 1) if it is.
-**
-** This is so that expressions can be written as:
-**
-**   if( isOpen(pPager->jfd) ){ ...
-**
-** instead of
-**
-**   if( pPager->jfd->pMethods ){ ...
-*/
-#define isOpen(pFd) ((pFd)->pMethods!=0)
-
 #ifdef SQLITE_DIRECT_OVERFLOW_READ
 /*
 ** Return true if page pgno can be read directly from the database file
@@ -915,7 +906,9 @@ static int assert_pager_state(Pager *p){
       if( !pagerUseWal(pPager) ){
         assert( p->eLock>=RESERVED_LOCK );
       }
-      assert( pPager->dbSize==pPager->dbOrigSize );
+#ifndef SQLITE_OMIT_CONCURRENT
+      assert( pPager->dbSize==pPager->dbOrigSize || pPager->pAllRead );
+#endif
       assert( pPager->dbOrigSize==pPager->dbFileSize );
       assert( pPager->dbOrigSize==pPager->dbHintSize );
       assert( pPager->setSuper==0 );
@@ -934,6 +927,7 @@ static int assert_pager_state(Pager *p){
         assert( isOpen(p->jfd) 
              || p->journalMode==PAGER_JOURNALMODE_OFF 
              || p->journalMode==PAGER_JOURNALMODE_WAL 
+             || p->journalMode==PAGER_JOURNALMODE_WAL2
         );
       }
       assert( pPager->dbOrigSize==pPager->dbFileSize );
@@ -948,6 +942,7 @@ static int assert_pager_state(Pager *p){
       assert( isOpen(p->jfd) 
            || p->journalMode==PAGER_JOURNALMODE_OFF 
            || p->journalMode==PAGER_JOURNALMODE_WAL 
+           || p->journalMode==PAGER_JOURNALMODE_WAL2
            || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC)
       );
       assert( pPager->dbOrigSize<=pPager->dbHintSize );
@@ -960,6 +955,7 @@ static int assert_pager_state(Pager *p){
       assert( isOpen(p->jfd) 
            || p->journalMode==PAGER_JOURNALMODE_OFF 
            || p->journalMode==PAGER_JOURNALMODE_WAL 
+           || p->journalMode==PAGER_JOURNALMODE_WAL2
            || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC)
       );
       break;
@@ -1800,6 +1796,53 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){
   return rc;
 }
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** If they are not already, begin recording all pages read from the pager layer
+** by the b-tree layer This is used by concurrent transactions. Return
+** SQLITE_OK if successful, or an SQLite error code (SQLITE_NOMEM) if an error
+** occurs.
+*/
+int sqlite3PagerBeginConcurrent(Pager *pPager){
+  int rc = SQLITE_OK;
+  if( pPager->pAllRead==0 ){
+    pPager->pAllRead = sqlite3BitvecCreate(pPager->dbSize);
+    pPager->dbOrigSize = pPager->dbSize;
+    if( pPager->pAllRead==0 ){
+      rc = SQLITE_NOMEM;
+    }
+  }
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Stop recording all pages read from the pager layer by the b-tree layer
+** and discard any current records.
+*/
+void sqlite3PagerEndConcurrent(Pager *pPager){
+  sqlite3BitvecDestroy(pPager->pAllRead);
+  pPager->pAllRead = 0;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Return true if the database is in wal mode. False otherwise.
+*/
+int sqlite3PagerIsWal(Pager *pPager){
+  return pPager->pWal!=0;
+}
+#endif /* SQLITE_OMIT_CONCURRENT */
+
+/*
+** Free the Pager.pInJournal and Pager.pAllRead bitvec objects.
+*/
+static void pagerFreeBitvecs(Pager *pPager){
+  sqlite3BitvecDestroy(pPager->pInJournal);
+  pPager->pInJournal = 0;
+  sqlite3PagerEndConcurrent(pPager);
+}
+
 /*
 ** This function is a no-op if the pager is in exclusive mode and not
 ** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN
@@ -1824,8 +1867,7 @@ static void pager_unlock(Pager *pPager){
        || pPager->eState==PAGER_ERROR 
   );
 
-  sqlite3BitvecDestroy(pPager->pInJournal);
-  pPager->pInJournal = 0;
+  pagerFreeBitvecs(pPager);
   releaseAllSavepoints(pPager);
 
   if( pagerUseWal(pPager) ){
@@ -2058,7 +2100,7 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){
       }
       pPager->journalOff = 0;
     }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST
-      || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL)
+      || (pPager->exclusiveMode && pPager->journalMode<PAGER_JOURNALMODE_WAL)
     ){
       rc = zeroJournalHdr(pPager, hasSuper||pPager->tempFile);
       pPager->journalOff = 0;
@@ -2072,7 +2114,8 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){
       assert( sqlite3JournalIsInMemory(pPager->jfd)==0 );
       assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE 
            || pPager->journalMode==PAGER_JOURNALMODE_MEMORY 
-           || pPager->journalMode==PAGER_JOURNALMODE_WAL 
+           || pPager->journalMode==PAGER_JOURNALMODE_WAL
+           || pPager->journalMode==PAGER_JOURNALMODE_WAL2
       );
       sqlite3OsClose(pPager->jfd);
       if( bDelete ){
@@ -2092,8 +2135,7 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){
   }
 #endif
 
-  sqlite3BitvecDestroy(pPager->pInJournal);
-  pPager->pInJournal = 0;
+  pagerFreeBitvecs(pPager);
   pPager->nRec = 0;
   if( rc==SQLITE_OK ){
     if( MEMDB || pagerFlushOnCommit(pPager, bCommit) ){
@@ -3103,8 +3145,24 @@ static int pagerRollbackWal(Pager *pPager){
   **   + Reload page content from the database (if refcount>0).
   */
   pPager->dbSize = pPager->dbOrigSize;
-  rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager);
+  rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager, 
+#ifdef SQLITE_OMIT_CONCURRENT
+      0
+#else
+      pPager->pAllRead!=0
+#endif
+  );
   pList = sqlite3PcacheDirtyList(pPager->pPCache);
+
+#ifndef SQLITE_OMIT_CONCURRENT
+  /* If this is an CONCURRENT transaction, then page 1 must be reread from 
+  ** the db file, even if it is not dirty. This is because the b-tree layer 
+  ** may have already zeroed the nFree and iTrunk header fields.  */
+  if( rc==SQLITE_OK && (pList==0 || pList->pgno!=1) && pPager->pAllRead ){
+    rc = pagerUndoCallback((void*)pPager, 1);
+  }
+#endif
+
   while( pList && rc==SQLITE_OK ){
     PgHdr *pNext = pList->pDirty;
     rc = pagerUndoCallback((void *)pPager, pList->pgno);
@@ -3154,6 +3212,8 @@ static int pagerWalFrames(
       if( p->pgno<=nTruncate ){
         ppNext = &p->pDirty;
         nList++;
+        PAGERTRACE(("TO-WAL %d page %d hash(%08x)\n",
+                     PAGERID(pPager), p->pgno, pager_pagehash(p)));
       }
     }
     assert( pList );
@@ -3206,8 +3266,21 @@ static int pagerBeginReadTransaction(Pager *pPager){
 
   rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed);
   if( rc!=SQLITE_OK || changed ){
+    if( pPager->aOpenTransTm ){
+      pPager->aOpenTransTm[OPEN_TRANS_BEFORERESET] = sqlite3STimeNow();
+    }
     pager_reset(pPager);
+    if( pPager->aOpenTransTm ){
+      pPager->aOpenTransTm[OPEN_TRANS_AFTERRESET] = sqlite3STimeNow();
+    }
     if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0);
+    if( pPager->aOpenTransTm ){
+      pPager->aOpenTransTm[OPEN_TRANS_AFTERUNFETCH] = sqlite3STimeNow();
+    }
+    assert( pPager->journalMode==PAGER_JOURNALMODE_WAL
+         || pPager->journalMode==PAGER_JOURNALMODE_WAL2
+    );
+    pPager->journalMode = sqlite3WalJournalMode(pPager->pWal);
   }
 
   return rc;
@@ -3303,9 +3376,9 @@ static int pagerOpenWalIfPresent(Pager *pPager){
           rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0);
         }else{
           testcase( sqlite3PcachePagecount(pPager->pPCache)==0 );
-          rc = sqlite3PagerOpenWal(pPager, 0);
+          rc = sqlite3PagerOpenWal(pPager, 0, 0);
         }
-      }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){
+      }else if( pPager->journalMode>=PAGER_JOURNALMODE_WAL ){
         pPager->journalMode = PAGER_JOURNALMODE_DELETE;
       }
     }
@@ -4225,7 +4298,7 @@ static int syncJournal(Pager *pPager, int newHdr){
   assert( assert_pager_state(pPager) );
   assert( !pagerUseWal(pPager) );
 
-  rc = sqlite3PagerExclusiveLock(pPager);
+  rc = sqlite3PagerExclusiveLock(pPager, 0, 0);
   if( rc!=SQLITE_OK ) return rc;
 
   if( !pPager->noSync ){
@@ -4576,6 +4649,12 @@ static int pagerStress(void *p, PgHdr *pPg){
   pPager->aStat[PAGER_STAT_SPILL]++;
   pPg->pDirty = 0;
   if( pagerUseWal(pPager) ){
+#ifndef SQLITE_OMIT_CONCURRENT
+    /* If the transaction is a "BEGIN CONCURRENT" transaction, the page 
+    ** cannot be flushed to disk. Return early in this case. */
+    if( pPager->pAllRead ) return SQLITE_OK;
+#endif
+
     /* Write a single frame for this page to the log. */
     rc = subjournalPageIfRequired(pPg); 
     if( rc==SQLITE_OK ){
@@ -4816,6 +4895,7 @@ int sqlite3PagerOpen(
     nPathname + 8 + 1 +                  /* Journal filename */
 #ifndef SQLITE_OMIT_WAL
     nPathname + 4 + 1 +                  /* WAL filename */
+    nPathname + 5 + 1 +                  /* Second WAL filename */
 #endif
     3                                    /* Terminator */
   );
@@ -4868,6 +4948,8 @@ int sqlite3PagerOpen(
     sqlite3FileSuffix3(zFilename, pPager->zWal);
     pPtr = (u8*)(pPager->zWal + sqlite3Strlen30(pPager->zWal)+1);
 #endif
+    memcpy(pPtr, zPathname, nPathname);   pPtr += nPathname;
+    memcpy(pPtr, "-wal2", 5);             pPtr += 5 + 1;
   }else{
     pPager->zWal = 0;
   }
@@ -5495,6 +5577,18 @@ static int getPageNormal(
   assert( assert_pager_state(pPager) );
   assert( pPager->hasHeldSharedLock==1 );
 
+#ifndef SQLITE_OMIT_CONCURRENT
+  /* If this is an CONCURRENT transaction and the page being read was
+  ** present in the database file when the transaction was opened,
+  ** mark it as read in the pAllRead vector.  */
+  pPg = 0;
+  if( pPager->pAllRead && pgno<=pPager->dbOrigSize ){
+    PAGERTRACE(("USING page %d\n", pgno));
+    rc = sqlite3BitvecSet(pPager->pAllRead, pgno);
+    if( rc!=SQLITE_OK ) goto pager_acquire_err;
+  }
+#endif
+
   if( pgno==0 ) return SQLITE_CORRUPT_BKPT;
   pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3);
   if( pBase==0 ){
@@ -5835,11 +5929,14 @@ static int pager_open_journal(Pager *pPager){
 ** Begin a write-transaction on the specified pager object. If a 
 ** write-transaction has already been opened, this function is a no-op.
 **
-** If the exFlag argument is false, then acquire at least a RESERVED
-** lock on the database file. If exFlag is true, then acquire at least
+** If the exFlag argument is 0, then acquire at least a RESERVED
+** lock on the database file. If exFlag is >0, then acquire at least
 ** an EXCLUSIVE lock. If such a lock is already held, no locking 
 ** functions need be called.
 **
+** If (exFlag<0) and the database is in WAL mode, do not take any locks.
+** The transaction will run in CONCURRENT mode instead.
+**
 ** If the subjInMemory argument is non-zero, then any sub-journal opened
 ** within this transaction will be opened as an in-memory file. This
 ** has no effect if the sub-journal is already opened (as it may be when
@@ -5857,7 +5954,6 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){
 
   if( pPager->eState==PAGER_READER ){
     assert( pPager->pInJournal==0 );
-
     if( pagerUseWal(pPager) ){
       /* If the pager is configured to use locking_mode=exclusive, and an
       ** exclusive lock on the database is not already held, obtain it now.
@@ -5873,9 +5969,10 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){
       /* Grab the write lock on the log file. If successful, upgrade to
       ** PAGER_RESERVED state. Otherwise, return an error code to the caller.
       ** The busy-handler is not invoked if another connection already
-      ** holds the write-lock. If possible, the upper layer will call it.
-      */
-      rc = sqlite3WalBeginWriteTransaction(pPager->pWal);
+      ** holds the write-lock. If possible, the upper layer will call it.  */
+      if( exFlag>=0 ){
+        rc = sqlite3WalBeginWriteTransaction(pPager->pWal);
+      }
     }else{
       /* Obtain a RESERVED lock on the database file. If the exFlag parameter
       ** is true, then immediately upgrade this to an EXCLUSIVE lock. The
@@ -5883,7 +5980,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){
       ** lock, but not when obtaining the RESERVED lock.
       */
       rc = pagerLockDb(pPager, RESERVED_LOCK);
-      if( rc==SQLITE_OK && exFlag ){
+      if( rc==SQLITE_OK && exFlag>0 ){
         rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK);
       }
     }
@@ -6183,7 +6280,7 @@ int sqlite3PagerWrite(PgHdr *pPg){
 ** to sqlite3PagerWrite().  In other words, return TRUE if it is ok
 ** to change the content of the page.
 */
-#ifndef NDEBUG
+#if !defined(SQLITE_OMIT_CONCURRENT) || !defined(NDEBUG)
 int sqlite3PagerIswriteable(DbPage *pPg){
   return pPg->flags & PGHDR_WRITEABLE;
 }
@@ -6339,17 +6436,26 @@ int sqlite3PagerSync(Pager *pPager, const char *zSuper){
 }
 
 /*
-** This function may only be called while a write-transaction is active in
-** rollback. If the connection is in WAL mode, this call is a no-op. 
-** Otherwise, if the connection does not already have an EXCLUSIVE lock on 
-** the database file, an attempt is made to obtain one.
+** This function is called to ensure that all locks required to commit the
+** current write-transaction to the database file are held. If the db is
+** in rollback mode, this means the EXCLUSIVE lock on the database file.
 **
-** If the EXCLUSIVE lock is already held or the attempt to obtain it is
-** successful, or the connection is in WAL mode, SQLITE_OK is returned.
-** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is 
-** returned.
+** Or, if this is a non-CONCURRENT transaction on a wal-mode database, this
+** function is a no-op.
+**
+** If this is an CONCURRENT transaction on a wal-mode database, this function
+** attempts to obtain the WRITER lock on the wal file and also checks to
+** see that the transaction can be safely committed (does not commit with 
+** any other transaction committed since it was opened).
+**
+** If the required locks are already held or successfully obtained and
+** the transaction can be committed, SQLITE_OK is returned. If a required lock
+** cannot be obtained, SQLITE_BUSY is returned. Or, if the current transaction
+** is CONCURRENT and cannot be committed due to a conflict, SQLITE_BUSY_SNAPSHOT
+** is returned. Otherwise, if some other error occurs (IO error, OOM etc.),
+** and SQLite error code is returned.
 */
-int sqlite3PagerExclusiveLock(Pager *pPager){
+int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1, Pgno *piConflict){
   int rc = pPager->errCode;
   assert( assert_pager_state(pPager) );
   if( rc==SQLITE_OK ){
@@ -6361,10 +6467,73 @@ int sqlite3PagerExclusiveLock(Pager *pPager){
     if( 0==pagerUseWal(pPager) ){
       rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK);
     }
+#ifndef SQLITE_OMIT_CONCURRENT
+    else{
+      if( pPager->pAllRead ){
+        /* This is an CONCURRENT transaction. Attempt to lock the wal database
+        ** here. If SQLITE_BUSY (but not SQLITE_BUSY_SNAPSHOT) is returned,
+        ** invoke the busy-handler and try again for as long as it returns
+        ** non-zero.  */
+        do {
+          rc = sqlite3WalLockForCommit(
+              pPager->pWal, pPage1, pPager->pAllRead, piConflict
+          );
+        }while( rc==SQLITE_BUSY 
+             && pPager->xBusyHandler(pPager->pBusyHandlerArg) 
+        );
+      }
+    }
+#endif /* SQLITE_OMIT_CONCURRENT */
   }
   return rc;
 }
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** This function is called as part of committing an CONCURRENT transaction.
+** At this point the wal WRITER lock is held, and all pages in the cache 
+** except for page 1 are compatible with the snapshot at the head of the
+** wal file. 
+**
+** This function updates the in-memory data structures and reloads the
+** contents of page 1 so that the client is operating on the snapshot 
+** at the head of the wal file.
+**
+** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
+*/
+int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage *pPage1){
+  int rc;
+
+  assert( pPager->pWal && pPager->pAllRead );
+  rc = sqlite3WalUpgradeSnapshot(pPager->pWal);
+  if( rc==SQLITE_OK ){
+    rc = readDbPage(pPage1);
+  }
+
+  return rc;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** Set the in-memory cache of the database file size to nSz pages.
+*/
+void sqlite3PagerSetDbsize(Pager *pPager, Pgno nSz){
+  pPager->dbSize = nSz;
+}
+
+/* !defined(SQLITE_OMIT_CONCURRENT)
+**
+** If this is a WAL mode connection and the WRITER lock is currently held,
+** relinquish it.
+*/
+void sqlite3PagerDropExclusiveLock(Pager *pPager){
+  if( pagerUseWal(pPager) ){
+    sqlite3WalEndWriteTransaction(pPager->pWal);
+  }
+}
+#endif  /* SQLITE_OMIT_CONCURRENT */
+
+
 /*
 ** Sync the database file for the pager pPager. zSuper points to the name
 ** of a super-journal file that should be written into the individual
@@ -7285,7 +7454,8 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){
             || eMode==PAGER_JOURNALMODE_OFF       /* 2 */
             || eMode==PAGER_JOURNALMODE_TRUNCATE  /* 3 */
             || eMode==PAGER_JOURNALMODE_MEMORY    /* 4 */
-            || eMode==PAGER_JOURNALMODE_WAL       /* 5 */ );
+            || eMode==PAGER_JOURNALMODE_WAL       /* 5 */
+            || eMode==PAGER_JOURNALMODE_WAL2      /* 6 */ );
 
   /* This routine is only called from the OP_JournalMode opcode, and
   ** the logic there will never allow a temporary file to be changed
@@ -7319,9 +7489,12 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){
     assert( (PAGER_JOURNALMODE_MEMORY & 5)==4 );
     assert( (PAGER_JOURNALMODE_OFF & 5)==0 );
     assert( (PAGER_JOURNALMODE_WAL & 5)==5 );
+    assert( (PAGER_JOURNALMODE_WAL2 & 5)==4 );
 
     assert( isOpen(pPager->fd) || pPager->exclusiveMode );
-    if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0 ){
+    if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0 
+     && eMode!=PAGER_JOURNALMODE_WAL2       /* TODO: fix this if possible */
+    ){
       /* In this case we would like to delete the journal file. If it is
       ** not possible, then that is not a problem. Deleting the journal file
       ** here is an optimization only.
@@ -7494,7 +7667,7 @@ static int pagerExclusiveLock(Pager *pPager){
 ** lock on the database file and use heap-memory to store the wal-index
 ** in. Otherwise, use the normal shared-memory.
 */
-static int pagerOpenWal(Pager *pPager){
+static int pagerOpenWal(Pager *pPager, int bWal2){
   int rc = SQLITE_OK;
 
   assert( pPager->pWal==0 && pPager->tempFile==0 );
@@ -7515,7 +7688,7 @@ static int pagerOpenWal(Pager *pPager){
   if( rc==SQLITE_OK ){
     rc = sqlite3WalOpen(pPager->pVfs,
         pPager->fd, pPager->zWal, pPager->exclusiveMode,
-        pPager->journalSizeLimit, &pPager->pWal
+        pPager->journalSizeLimit, bWal2, &pPager->pWal
     );
   }
   pagerFixMaplimit(pPager);
@@ -7541,6 +7714,7 @@ static int pagerOpenWal(Pager *pPager){
 */
 int sqlite3PagerOpenWal(
   Pager *pPager,                  /* Pager object */
+  int bWal2,                      /* Open in wal2 mode if not already open */
   int *pbOpen                     /* OUT: Set to true if call is a no-op */
 ){
   int rc = SQLITE_OK;             /* Return code */
@@ -7557,9 +7731,9 @@ int sqlite3PagerOpenWal(
     /* Close any rollback journal previously open */
     sqlite3OsClose(pPager->jfd);
 
-    rc = pagerOpenWal(pPager);
+    rc = pagerOpenWal(pPager, bWal2);
     if( rc==SQLITE_OK ){
-      pPager->journalMode = PAGER_JOURNALMODE_WAL;
+      pPager->journalMode = bWal2?PAGER_JOURNALMODE_WAL2:PAGER_JOURNALMODE_WAL;
       pPager->eState = PAGER_OPEN;
     }
   }else{
@@ -7581,7 +7755,9 @@ int sqlite3PagerOpenWal(
 int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){
   int rc = SQLITE_OK;
 
-  assert( pPager->journalMode==PAGER_JOURNALMODE_WAL );
+  assert( pPager->journalMode==PAGER_JOURNALMODE_WAL 
+       || pPager->journalMode==PAGER_JOURNALMODE_WAL2
+  );
 
   /* If the log file is not already open, but does exist in the file-system,
   ** it may need to be checkpointed before the connection can switch to
@@ -7596,7 +7772,7 @@ int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){
       );
     }
     if( rc==SQLITE_OK && logexists ){
-      rc = pagerOpenWal(pPager);
+      rc = pagerOpenWal(pPager, 0);
     }
   }
     
@@ -7719,6 +7895,11 @@ void sqlite3PagerSnapshotUnlock(Pager *pPager){
 }
 
 #endif /* SQLITE_ENABLE_SNAPSHOT */
+
+int sqlite3PagerWalInfo(Pager *pPager, u32 *pnPrior, u32 *pnFrame){
+  return sqlite3WalInfo(pPager->pWal, pnPrior, pnFrame);
+}
+
 #endif /* !SQLITE_OMIT_WAL */
 
 #ifdef SQLITE_ENABLE_ZIPVFS
@@ -7735,4 +7916,8 @@ int sqlite3PagerWalFramesize(Pager *pPager){
 }
 #endif
 
+void sqlite3PagerOpenTransTm(Pager *pPager, u64 *a){
+  pPager->aOpenTransTm = a;
+}
+
 #endif /* SQLITE_OMIT_DISKIO */

src/pager.h

@@ -82,6 +82,23 @@ typedef struct PgHdr DbPage;
 #define PAGER_JOURNALMODE_TRUNCATE    3   /* Commit by truncating journal */
 #define PAGER_JOURNALMODE_MEMORY      4   /* In-memory journal file */
 #define PAGER_JOURNALMODE_WAL         5   /* Use write-ahead logging */
+#define PAGER_JOURNALMODE_WAL2        6   /* Use write-ahead logging mode 2 */
+
+#define isWalMode(x) ((x)==PAGER_JOURNALMODE_WAL || (x)==PAGER_JOURNALMODE_WAL2)
+
+/*
+** The argument to this macro is a file descriptor (type sqlite3_file*).
+** Return 0 if it is not open, or non-zero (but not 1) if it is.
+**
+** This is so that expressions can be written as:
+**
+**   if( isOpen(pPager->jfd) ){ ...
+**
+** instead of
+**
+**   if( pPager->jfd->pMethods ){ ...
+*/
+#define isOpen(pFd) ((pFd)->pMethods!=0)
 
 /*
 ** Flags that make up the mask passed to sqlite3PagerGet().
@@ -163,7 +180,7 @@ void *sqlite3PagerGetExtra(DbPage *);
 void sqlite3PagerPagecount(Pager*, int*);
 int sqlite3PagerBegin(Pager*, int exFlag, int);
 int sqlite3PagerCommitPhaseOne(Pager*,const char *zSuper, int);
-int sqlite3PagerExclusiveLock(Pager*);
+int sqlite3PagerExclusiveLock(Pager*, DbPage *pPage1, Pgno*);
 int sqlite3PagerSync(Pager *pPager, const char *zSuper);
 int sqlite3PagerCommitPhaseTwo(Pager*);
 int sqlite3PagerRollback(Pager*);
@@ -171,11 +188,12 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int n);
 int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint);
 int sqlite3PagerSharedLock(Pager *pPager);
 
+
 #ifndef SQLITE_OMIT_WAL
   int sqlite3PagerCheckpoint(Pager *pPager, sqlite3*, int, int*, int*);
   int sqlite3PagerWalSupported(Pager *pPager);
   int sqlite3PagerWalCallback(Pager *pPager);
-  int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen);
+  int sqlite3PagerOpenWal(Pager *pPager, int, int *pisOpen);
   int sqlite3PagerCloseWal(Pager *pPager, sqlite3*);
 # ifdef SQLITE_ENABLE_SNAPSHOT
   int sqlite3PagerSnapshotGet(Pager*, sqlite3_snapshot **ppSnapshot);
@@ -225,10 +243,26 @@ void sqlite3PagerTruncateImage(Pager*,Pgno);
 
 void sqlite3PagerRekey(DbPage*, Pgno, u16);
 
+#ifndef SQLITE_OMIT_CONCURRENT
+void sqlite3PagerEndConcurrent(Pager*);
+int sqlite3PagerBeginConcurrent(Pager*);
+void sqlite3PagerDropExclusiveLock(Pager*);
+int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage*);
+void sqlite3PagerSetDbsize(Pager *pPager, Pgno);
+int sqlite3PagerIsWal(Pager*);
+#else
+# define sqlite3PagerEndConcurrent(x)
+#endif
+
+#if defined(SQLITE_DEBUG) || !defined(SQLITE_OMIT_CONCURRENT)
+int sqlite3PagerIswriteable(DbPage*);
+#endif
+
+int sqlite3PagerWalInfo(Pager*, u32 *pnPrior, u32 *pnFrame);
+
 /* Functions to support testing and debugging. */
 #if !defined(NDEBUG) || defined(SQLITE_TEST)
   Pgno sqlite3PagerPagenumber(DbPage*);
-  int sqlite3PagerIswriteable(DbPage*);
 #endif
 #ifdef SQLITE_TEST
   int *sqlite3PagerStats(Pager*);
@@ -240,4 +274,6 @@ void sqlite3PagerRekey(DbPage*, Pgno, u16);
 # define enable_simulated_io_errors()
 #endif
 
+void sqlite3PagerOpenTransTm(Pager*, u64*);
+
 #endif /* SQLITE_PAGER_H */

src/parse.y

@@ -105,6 +105,13 @@
 */
 struct TrigEvent { int a; IdList * b; };
 
+/*
+** Generate a syntax error
+*/
+static void parserSyntaxError(Parse *pParse, Token *p){
+  sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", p);
+}
+
 struct FrameBound     { int eType; Expr *pExpr; };
 
 /*
@@ -164,7 +171,16 @@ trans_opt ::= TRANSACTION nm.
 transtype(A) ::= .             {A = TK_DEFERRED;}
 transtype(A) ::= DEFERRED(X).  {A = @X; /*A-overwrites-X*/}
 transtype(A) ::= IMMEDIATE(X). {A = @X; /*A-overwrites-X*/}
-transtype(A) ::= EXCLUSIVE(X). {A = @X; /*A-overwrites-X*/}
+transtype(A) ::= ID(X). {
+   Token *p = &X;
+   if( p->n==9 && sqlite3_strnicmp(p->z,"exclusive",9)==0 ){
+     A = TK_EXCLUSIVE;
+   }else if( p->n==10 && sqlite3_strnicmp(p->z,"concurrent",10)==0 ){
+     A = TK_CONCURRENT;  /*A-overwrites-X*/
+   }else{
+     parserSyntaxError(pParse, p);
+   }
+}
 cmd ::= COMMIT|END(X) trans_opt.   {sqlite3EndTransaction(pParse,@X);}
 cmd ::= ROLLBACK(X) trans_opt.     {sqlite3EndTransaction(pParse,@X);}
 
@@ -296,7 +312,6 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,A,Y);}
 //
 %token_class id  ID|INDEXED.
 
-
 // And "ids" is an identifer-or-string.
 //
 %token_class ids  ID|STRING.
@@ -1119,7 +1134,7 @@ expr(A) ::= VARIABLE(X).     {
     Token t = X; /*A-overwrites-X*/
     assert( t.n>=2 );
     if( pParse->nested==0 ){
-      sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t);
+      parserSyntaxError(pParse, &t);
       A = 0;
     }else{
       A = sqlite3PExpr(pParse, TK_REGISTER, 0, 0);
@@ -1901,6 +1916,7 @@ filter_clause(A) ::= FILTER LP WHERE expr(X) RP.  { A = X; }
   UPLUS           /* Unary plus */
   TRUTH           /* IS TRUE or IS FALSE or IS NOT TRUE or IS NOT FALSE */
   REGISTER        /* Reference to a VDBE register */
+  CONCURRENT      /* BEGIN CONCURRENT */
   VECTOR          /* Vector */
   SELECT_COLUMN   /* Choose a single column from a multi-column SELECT */
   IF_NULL_ROW     /* the if-null-row operator */

src/pcache1.c

@@ -649,6 +649,11 @@ static void pcache1TruncateUnsafe(
 ){
   TESTONLY( int nPage = 0; )  /* To assert pCache->nPage is correct */
   unsigned int h, iStop;
+
+#define TRUNCATE_UNSAFE_TIMEOUT 1500000
+  u64 tmTruncate = 0;
+  unsigned int nTmPage = 0;
+
   assert( sqlite3_mutex_held(pCache->pGroup->mutex) );
   assert( pCache->iMaxKey >= iLimit );
   assert( pCache->nHash > 0 );
@@ -666,17 +671,30 @@ static void pcache1TruncateUnsafe(
     h = pCache->nHash/2;
     iStop = h - 1;
   }
+
+  if( iLimit<=1 ){
+    tmTruncate = sqlite3STimeNow();
+    nTmPage = pCache->nPage;
+  }
+
   for(;;){
     PgHdr1 **pp;
     PgHdr1 *pPage;
     assert( h<pCache->nHash );
     pp = &pCache->apHash[h]; 
+
     while( (pPage = *pp)!=0 ){
       if( pPage->iKey>=iLimit ){
         pCache->nPage--;
         *pp = pPage->pNext;
         if( PAGE_IS_UNPINNED(pPage) ) pcache1PinPage(pPage);
-        pcache1FreePage(pPage);
+        if( pcache1.separateCache ){
+          pPage->pNext = pCache->pFree;
+          pCache->pFree = pPage;
+          (*pCache->pnPurgeable)--;
+        }else{
+          pcache1FreePage(pPage);
+        }
       }else{
         pp = &pPage->pNext;
         TESTONLY( if( nPage>=0 ) nPage++; )
@@ -685,6 +703,17 @@ static void pcache1TruncateUnsafe(
     if( h==iStop ) break;
     h = (h+1) % pCache->nHash;
   }
+
+  if( iLimit<=1 ){
+    tmTruncate = sqlite3STimeNow() - tmTruncate;
+    if( tmTruncate>TRUNCATE_UNSAFE_TIMEOUT ){
+      sqlite3_log(SQLITE_WARNING,
+          "slow pcache.xTruncate(1) (v=7): (%d) nPage: %d -> %d", 
+          (int)tmTruncate, (int)nTmPage, (int)pCache->nPage
+      );
+    }
+  }
+
   assert( nPage<0 || pCache->nPage==(unsigned)nPage );
 }
 
@@ -1174,6 +1203,14 @@ static void pcache1Destroy(sqlite3_pcache *p){
   assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) );
   pcache1EnterMutex(pGroup);
   if( pCache->nPage ) pcache1TruncateUnsafe(pCache, 0);
+  if( pcache1.separateCache ){
+    PgHdr1 *p1 = pCache->pFree;
+    while( p1 ){
+      PgHdr1 *pNext = p1->pNext;
+      if( p1->isBulkLocal==0 ) pcache1Free(p1->page.pBuf);
+      p1 = pNext;
+    }
+  }
   assert( pGroup->nMaxPage >= pCache->nMax );
   pGroup->nMaxPage -= pCache->nMax;
   assert( pGroup->nMinPage >= pCache->nMin );

src/pragma.c

@@ -262,7 +262,7 @@ const char *sqlite3JournalModename(int eMode){
   static char * const azModeName[] = {
     "delete", "persist", "off", "truncate", "memory"
 #ifndef SQLITE_OMIT_WAL
-     , "wal"
+     , "wal", "wal2"
 #endif
   };
   assert( PAGER_JOURNALMODE_DELETE==0 );
@@ -271,6 +271,7 @@ const char *sqlite3JournalModename(int eMode){
   assert( PAGER_JOURNALMODE_TRUNCATE==3 );
   assert( PAGER_JOURNALMODE_MEMORY==4 );
   assert( PAGER_JOURNALMODE_WAL==5 );
+  assert( PAGER_JOURNALMODE_WAL2==6 );
   assert( eMode>=0 && eMode<=ArraySize(azModeName) );
 
   if( eMode==ArraySize(azModeName) ) return 0;

src/pragma.h

@@ -441,6 +441,15 @@ static const PragmaName aPragmaName[] = {
   /* iArg:      */ 0 },
 #endif
 #endif
+#endif
+#if !defined(SQLITE_OMIT_FLAG_PRAGMAS)
+#if defined(SQLITE_ENABLE_NOOP_UPDATE)
+ {/* zName:     */ "noop_update",
+  /* ePragTyp:  */ PragTyp_FLAG,
+  /* ePragFlg:  */ PragFlg_Result0|PragFlg_NoColumns1,
+  /* ColNames:  */ 0, 0,
+  /* iArg:      */ SQLITE_NoopUpdate },
+#endif
 #endif
  {/* zName:     */ "optimize",
   /* ePragTyp:  */ PragTyp_OPTIMIZE,

src/random.c

@@ -129,6 +129,28 @@ void sqlite3_randomness(int N, void *pBuf){
   sqlite3_mutex_leave(mutex);
 }
 
+/*
+** Initialize a fast PRNG.  A Fast PRNG is called "fast" because it does
+** not need a mutex to operate, though it does use a mutex to initialize.
+** The quality of the randomness is not as good as the global PRNG.
+*/
+void sqlite3FastPrngInit(FastPrng *pPrng){
+  sqlite3_randomness(sizeof(*pPrng), pPrng);
+  pPrng->x |= 1;
+}
+
+/*
+** Generate N bytes of pseudo-randomness using a FastPrng
+*/
+void sqlite3FastRandomness(FastPrng *pPrng, int N, void *P){
+  unsigned char *pOut = (unsigned char*)P;
+  while( N-->0 ){
+    pPrng->x = ((pPrng->x)>>1) ^ ((1+~((pPrng->x)&1)) & 0xd0000001);
+    pPrng->y = (pPrng->y)*1103515245 + 12345;
+    *(pOut++) = (pPrng->x ^ pPrng->y) & 0xff;
+  }
+}
+
 #ifndef SQLITE_UNTESTABLE
 /*
 ** For testing purposes, we sometimes want to preserve the state of

src/select.c

@@ -2238,7 +2238,7 @@ int sqlite3ColumnsFromExprList(
         if( zName[j]==':' ) nName = j;
       }
       zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt);
-      if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt);
+      if( cnt>3 ) sqlite3FastRandomness(&db->sPrng, sizeof(cnt), &cnt);
     }
     pCol->zCnName = zName;
     pCol->hName = sqlite3StrIHash(zName);

src/sqlite.h.in

@@ -10284,6 +10284,31 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp(
 */
 SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);
 
+/*
+** CAPI3REF: Wal related information regarding the most recent COMMIT
+** EXPERIMENTAL
+**
+** This function reports on the state of the wal file (if any) for database 
+** zDb, which should be "main", "temp", or the name of the attached database.
+** Its results - the values written to the output parameters - are only
+** defined if the most recent SQL command on the connection was a successful 
+** COMMIT that wrote data to wal-mode database zDb.
+**
+** Assuming the above conditions are met, output parameter (*pnFrame) is set
+** to the total number of frames in the wal file. Parameter (*pnPrior) is
+** set to the number of frames that were present in the wal file before the
+** most recent transaction was committed. So that the number of frames written
+** by the most recent transaction is (*pnFrame)-(*pnPrior).
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. It
+** is not an error if this function is called at a time when the results
+** are undefined.
+*/
+SQLITE_EXPERIMENTAL int sqlite3_wal_info(
+  sqlite3 *db, const char *zDb, 
+  unsigned int *pnPrior, unsigned int *pnFrame
+);
+
 /*
 ** CAPI3REF: Serialize a database
 **

src/sqliteInt.h

@@ -1183,6 +1183,7 @@ typedef struct DbFixer DbFixer;
 typedef struct Schema Schema;
 typedef struct Expr Expr;
 typedef struct ExprList ExprList;
+typedef struct FastPrng FastPrng;
 typedef struct FKey FKey;
 typedef struct FuncDestructor FuncDestructor;
 typedef struct FuncDef FuncDef;
@@ -1306,6 +1307,14 @@ typedef int VList;
 # define SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_SYNCHRONOUS
 #endif
 
+/*
+** State of a simple PRNG used for the per-connection and per-pager
+** pseudo-random number generators.
+*/
+struct FastPrng {
+  unsigned int x, y;
+};
+
 /*
 ** Each database file to be accessed by the system is an instance
 ** of the following structure.  There are normally two of these structures
@@ -1553,6 +1562,7 @@ struct sqlite3 {
   u32 dbOptFlags;               /* Flags to enable/disable optimizations */
   u8 enc;                       /* Text encoding */
   u8 autoCommit;                /* The auto-commit flag. */
+  u8 eConcurrent;               /* CONCURRENT_* value */
   u8 temp_store;                /* 1: file 2: memory 0: default */
   u8 mallocFailed;              /* True if we have seen a malloc failure */
   u8 bBenignMalloc;             /* Do not require OOMs if true */
@@ -1566,6 +1576,7 @@ struct sqlite3 {
   u8 nSqlExec;                  /* Number of pending OP_SqlExec opcodes */
   u8 eOpenState;                /* Current condition of the connection */
   int nextPagesize;             /* Pagesize after VACUUM if >0 */
+  FastPrng sPrng;               /* State of the per-connection PRNG */
   i64 nChange;                  /* Value returned by sqlite3_changes() */
   i64 nTotalChange;             /* Value returned by sqlite3_total_changes() */
   int aLimit[SQLITE_N_LIMIT];   /* Limits */
@@ -1675,6 +1686,13 @@ struct sqlite3 {
 #endif
 };
 
+/*
+** Candidate values for sqlite3.eConcurrent
+*/
+#define CONCURRENT_NONE   0
+#define CONCURRENT_OPEN   1
+#define CONCURRENT_SCHEMA 2
+
 /*
 ** A macro to discover the encoding of a database.
 */
@@ -1735,6 +1753,7 @@ struct sqlite3 {
                                           /*   the count using a callback. */
 #define SQLITE_CorruptRdOnly  HI(0x00002) /* Prohibit writes due to error */
 
+#define SQLITE_NoopUpdate     0x01000000  /* UPDATE operations are no-ops */
 /* Flags used only if debugging */
 #ifdef SQLITE_DEBUG
 #define SQLITE_SqlTrace       HI(0x0100000) /* Debug print SQL as it executes */
@@ -4756,6 +4775,8 @@ Vdbe *sqlite3GetVdbe(Parse*);
 void sqlite3PrngSaveState(void);
 void sqlite3PrngRestoreState(void);
 #endif
+void sqlite3FastPrngInit(FastPrng*);
+void sqlite3FastRandomness(FastPrng*, int N, void *P);
 void sqlite3RollbackAll(sqlite3*,int);
 void sqlite3CodeVerifySchema(Parse*, int);
 void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb);

src/test1.c

@@ -8227,6 +8227,41 @@ static int SQLITE_TCLAPI test_dbconfig_maindbname_icecube(
   }
 }
 
+/*
+** Usage: sqlite3_wal_info DB DBNAME
+*/
+static int SQLITE_TCLAPI test_wal_info(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int rc;
+  sqlite3 *db;
+  char *zName;
+  unsigned int nPrior;
+  unsigned int nFrame;
+
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME");
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+  zName = Tcl_GetString(objv[2]);
+
+  rc = sqlite3_wal_info(db, zName, &nPrior, &nFrame);
+  if( rc!=SQLITE_OK ){
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+    return TCL_ERROR;
+  }else{
+    Tcl_Obj *pNew = Tcl_NewObj();
+    Tcl_ListObjAppendElement(interp, pNew, Tcl_NewWideIntObj((i64)nPrior));
+    Tcl_ListObjAppendElement(interp, pNew, Tcl_NewWideIntObj((i64)nFrame));
+    Tcl_SetObjResult(interp, pNew);
+  }
+  return TCL_OK;
+}
+
 /*
 ** Usage: sqlite3_mmap_warm DB DBNAME
 */
@@ -8790,8 +8825,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_snapshot_open_blob", test_snapshot_open_blob, 0 },
      { "sqlite3_snapshot_cmp_blob", test_snapshot_cmp_blob, 0 },
 #endif
-     { "sqlite3_delete_database", test_delete_database,    0 },
-     { "atomic_batch_write",      test_atomic_batch_write, 0 },
+     { "sqlite3_delete_database", test_delete_database, 0 },
+     { "sqlite3_wal_info", test_wal_info, 0 },
+     { "atomic_batch_write",      test_atomic_batch_write,     0   },
      { "sqlite3_mmap_warm",       test_mmap_warm,          0 },
      { "sqlite3_config_sorterref", test_config_sorterref,   0 },
      { "sqlite3_autovacuum_pages", test_autovacuum_pages,   0 },

src/test_config.c

@@ -681,6 +681,12 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
   Tcl_SetVar2(interp, "sqlite_options", "truncate_opt", "1", TCL_GLOBAL_ONLY);
 #endif
 
+#ifndef SQLITE_OMIT_CONCURRENT
+  Tcl_SetVar2(interp, "sqlite_options", "concurrent", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "concurrent", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_OMIT_UTF16
   Tcl_SetVar2(interp, "sqlite_options", "utf16", "0", TCL_GLOBAL_ONLY);
 #else

src/test_hexio.c

@@ -190,7 +190,7 @@ static int SQLITE_TCLAPI hexio_write(
 }
 
 /*
-** USAGE:   hexio_get_int   HEXDATA
+** USAGE:   hexio_get_int [-littleendian] HEXDATA
 **
 ** Interpret the HEXDATA argument as a big-endian integer.  Return
 ** the value of that integer.  HEXDATA can contain between 2 and 8
@@ -207,12 +207,20 @@ static int SQLITE_TCLAPI hexio_get_int(
   const unsigned char *zIn;
   unsigned char *aOut;
   unsigned char aNum[4];
+  int bLittle = 0;
 
-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "HEXDATA");
+  if( objc==3 ){
+    int n;
+    char *z = Tcl_GetStringFromObj(objv[1], &n);
+    if( n>=2 && n<=13 && memcmp(z, "-littleendian", n)==0 ){
+      bLittle = 1;
+    }
+  }
+  if( (objc-bLittle)!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "[-littleendian] HEXDATA");
     return TCL_ERROR;
   }
-  zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1], &nIn);
+  zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1+bLittle], &nIn);
   aOut = sqlite3_malloc( 1 + nIn/2 );
   if( aOut==0 ){
     return TCL_ERROR;
@@ -225,7 +233,11 @@ static int SQLITE_TCLAPI hexio_get_int(
     memcpy(&aNum[4-nOut], aOut, nOut);
   }
   sqlite3_free(aOut);
-  val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3];
+  if( bLittle ){
+    val = (aNum[3]<<24) | (aNum[2]<<16) | (aNum[1]<<8) | aNum[0];
+  }else{
+    val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3];
+  }
   Tcl_SetObjResult(interp, Tcl_NewIntObj(val));
   return TCL_OK;
 }

src/test_superlock.c

@@ -41,6 +41,8 @@ typedef struct SuperlockBusy SuperlockBusy;
 struct Superlock {
   sqlite3 *db;                    /* Database handle used to lock db */
   int bWal;                       /* True if db is a WAL database */
+  int bRecoveryLocked;            /* True if WAL RECOVERY lock is held */
+  int bReaderLocked;              /* True if WAL READER locks are held */
 };
 typedef struct Superlock Superlock;
 
@@ -107,12 +109,13 @@ static int superlockShmLock(
 ** Invoke the supplied busy-handler as required.
 */
 static int superlockWalLock(
-  sqlite3 *db,                    /* Database handle open on WAL database */
+  Superlock *pLock,               /* Superlock handle */
   SuperlockBusy *pBusy            /* Busy handler wrapper object */
 ){
   int rc;                         /* Return code */
   sqlite3_file *fd = 0;           /* Main database file handle */
   void volatile *p = 0;           /* Pointer to first page of shared memory */
+  sqlite3 *db = pLock->db;
 
   /* Obtain a pointer to the sqlite3_file object open on the main db file. */
   rc = sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, (void *)&fd);
@@ -121,8 +124,10 @@ static int superlockWalLock(
   /* Obtain the "recovery" lock. Normally, this lock is only obtained by
   ** clients running database recovery.  
   */
+  assert( pLock->bRecoveryLocked==0 );
   rc = superlockShmLock(fd, 2, 1, pBusy);
   if( rc!=SQLITE_OK ) return rc;
+  pLock->bRecoveryLocked = 1;
 
   /* Zero the start of the first shared-memory page. This means that any
   ** clients that open read or write transactions from this point on will
@@ -139,7 +144,9 @@ static int superlockWalLock(
   ** are held, it is guaranteed that there are no active reader, writer or 
   ** checkpointer clients.
   */
+  assert( pLock->bReaderLocked==0 );
   rc = superlockShmLock(fd, 3, SQLITE_SHM_NLOCK-3, pBusy);
+  if( rc==SQLITE_OK ) pLock->bReaderLocked = 1;
   return rc;
 }
 
@@ -156,8 +163,14 @@ void sqlite3demo_superunlock(void *pLock){
     sqlite3_file *fd = 0;
     rc = sqlite3_file_control(p->db, "main", SQLITE_FCNTL_FILE_POINTER, (void *)&fd);
     if( rc==SQLITE_OK ){
-      fd->pMethods->xShmLock(fd, 2, 1, flags);
-      fd->pMethods->xShmLock(fd, 3, SQLITE_SHM_NLOCK-3, flags);
+      if( p->bRecoveryLocked ){
+        fd->pMethods->xShmLock(fd, 2, 1, flags);
+        p->bRecoveryLocked = 0;
+      }
+      if( p->bReaderLocked ){
+        fd->pMethods->xShmLock(fd, 3, SQLITE_SHM_NLOCK-3, flags);
+        p->bReaderLocked = 0;
+      }
     }
   }
   sqlite3_close(p->db);
@@ -232,7 +245,7 @@ int sqlite3demo_superlock(
     if( SQLITE_OK==(rc = superlockIsWal(pLock)) && pLock->bWal ){
       rc = sqlite3_exec(pLock->db, "COMMIT", 0, 0, 0);
       if( rc==SQLITE_OK ){
-        rc = superlockWalLock(pLock->db, &busy);
+        rc = superlockWalLock(pLock, &busy);
       }
     }
   }

src/test_tclsh.c

@@ -99,6 +99,7 @@ const char *sqlite3TestInit(Tcl_Interp *interp){
   extern int Fts5tcl_Init(Tcl_Interp *);
   extern int SqliteRbu_Init(Tcl_Interp*);
   extern int Sqlitetesttcl_Init(Tcl_Interp*);
+  extern int Bgckpt_Init(Tcl_Interp*);
 #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
   extern int Sqlitetestfts3_Init(Tcl_Interp *interp);
 #endif
@@ -168,6 +169,8 @@ const char *sqlite3TestInit(Tcl_Interp *interp){
   Fts5tcl_Init(interp);
   SqliteRbu_Init(interp);
   Sqlitetesttcl_Init(interp);
+  Bgckpt_Init(interp);
+
 
 #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
   Sqlitetestfts3_Init(interp);

src/update.c

@@ -461,6 +461,17 @@ void sqlite3Update(
   */
   chngRowid = chngPk = 0;
   for(i=0; i<pChanges->nExpr; i++){
+#if defined(SQLITE_ENABLE_NOOP_UPDATE) && !defined(SQLITE_OMIT_FLAG_PRAGMAS)
+    if( db->flags & SQLITE_NoopUpdate ){
+      Token x;
+      sqlite3ExprDelete(db, pChanges->a[i].pExpr);
+      x.z = pChanges->a[i].zEName;
+      x.n = sqlite3Strlen30(x.z);
+      pChanges->a[i].pExpr =
+         sqlite3PExpr(pParse, TK_UPLUS, sqlite3ExprAlloc(db, TK_ID, &x, 0), 0);
+      if( db->mallocFailed ) goto update_cleanup;
+    }
+#endif
     u8 hCol = sqlite3StrIHash(pChanges->a[i].zEName);
     /* If this is an UPDATE with a FROM clause, do not resolve expressions
     ** here. The call to sqlite3Select() below will do that. */

src/vacuum.c

@@ -390,6 +390,7 @@ end_of_vacuum:
   ** is closed by the DETACH.
   */
   db->autoCommit = 1;
+  assert( db->eConcurrent==0 );
 
   if( pDb ){
     sqlite3BtreeClose(pDb->pBt);

src/vdbe.c

@@ -3620,6 +3620,7 @@ case OP_Savepoint: {
       ** is committed. 
       */
       int isTransaction = pSavepoint->pNext==0 && db->isTransactionSavepoint;
+      assert( db->eConcurrent==0 || db->isTransactionSavepoint==0 );
       if( isTransaction && p1==SAVEPOINT_RELEASE ){
         if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){
           goto vdbe_return;
@@ -3706,23 +3707,31 @@ case OP_Savepoint: {
   break;
 }
 
-/* Opcode: AutoCommit P1 P2 * * *
+/* Opcode: AutoCommit P1 P2 P3 * *
 **
 ** Set the database auto-commit flag to P1 (1 or 0). If P2 is true, roll
 ** back any currently active btree transactions. If there are any active
 ** VMs (apart from this one), then a ROLLBACK fails.  A COMMIT fails if
 ** there are active writing VMs or active VMs that use shared cache.
 **
+** If P3 is non-zero, then this instruction is being executed as part of
+** a "BEGIN CONCURRENT" command.
+**
 ** This instruction causes the VM to halt.
 */
 case OP_AutoCommit: {
   int desiredAutoCommit;
   int iRollback;
+  int bConcurrent;
+  int hrc;
 
   desiredAutoCommit = pOp->p1;
   iRollback = pOp->p2;
+  bConcurrent = pOp->p3;
   assert( desiredAutoCommit==1 || desiredAutoCommit==0 );
   assert( desiredAutoCommit==1 || iRollback==0 );
+  assert( desiredAutoCommit==0 || bConcurrent==0 );
+  assert( db->autoCommit==0 || db->eConcurrent==CONCURRENT_NONE );
   assert( db->nVdbeActive>0 );  /* At least this one VM is active */
   assert( p->bIsReader );
 
@@ -3731,10 +3740,17 @@ case OP_AutoCommit: {
       assert( desiredAutoCommit==1 );
       sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK);
       db->autoCommit = 1;
-    }else if( desiredAutoCommit && db->nVdbeWrite>0 ){
-      /* If this instruction implements a COMMIT and other VMs are writing
-      ** return an error indicating that the other VMs must complete first. 
-      */
+      db->eConcurrent = CONCURRENT_NONE;
+    }else if( desiredAutoCommit
+            && (db->nVdbeWrite>0 || (db->eConcurrent && db->nVdbeActive>1)) ){
+      /* A transaction may only be committed if there are no other active
+      ** writer VMs. If the transaction is CONCURRENT, then it may only be
+      ** committed if there are no active VMs at all (readers or writers).
+      **
+      ** If this instruction is a COMMIT and the transaction may not be
+      ** committed due to one of the conditions above, return an error
+      ** indicating that other VMs must complete before the COMMIT can 
+      ** be processed.  */
       sqlite3VdbeError(p, "cannot commit transaction - "
                           "SQL statements in progress");
       rc = SQLITE_BUSY;
@@ -3744,12 +3760,16 @@ case OP_AutoCommit: {
     }else{
       db->autoCommit = (u8)desiredAutoCommit;
     }
-    if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){
+    hrc = sqlite3VdbeHalt(p);
+    if( (hrc & 0xFF)==SQLITE_BUSY ){
       p->pc = (int)(pOp - aOp);
       db->autoCommit = (u8)(1-desiredAutoCommit);
-      p->rc = rc = SQLITE_BUSY;
+      p->rc = hrc;
+      rc = SQLITE_BUSY;
       goto vdbe_return;
     }
+    assert( bConcurrent==CONCURRENT_NONE || bConcurrent==CONCURRENT_OPEN );
+    db->eConcurrent = (u8)bConcurrent;
     sqlite3CloseSavepoints(db);
     if( p->rc==SQLITE_OK ){
       rc = SQLITE_DONE;
@@ -3830,7 +3850,16 @@ case OP_Transaction: {
   pBt = pDb->pBt;
 
   if( pBt ){
+    u64 aOpenTransTm[OPEN_TRANS_NTIME];
+    memset(aOpenTransTm, 0, sizeof(aOpenTransTm));
+    sqlite3BtreeOpenTransTm(pBt, aOpenTransTm);
     rc = sqlite3BtreeBeginTrans(pBt, pOp->p2, &iMeta);
+    sqlite3BtreeOpenTransTm(pBt, 0);
+    if( aOpenTransTm[OPEN_TRANS_START] ){
+      aOpenTransTm[OPEN_TRANS_DONE] = sqlite3STimeNow();
+    }
+    sqlite3OpenTransLog(aOpenTransTm);
+
     testcase( rc==SQLITE_BUSY_SNAPSHOT );
     testcase( rc==SQLITE_BUSY_RECOVERY );
     if( rc!=SQLITE_OK ){
@@ -3962,6 +3991,17 @@ case OP_SetCookie: {
   pDb = &db->aDb[pOp->p1];
   assert( pDb->pBt!=0 );
   assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) );
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( db->eConcurrent 
+   && (pOp->p2==BTREE_USER_VERSION || pOp->p2==BTREE_APPLICATION_ID)
+  ){
+    rc = SQLITE_ERROR;
+    sqlite3VdbeError(p, "cannot modify %s within CONCURRENT transaction",
+        pOp->p2==BTREE_USER_VERSION ? "user_version" : "application_id"
+    );
+    goto abort_due_to_error;
+  }
+#endif
   /* See note about index shifting on OP_ReadCookie */
   rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, pOp->p3);
   if( pOp->p2==BTREE_SCHEMA_VERSION ){
@@ -4111,6 +4151,11 @@ case OP_OpenWrite:
   pX = pDb->pBt;
   assert( pX!=0 );
   if( pOp->opcode==OP_OpenWrite ){
+#ifndef SQLITE_OMIT_CONCURRENT
+    if( db->eConcurrent==CONCURRENT_OPEN && p2==1 && iDb!=1 ){
+      db->eConcurrent = CONCURRENT_SCHEMA;
+    }
+#endif
     assert( OPFLAG_FORDELETE==BTREE_FORDELETE );
     wrFlag = BTREE_WRCSR | (pOp->p5 & OPFLAG_FORDELETE);
     assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
@@ -7558,6 +7603,7 @@ case OP_JournalMode: {    /* out2 */
        || eNew==PAGER_JOURNALMODE_OFF
        || eNew==PAGER_JOURNALMODE_MEMORY
        || eNew==PAGER_JOURNALMODE_WAL
+       || eNew==PAGER_JOURNALMODE_WAL2
        || eNew==PAGER_JOURNALMODE_QUERY
   );
   assert( pOp->p1>=0 && pOp->p1<db->nDb );
@@ -7576,16 +7622,25 @@ case OP_JournalMode: {    /* out2 */
   /* Do not allow a transition to journal_mode=WAL for a database
   ** in temporary storage or if the VFS does not support shared memory 
   */
-  if( eNew==PAGER_JOURNALMODE_WAL
+  if( isWalMode(eNew)
    && (sqlite3Strlen30(zFilename)==0           /* Temp file */
        || !sqlite3PagerWalSupported(pPager))   /* No shared-memory support */
   ){
     eNew = eOld;
   }
 
-  if( (eNew!=eOld)
-   && (eOld==PAGER_JOURNALMODE_WAL || eNew==PAGER_JOURNALMODE_WAL)
-  ){
+  if( eNew!=eOld && (isWalMode(eNew) || isWalMode(eOld)) ){
+
+    /* Prevent changing directly to wal2 from wal mode. And vice versa. */
+    if( isWalMode(eNew) && isWalMode(eOld) ){
+      rc = SQLITE_ERROR;
+      sqlite3VdbeError(p, "cannot change from %s to %s mode",
+          sqlite3JournalModename(eOld), sqlite3JournalModename(eNew)
+      );
+      goto abort_due_to_error;
+    }
+
+    /* Prevent switching into or out of wal/wal2 mode mid-transaction */
     if( !db->autoCommit || db->nVdbeRead>1 ){
       rc = SQLITE_ERROR;
       sqlite3VdbeError(p,
@@ -7593,31 +7648,33 @@ case OP_JournalMode: {    /* out2 */
           (eNew==PAGER_JOURNALMODE_WAL ? "into" : "out of")
       );
       goto abort_due_to_error;
-    }else{
+    }
  
-      if( eOld==PAGER_JOURNALMODE_WAL ){
-        /* If leaving WAL mode, close the log file. If successful, the call
-        ** to PagerCloseWal() checkpoints and deletes the write-ahead-log 
-        ** file. An EXCLUSIVE lock may still be held on the database file 
-        ** after a successful return. 
-        */
-        rc = sqlite3PagerCloseWal(pPager, db);
-        if( rc==SQLITE_OK ){
-          sqlite3PagerSetJournalMode(pPager, eNew);
-        }
-      }else if( eOld==PAGER_JOURNALMODE_MEMORY ){
-        /* Cannot transition directly from MEMORY to WAL.  Use mode OFF
-        ** as an intermediate */
-        sqlite3PagerSetJournalMode(pPager, PAGER_JOURNALMODE_OFF);
-      }
-  
-      /* Open a transaction on the database file. Regardless of the journal
-      ** mode, this transaction always uses a rollback journal.
+    if( isWalMode(eOld) ){
+      /* If leaving WAL mode, close the log file. If successful, the call
+      ** to PagerCloseWal() checkpoints and deletes the write-ahead-log 
+      ** file. An EXCLUSIVE lock may still be held on the database file 
+      ** after a successful return. 
       */
-      assert( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_WRITE );
+      rc = sqlite3PagerCloseWal(pPager, db);
       if( rc==SQLITE_OK ){
-        rc = sqlite3BtreeSetVersion(pBt, (eNew==PAGER_JOURNALMODE_WAL ? 2 : 1));
+        sqlite3PagerSetJournalMode(pPager, eNew);
       }
+    }else if( eOld==PAGER_JOURNALMODE_MEMORY ){
+      /* Cannot transition directly from MEMORY to WAL.  Use mode OFF
+      ** as an intermediate */
+      sqlite3PagerSetJournalMode(pPager, PAGER_JOURNALMODE_OFF);
+    }
+
+    /* Open a transaction on the database file. Regardless of the journal
+    ** mode, this transaction always uses a rollback journal.
+    */
+    assert( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_WRITE );
+    if( rc==SQLITE_OK ){
+      /* 1==rollback, 2==wal, 3==wal2 */
+      rc = sqlite3BtreeSetVersion(pBt, 
+          1 + isWalMode(eNew) + (eNew==PAGER_JOURNALMODE_WAL2)
+      );
     }
   }
 #endif /* ifndef SQLITE_OMIT_WAL */
@@ -7753,6 +7810,11 @@ case OP_CursorUnlock: {
 */
 case OP_TableLock: {
   u8 isWriteLock = (u8)pOp->p3;
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( isWriteLock && db->eConcurrent && pOp->p2==1 && pOp->p1!=1 ){
+    db->eConcurrent = CONCURRENT_SCHEMA;
+  }
+#endif
   if( isWriteLock || 0==(db->flags&SQLITE_ReadUncommit) ){
     int p1 = pOp->p1; 
     assert( p1>=0 && p1<db->nDb );

src/vdbe.h

@@ -392,4 +392,7 @@ void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*);
 void sqlite3VdbePrintOp(FILE*, int, VdbeOp*);
 #endif
 
+void sqlite3OpenTransLog(u64*);
+u64 sqlite3STimeNow();
+
 #endif /* SQLITE_VDBE_H */

src/vdbeInt.h

@@ -489,6 +489,17 @@ struct Vdbe {
 #endif
 };
 
+#define OPEN_TRANS_START           0
+#define OPEN_TRANS_BEFORERESET     1
+#define OPEN_TRANS_AFTERRESET      2
+#define OPEN_TRANS_AFTERUNFETCH    3     /* After pager_reset() */
+#define OPEN_TRANS_DONE            4
+
+#define OPEN_TRANS_NTIME 5
+
+/* Call sqlite3_log() if opening a read transaction is slower than this */
+#define OPEN_TRANS_TIMEOUT 2000000
+
 /*
 ** The following are allowed values for Vdbe.eVdbeState
 */

src/vdbeaux.c

@@ -2800,10 +2800,27 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
         assert( i!=1 );
         nTrans++;
       }
-      rc = sqlite3PagerExclusiveLock(pPager);
+      rc = sqlite3BtreeExclusiveLock(pBt);
       sqlite3BtreeLeave(pBt);
     }
   }
+
+#ifndef SQLITE_OMIT_CONCURRENT
+  if( db->eConcurrent && (rc & 0xFF)==SQLITE_BUSY ){
+    /* An SQLITE_BUSY or SQLITE_BUSY_SNAPSHOT was encountered while 
+    ** attempting to take the WRITER lock on a wal file. Release the
+    ** WRITER locks on all wal files and return early.  */
+    for(i=0; i<db->nDb; i++){
+      Btree *pBt = db->aDb[i].pBt;
+      if( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE ){
+        sqlite3BtreeEnter(pBt);
+        sqlite3PagerDropExclusiveLock(sqlite3BtreePager(pBt));
+        sqlite3BtreeLeave(pBt);
+      }
+    }
+  }
+#endif
+
   if( rc!=SQLITE_OK ){
     return rc;
   }
@@ -3205,6 +3222,7 @@ int sqlite3VdbeHalt(Vdbe *p){
           sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK);
           sqlite3CloseSavepoints(db);
           db->autoCommit = 1;
+          db->eConcurrent = CONCURRENT_NONE;
           p->nChange = 0;
         }
       }
@@ -3243,9 +3261,9 @@ int sqlite3VdbeHalt(Vdbe *p){
           ** is required. */
           rc = vdbeCommit(db, p);
         }
-        if( rc==SQLITE_BUSY && p->readOnly ){
+        if( (rc & 0xFF)==SQLITE_BUSY && p->readOnly ){
           sqlite3VdbeLeave(p);
-          return SQLITE_BUSY;
+          return rc;
         }else if( rc!=SQLITE_OK ){
           p->rc = rc;
           sqlite3RollbackAll(db, SQLITE_OK);
@@ -3270,6 +3288,7 @@ int sqlite3VdbeHalt(Vdbe *p){
         sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK);
         sqlite3CloseSavepoints(db);
         db->autoCommit = 1;
+        db->eConcurrent = CONCURRENT_NONE;
         p->nChange = 0;
       }
     }
@@ -3291,6 +3310,7 @@ int sqlite3VdbeHalt(Vdbe *p){
         sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK);
         sqlite3CloseSavepoints(db);
         db->autoCommit = 1;
+        db->eConcurrent = CONCURRENT_NONE;
         p->nChange = 0;
       }
     }
@@ -5196,6 +5216,31 @@ int sqlite3NotPureFunc(sqlite3_context *pCtx){
   return 1;
 }
 
+#include <sys/time.h>
+void sqlite3OpenTransLog(u64 *aOpenTransTm){
+  u64 i1 = aOpenTransTm[OPEN_TRANS_START];
+  if( aOpenTransTm[OPEN_TRANS_DONE]>(i1+OPEN_TRANS_TIMEOUT) ){
+    sqlite3_log(SQLITE_WARNING, 
+        "slow open transaction (v=7): (%d, %d, %d, %d)",
+        (aOpenTransTm[OPEN_TRANS_BEFORERESET]==0) ? 0 :
+            (int)(aOpenTransTm[OPEN_TRANS_BEFORERESET] - i1),
+
+        (aOpenTransTm[OPEN_TRANS_AFTERRESET]==0) ? 0 :
+            (int)(aOpenTransTm[OPEN_TRANS_AFTERRESET] - i1),
+
+        (aOpenTransTm[OPEN_TRANS_AFTERUNFETCH]==0) ? 0 :
+            (int)(aOpenTransTm[OPEN_TRANS_AFTERUNFETCH] - i1),
+
+        (int)(aOpenTransTm[OPEN_TRANS_DONE] - i1)
+    );
+  }
+}
+u64 sqlite3STimeNow(){
+  struct timeval time;
+  gettimeofday(&time, 0);
+  return ((u64)time.tv_sec * 1000000 + (u64)time.tv_usec);
+}
+
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 /*
 ** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored

src/wal.h

@@ -26,7 +26,7 @@
 #define CKPT_SYNC_FLAGS(X)  (((X)>>2)&0x03)
 
 #ifdef SQLITE_OMIT_WAL
-# define sqlite3WalOpen(x,y,z)                   0
+# define sqlite3WalOpen(w,x,y,z)                 0
 # define sqlite3WalLimit(x,y)
 # define sqlite3WalClose(v,w,x,y,z)              0
 # define sqlite3WalBeginReadTransaction(y,z)     0
@@ -34,7 +34,7 @@
 # define sqlite3WalDbsize(y)                     0
 # define sqlite3WalBeginWriteTransaction(y)      0
 # define sqlite3WalEndWriteTransaction(x)        0
-# define sqlite3WalUndo(x,y,z)                   0
+# define sqlite3WalUndo(w,x,y,z)                 0
 # define sqlite3WalSavepoint(y,z)
 # define sqlite3WalSavepointUndo(y,z)            0
 # define sqlite3WalFrames(u,v,w,x,y,z)           0
@@ -45,6 +45,7 @@
 # define sqlite3WalFramesize(z)                  0
 # define sqlite3WalFindFrame(x,y,z)              0
 # define sqlite3WalFile(x)                       0
+# define sqlite3WalJournalMode(x)                0
 #else
 
 #define WAL_SAVEPOINT_NDATA 4
@@ -55,7 +56,7 @@
 typedef struct Wal Wal;
 
 /* Open and close a connection to a write-ahead log. */
-int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**);
+int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *,int,i64,int,Wal**);
 int sqlite3WalClose(Wal *pWal, sqlite3*, int sync_flags, int, u8 *);
 
 /* Set the limiting size of a WAL file. */
@@ -83,7 +84,7 @@ int sqlite3WalBeginWriteTransaction(Wal *pWal);
 int sqlite3WalEndWriteTransaction(Wal *pWal);
 
 /* Undo any frames written (but not committed) to the log */
-int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx);
+int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx, int);
 
 /* Return an integer that records the current (uncommitted) write
 ** position in the WAL */
@@ -136,6 +137,15 @@ int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapshot);
 void sqlite3WalSnapshotUnlock(Wal *pWal);
 #endif
 
+#ifndef SQLITE_OMIT_CONCURRENT
+/* Tell the wal layer that we want to commit a concurrent transaction */
+int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPg, Bitvec *pRead, Pgno*);
+
+/* Upgrade the state of the client to take into account changes written
+** by other connections */
+int sqlite3WalUpgradeSnapshot(Wal *pWal);
+#endif /* SQLITE_OMIT_CONCURRENT */
+
 #ifdef SQLITE_ENABLE_ZIPVFS
 /* If the WAL file is not empty, return the number of bytes of content
 ** stored in each frame (i.e. the db page-size when the WAL was created).
@@ -146,10 +156,19 @@ int sqlite3WalFramesize(Wal *pWal);
 /* Return the sqlite3_file object for the WAL file */
 sqlite3_file *sqlite3WalFile(Wal *pWal);
 
+/* Return the journal mode (WAL or WAL2) used by this Wal object. */
+int sqlite3WalJournalMode(Wal *pWal);
+
 #ifdef SQLITE_ENABLE_SETLK_TIMEOUT
 int sqlite3WalWriteLock(Wal *pWal, int bLock);
 void sqlite3WalDb(Wal *pWal, sqlite3 *db);
 #endif
 
+/* sqlite3_wal_info() data */
+int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame);
+
+/* sqlite3_wal_info() data */
+int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame);
+
 #endif /* ifndef SQLITE_OMIT_WAL */
 #endif /* SQLITE_WAL_H */

test/bc_test1.c

@@ -0,0 +1,556 @@
+/*
+** 2016-05-07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+
+#include <sqlite3.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include "tt3_core.c"
+
+#ifdef USE_OSINST
+# include "../src/test_osinst.c"
+#else
+# define vfslog_time() 0
+#endif
+
+typedef struct Config Config;
+typedef struct ThreadCtx ThreadCtx;
+
+#define THREAD_TIME_INSERT   0
+#define THREAD_TIME_COMMIT   1
+#define THREAD_TIME_ROLLBACK 2
+#define THREAD_TIME_WRITER   3
+#define THREAD_TIME_CKPT     4
+
+struct ThreadCtx {
+  Config *pConfig;
+  Sqlite *pDb;
+  Error *pErr;
+  sqlite3_int64 aTime[5];
+};
+
+struct Config {
+  int nIPT;                       /* --inserts-per-transaction */
+  int nThread;                    /* --threads */
+  int nSecond;                    /* --seconds */
+  int bMutex;                     /* --mutex */
+  int nAutoCkpt;                  /* --autockpt */
+  int bRm;                        /* --rm */
+  int bClearCache;                /* --clear-cache */
+  int nMmap;                      /* mmap limit in MB */
+  char *zFile;
+  int bOsinst;                    /* True to use osinst */
+
+  ThreadCtx *aCtx;                /* Array of size nThread */
+
+  pthread_cond_t cond;
+  pthread_mutex_t mutex;
+  int nCondWait;                  /* Number of threads waiting on hCond */
+  sqlite3_vfs *pVfs;
+};
+
+
+typedef struct VfsWrapperFd VfsWrapperFd;
+struct VfsWrapperFd {
+  sqlite3_file base;              /* Base class */
+  int bWriter;                    /* True if holding shm WRITER lock */
+  int iTid;
+  Config *pConfig;
+  sqlite3_file *pFd;              /* Underlying file descriptor */
+};
+
+/* Methods of the wrapper VFS */
+static int vfsWrapOpen(sqlite3_vfs*, const char*, sqlite3_file*, int, int*);
+static int vfsWrapDelete(sqlite3_vfs*, const char*, int);
+static int vfsWrapAccess(sqlite3_vfs*, const char*, int, int*);
+static int vfsWrapFullPathname(sqlite3_vfs*, const char *, int, char*);
+static void *vfsWrapDlOpen(sqlite3_vfs*, const char*);
+static void vfsWrapDlError(sqlite3_vfs*, int, char*);
+static void (*vfsWrapDlSym(sqlite3_vfs*,void*, const char*))(void);
+static void vfsWrapDlClose(sqlite3_vfs*, void*);
+static int vfsWrapRandomness(sqlite3_vfs*, int, char*);
+static int vfsWrapSleep(sqlite3_vfs*, int);
+static int vfsWrapCurrentTime(sqlite3_vfs*, double*);
+static int vfsWrapGetLastError(sqlite3_vfs*, int, char*);
+static int vfsWrapCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
+static int vfsWrapSetSystemCall(sqlite3_vfs*, const char*, sqlite3_syscall_ptr);
+static sqlite3_syscall_ptr vfsWrapGetSystemCall(sqlite3_vfs*, const char*);
+static const char *vfsWrapNextSystemCall(sqlite3_vfs*, const char*);
+
+/* Methods of wrapper sqlite3_io_methods object (see vfsWrapOpen()) */
+static int vfsWrapClose(sqlite3_file*);
+static int vfsWrapRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
+static int vfsWrapWrite(sqlite3_file*, const void*, int iAmt, sqlite3_int64);
+static int vfsWrapTruncate(sqlite3_file*, sqlite3_int64 size);
+static int vfsWrapSync(sqlite3_file*, int flags);
+static int vfsWrapFileSize(sqlite3_file*, sqlite3_int64 *pSize);
+static int vfsWrapLock(sqlite3_file*, int);
+static int vfsWrapUnlock(sqlite3_file*, int);
+static int vfsWrapCheckReservedLock(sqlite3_file*, int *pResOut);
+static int vfsWrapFileControl(sqlite3_file*, int op, void *pArg);
+static int vfsWrapSectorSize(sqlite3_file*);
+static int vfsWrapDeviceCharacteristics(sqlite3_file*);
+static int vfsWrapShmMap(sqlite3_file*, int iPg, int, int, void volatile**);
+static int vfsWrapShmLock(sqlite3_file*, int offset, int n, int flags);
+static void vfsWrapShmBarrier(sqlite3_file*);
+static int vfsWrapShmUnmap(sqlite3_file*, int deleteFlag);
+static int vfsWrapFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **);
+static int vfsWrapUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
+
+static int vfsWrapOpen(
+  sqlite3_vfs *pVfs, 
+  const char *zName, 
+  sqlite3_file *pFd, 
+  int flags, 
+  int *fout
+){
+  static sqlite3_io_methods methods = {
+    3,
+    vfsWrapClose, vfsWrapRead, vfsWrapWrite,
+    vfsWrapTruncate, vfsWrapSync, vfsWrapFileSize,
+    vfsWrapLock, vfsWrapUnlock, vfsWrapCheckReservedLock,
+    vfsWrapFileControl, vfsWrapSectorSize, vfsWrapDeviceCharacteristics,
+    vfsWrapShmMap, vfsWrapShmLock, vfsWrapShmBarrier,
+    vfsWrapShmUnmap, vfsWrapFetch, vfsWrapUnfetch
+  };
+
+  Config *pConfig = (Config*)pVfs->pAppData;
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  int rc;
+
+  memset(pWrapper, 0, sizeof(VfsWrapperFd));
+  if( flags & SQLITE_OPEN_MAIN_DB ){
+    pWrapper->iTid = (int)sqlite3_uri_int64(zName, "tid", 0);
+  }
+
+  pWrapper->pFd = (sqlite3_file*)&pWrapper[1];
+  pWrapper->pConfig = pConfig;
+  rc = pConfig->pVfs->xOpen(pConfig->pVfs, zName, pWrapper->pFd, flags, fout);
+  if( rc==SQLITE_OK ){
+    pWrapper->base.pMethods = &methods;
+  }
+  return rc;
+}
+
+static int vfsWrapDelete(sqlite3_vfs *pVfs, const char *a, int b){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xDelete(pConfig->pVfs, a, b);
+}
+static int vfsWrapAccess(sqlite3_vfs *pVfs, const char *a, int b, int *c){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xAccess(pConfig->pVfs, a, b, c);
+}
+static int vfsWrapFullPathname(sqlite3_vfs *pVfs, const char *a, int b, char*c){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xFullPathname(pConfig->pVfs, a, b, c);
+}
+static void *vfsWrapDlOpen(sqlite3_vfs *pVfs, const char *a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xDlOpen(pConfig->pVfs, a);
+}
+static void vfsWrapDlError(sqlite3_vfs *pVfs, int a, char *b){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xDlError(pConfig->pVfs, a, b);
+}
+static void (*vfsWrapDlSym(sqlite3_vfs *pVfs, void *a, const char *b))(void){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xDlSym(pConfig->pVfs, a, b);
+}
+static void vfsWrapDlClose(sqlite3_vfs *pVfs, void *a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xDlClose(pConfig->pVfs, a);
+}
+static int vfsWrapRandomness(sqlite3_vfs *pVfs, int a, char *b){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xRandomness(pConfig->pVfs, a, b);
+}
+static int vfsWrapSleep(sqlite3_vfs *pVfs, int a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xSleep(pConfig->pVfs, a);
+}
+static int vfsWrapCurrentTime(sqlite3_vfs *pVfs, double *a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xCurrentTime(pConfig->pVfs, a);
+}
+static int vfsWrapGetLastError(sqlite3_vfs *pVfs, int a, char *b){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xGetLastError(pConfig->pVfs, a, b);
+}
+static int vfsWrapCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xCurrentTimeInt64(pConfig->pVfs, a);
+}
+static int vfsWrapSetSystemCall(
+  sqlite3_vfs *pVfs, 
+  const char *a, 
+  sqlite3_syscall_ptr b
+){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xSetSystemCall(pConfig->pVfs, a, b);
+}
+static sqlite3_syscall_ptr vfsWrapGetSystemCall(
+  sqlite3_vfs *pVfs, 
+  const char *a
+){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xGetSystemCall(pConfig->pVfs, a);
+}
+static const char *vfsWrapNextSystemCall(sqlite3_vfs *pVfs, const char *a){
+  Config *pConfig = (Config*)pVfs->pAppData;
+  return pConfig->pVfs->xNextSystemCall(pConfig->pVfs, a);
+}
+
+static int vfsWrapClose(sqlite3_file *pFd){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  pWrapper->pFd->pMethods->xClose(pWrapper->pFd);
+  pWrapper->pFd = 0;
+  return SQLITE_OK;
+}
+static int vfsWrapRead(sqlite3_file *pFd, void *a, int b, sqlite3_int64 c){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xRead(pWrapper->pFd, a, b, c);
+}
+static int vfsWrapWrite(
+  sqlite3_file *pFd, 
+  const void *a, int b, 
+  sqlite3_int64 c
+){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xWrite(pWrapper->pFd, a, b, c);
+}
+static int vfsWrapTruncate(sqlite3_file *pFd, sqlite3_int64 a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xTruncate(pWrapper->pFd, a);
+}
+static int vfsWrapSync(sqlite3_file *pFd, int a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xSync(pWrapper->pFd, a);
+}
+static int vfsWrapFileSize(sqlite3_file *pFd, sqlite3_int64 *a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xFileSize(pWrapper->pFd, a);
+}
+static int vfsWrapLock(sqlite3_file *pFd, int a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xLock(pWrapper->pFd, a);
+}
+static int vfsWrapUnlock(sqlite3_file *pFd, int a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xUnlock(pWrapper->pFd, a);
+}
+static int vfsWrapCheckReservedLock(sqlite3_file *pFd, int *a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xCheckReservedLock(pWrapper->pFd, a);
+}
+static int vfsWrapFileControl(sqlite3_file *pFd, int a, void *b){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xFileControl(pWrapper->pFd, a, b);
+}
+static int vfsWrapSectorSize(sqlite3_file *pFd){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xSectorSize(pWrapper->pFd);
+}
+static int vfsWrapDeviceCharacteristics(sqlite3_file *pFd){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xDeviceCharacteristics(pWrapper->pFd);
+}
+static int vfsWrapShmMap(
+  sqlite3_file *pFd, 
+  int a, int b, int c, 
+  void volatile **d
+){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xShmMap(pWrapper->pFd, a, b, c, d);
+}
+static int vfsWrapShmLock(sqlite3_file *pFd, int offset, int n, int flags){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  Config *pConfig = pWrapper->pConfig;
+  int bMutex = 0;
+  int rc;
+
+  if(  (offset==0 && n==1)
+    && (flags & SQLITE_SHM_LOCK) && (flags & SQLITE_SHM_EXCLUSIVE)
+  ){
+    pthread_mutex_lock(&pConfig->mutex);
+    pWrapper->bWriter = 1;
+    bMutex = 1;
+    if( pWrapper->iTid ){
+      sqlite3_int64 t = vfslog_time();
+      pConfig->aCtx[pWrapper->iTid-1].aTime[THREAD_TIME_WRITER] -= t;
+    }
+  }
+
+  rc = pWrapper->pFd->pMethods->xShmLock(pWrapper->pFd, offset, n, flags);
+
+  if( (rc!=SQLITE_OK && bMutex)
+   || (offset==0 && (flags & SQLITE_SHM_UNLOCK) && pWrapper->bWriter)
+  ){
+    assert( pWrapper->bWriter );
+    pthread_mutex_unlock(&pConfig->mutex);
+    pWrapper->bWriter = 0;
+    if( pWrapper->iTid ){
+      sqlite3_int64 t = vfslog_time();
+      pConfig->aCtx[pWrapper->iTid-1].aTime[THREAD_TIME_WRITER] += t;
+    }
+  }
+
+  return rc;
+}
+static void vfsWrapShmBarrier(sqlite3_file *pFd){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xShmBarrier(pWrapper->pFd);
+}
+static int vfsWrapShmUnmap(sqlite3_file *pFd, int a){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xShmUnmap(pWrapper->pFd, a);
+}
+static int vfsWrapFetch(sqlite3_file *pFd, sqlite3_int64 a, int b, void **c){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xFetch(pWrapper->pFd, a, b, c);
+}
+static int vfsWrapUnfetch(sqlite3_file *pFd, sqlite3_int64 a, void *b){
+  VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd;
+  return pWrapper->pFd->pMethods->xUnfetch(pWrapper->pFd, a, b);
+}
+
+static void create_vfs(Config *pConfig){
+  static sqlite3_vfs vfs = {
+    3, 0, 0, 0, "wrapper", 0,
+    vfsWrapOpen, vfsWrapDelete, vfsWrapAccess,
+    vfsWrapFullPathname, vfsWrapDlOpen, vfsWrapDlError,
+    vfsWrapDlSym, vfsWrapDlClose, vfsWrapRandomness,
+    vfsWrapSleep, vfsWrapCurrentTime, vfsWrapGetLastError,
+    vfsWrapCurrentTimeInt64, vfsWrapSetSystemCall, vfsWrapGetSystemCall,
+    vfsWrapNextSystemCall
+  };
+  sqlite3_vfs *pVfs;
+
+  pVfs = sqlite3_vfs_find(0);
+  vfs.mxPathname = pVfs->mxPathname;
+  vfs.szOsFile = pVfs->szOsFile + sizeof(VfsWrapperFd);
+  vfs.pAppData = (void*)pConfig;
+  pConfig->pVfs = pVfs;
+
+  sqlite3_vfs_register(&vfs, 1);
+}
+
+
+/*
+** Wal hook used by connections in thread_main().
+*/
+static int thread_wal_hook(
+  void *pArg,                     /* Pointer to ThreadCtx object */
+  sqlite3 *db,
+  const char *zDb, 
+  int nFrame
+){
+  ThreadCtx *pCtx = (ThreadCtx*)pArg;
+  Config *pConfig = pCtx->pConfig;
+
+  if( pConfig->nAutoCkpt && nFrame>=pConfig->nAutoCkpt ){
+    pCtx->aTime[THREAD_TIME_CKPT] -= vfslog_time();
+    pthread_mutex_lock(&pConfig->mutex);
+    if( pConfig->nCondWait>=0 ){
+      pConfig->nCondWait++;
+      if( pConfig->nCondWait==pConfig->nThread ){
+        execsql(pCtx->pErr, pCtx->pDb, "PRAGMA wal_checkpoint");
+        pthread_cond_broadcast(&pConfig->cond);
+      }else{
+        pthread_cond_wait(&pConfig->cond, &pConfig->mutex);
+      }
+      pConfig->nCondWait--;
+    }
+    pthread_mutex_unlock(&pConfig->mutex);
+    pCtx->aTime[THREAD_TIME_CKPT] += vfslog_time();
+  }
+
+  return SQLITE_OK;
+}
+
+
+static char *thread_main(int iTid, void *pArg){
+  Config *pConfig = (Config*)pArg;
+  Error err = {0};                /* Error code and message */
+  Sqlite db = {0};                /* SQLite database connection */
+  int nAttempt = 0;               /* Attempted transactions */
+  int nCommit = 0;                /* Successful transactions */
+  int j;
+  ThreadCtx *pCtx = &pConfig->aCtx[iTid-1];
+  char *zUri = 0;
+
+#ifdef USE_OSINST
+  char *zOsinstName = 0;
+  char *zLogName = 0;
+  if( pConfig->bOsinst ){
+    zOsinstName = sqlite3_mprintf("osinst%d", iTid);
+    zLogName = sqlite3_mprintf("bc_test1.log.%d.%d", (int)getpid(), iTid);
+    zUri = sqlite3_mprintf(
+        "file:%s?vfs=%s&tid=%d", pConfig->zFile, zOsinstName, iTid
+    );
+    sqlite3_vfslog_new(zOsinstName, 0, zLogName);
+    opendb(&err, &db, zUri, 0);
+  }else
+#endif
+  {
+    zUri = sqlite3_mprintf("file:%s?tid=%d", pConfig->zFile, iTid);
+    opendb(&err, &db, zUri, 0);
+  }
+
+  sqlite3_busy_handler(db.db, 0, 0);
+  sql_script_printf(&err, &db, 
+      "PRAGMA wal_autocheckpoint = 0;"
+      "PRAGMA synchronous = 0;"
+      "PRAGMA mmap_size = %lld;",
+      (i64)(pConfig->nMmap) * 1024 * 1024
+  );
+
+  pCtx->pConfig = pConfig;
+  pCtx->pErr = &err;
+  pCtx->pDb = &db;
+  sqlite3_wal_hook(db.db, thread_wal_hook, (void*)pCtx);
+
+  while( !timetostop(&err) ){
+    execsql(&err, &db, "BEGIN CONCURRENT");
+
+    pCtx->aTime[THREAD_TIME_INSERT] -= vfslog_time();
+    for(j=0; j<pConfig->nIPT; j++){
+      execsql(&err, &db, 
+          "INSERT INTO t1 VALUES"
+          "(randomblob(10), randomblob(20), randomblob(30), randomblob(200))"
+      );
+    }
+    pCtx->aTime[THREAD_TIME_INSERT] += vfslog_time();
+
+    pCtx->aTime[THREAD_TIME_COMMIT] -= vfslog_time();
+    execsql(&err, &db, "COMMIT");
+    pCtx->aTime[THREAD_TIME_COMMIT] += vfslog_time();
+
+    pCtx->aTime[THREAD_TIME_ROLLBACK] -= vfslog_time();
+    nAttempt++;
+    if( err.rc==SQLITE_OK ){
+      nCommit++;
+    }else{
+      clear_error(&err, SQLITE_BUSY);
+      execsql(&err, &db, "ROLLBACK");
+    }
+    pCtx->aTime[THREAD_TIME_ROLLBACK] += vfslog_time();
+
+    if( pConfig->bClearCache ){
+      sqlite3_db_release_memory(db.db);
+    }
+  }
+
+  closedb(&err, &db);
+
+#ifdef USE_OSINST
+  if( pConfig->bOsinst ){
+    sqlite3_vfslog_finalize(zOsinstName);
+    sqlite3_free(zOsinstName);
+    sqlite3_free(zLogName);
+  }
+#endif
+  sqlite3_free(zUri);
+
+  pthread_mutex_lock(&pConfig->mutex);
+  pConfig->nCondWait = -1;
+  pthread_cond_broadcast(&pConfig->cond);
+  pthread_mutex_unlock(&pConfig->mutex);
+
+  return sqlite3_mprintf("commits: %d/%d insert: %dms"
+      " commit: %dms"
+      " rollback: %dms" 
+      " writer: %dms"
+      " checkpoint: %dms", 
+      nCommit, nAttempt, 
+      (int)(pCtx->aTime[THREAD_TIME_INSERT]/1000), 
+      (int)(pCtx->aTime[THREAD_TIME_COMMIT]/1000), 
+      (int)(pCtx->aTime[THREAD_TIME_ROLLBACK]/1000),
+      (int)(pCtx->aTime[THREAD_TIME_WRITER]/1000),
+      (int)(pCtx->aTime[THREAD_TIME_CKPT]/1000)
+  );
+}
+
+int main(int argc, const char **argv){
+  Error err = {0};                /* Error code and message */
+  Sqlite db = {0};                /* SQLite database connection */
+  Threadset threads = {0};        /* Test threads */
+  Config conf = {5, 3, 5};
+  int i;
+
+  CmdlineArg apArg[] = {
+    { "-seconds", CMDLINE_INT,  offsetof(Config, nSecond) },
+    { "-inserts", CMDLINE_INT,  offsetof(Config, nIPT) },
+    { "-threads", CMDLINE_INT,  offsetof(Config, nThread) },
+    { "-mutex",   CMDLINE_BOOL, offsetof(Config, bMutex) },
+    { "-rm",      CMDLINE_BOOL, offsetof(Config, bRm) },
+    { "-autockpt",CMDLINE_INT,  offsetof(Config, nAutoCkpt) },
+    { "-mmap",    CMDLINE_INT,  offsetof(Config, nMmap) },
+    { "-clear-cache",    CMDLINE_BOOL,  offsetof(Config, bClearCache) },
+    { "-file",    CMDLINE_STRING,  offsetof(Config, zFile) },
+    { "-osinst",  CMDLINE_BOOL,  offsetof(Config, bOsinst) },
+    { 0, 0, 0 }
+  };
+
+  conf.nAutoCkpt = 1000;
+  cmdline_process(apArg, argc, argv, (void*)&conf);
+  if( err.rc==SQLITE_OK ){
+    char *z = cmdline_construct(apArg, (void*)&conf);
+    printf("With: %s\n", z);
+    sqlite3_free(z);
+  }
+  if( conf.zFile==0 ){
+    conf.zFile = "xyz.db";
+  }
+
+  /* Create the special VFS - "wrapper". And the mutex and condition 
+  ** variable. */
+  create_vfs(&conf);
+  pthread_mutex_init(&conf.mutex, 0);
+  pthread_cond_init(&conf.cond, 0);
+
+  conf.aCtx = sqlite3_malloc(sizeof(ThreadCtx) * conf.nThread);
+  memset(conf.aCtx, 0, sizeof(ThreadCtx) * conf.nThread);
+
+  /* Ensure the schema has been created */
+  opendb(&err, &db, conf.zFile, conf.bRm);
+  sql_script(&err, &db,
+      "PRAGMA journal_mode = wal;"
+      "CREATE TABLE IF NOT EXISTS t1(a PRIMARY KEY, b, c, d) WITHOUT ROWID;"
+      "CREATE INDEX IF NOT EXISTS t1b ON t1(b);"
+      "CREATE INDEX IF NOT EXISTS t1c ON t1(c);"
+  );
+
+  setstoptime(&err, conf.nSecond*1000);
+  if( conf.nThread==1 ){
+    char *z = thread_main(1, (void*)&conf);
+    printf("Thread 0 says: %s\n", (z==0 ? "..." : z));
+    fflush(stdout);
+  }else{
+    for(i=0; i<conf.nThread; i++){
+      launch_thread(&err, &threads, thread_main, (void*)&conf);
+    }
+    join_all_threads(&err, &threads);
+  }
+
+  if( err.rc==SQLITE_OK ){
+    printf("Database is %dK\n", (int)(filesize(&err, conf.zFile) / 1024));
+  }
+  if( err.rc==SQLITE_OK ){
+    char *zWal = sqlite3_mprintf("%s-wal", conf.zFile);
+    printf("Wal file is %dK\n", (int)(filesize(&err, zWal) / 1024));
+  }
+
+  closedb(&err, &db);
+  print_and_free_err(&err);
+  return 0;
+}

test/concfault.test

@@ -0,0 +1,86 @@
+# 2015 Aug 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains fault injection tests designed to test the concurrent
+# transactions feature.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set testprefix concfault
+
+# This test will not work with an in-memory journal, as the database will
+# become corrupt if an error is injected into a transaction after it starts
+# writing data out to the db file.
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+do_test 1-pre1 {
+  execsql {
+    PRAGMA journal_mode = wal;
+    CREATE TABLE t1(a PRIMARY KEY, b);
+    INSERT INTO t1 VALUES(randomblob(1000), randomblob(100));
+    INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
+    INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
+    INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
+    INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
+    DELETE FROM t1 WHERE rowid%2;
+  }
+  faultsim_save_and_close
+} {}
+
+do_faultsim_test 1.1 -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { 
+    BEGIN CONCURRENT;
+      INSERT INTO t1 VALUES(randomblob(1000), randomblob(100));
+    COMMIT;
+  }
+} -test {
+  faultsim_test_result {0 {}} 
+  catchsql { ROLLBACK }
+  faultsim_integrity_check
+}
+
+do_faultsim_test 1.2 -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { 
+    BEGIN CONCURRENT;
+      INSERT INTO t1 VALUES(randomblob(1000), randomblob(100));
+    ROLLBACK;
+  }
+} -test {
+  faultsim_test_result {0 {}} 
+  catchsql { ROLLBACK }
+  faultsim_integrity_check
+}
+
+do_faultsim_test 1.3 -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { 
+    BEGIN CONCURRENT;
+      DELETE FROM t1;
+    COMMIT;
+  }
+} -test {
+  faultsim_test_result {0 {}} 
+  catchsql { ROLLBACK }
+  faultsim_integrity_check
+}
+
+finish_test
+

test/concfault2.test

@@ -0,0 +1,69 @@
+# 2018 Dec 28
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains fault injection tests designed to test the concurrent
+# transactions feature.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set testprefix concfault2
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  PRAGMA auto_vacuum = 0;
+  PRAGMA journal_mode = wal2;
+  CREATE TABLE t1(a PRIMARY KEY, b);
+  CREATE TABLE t2(a PRIMARY KEY, b);
+  INSERT INTO t1 VALUES(randomblob(1000), randomblob(100));
+  INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
+  INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
+  INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
+  INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
+  DELETE FROM t1 WHERE rowid%2;
+} {wal2}
+
+do_test 1.1 {
+  list [expr [file size test.db-wal]>75000] [file size test.db-shm]
+} {1 32768}
+
+faultsim_save_and_close
+
+do_faultsim_test 1 -prep {
+  faultsim_restore_and_reopen
+  execsql {
+    SELECT * FROM t1;
+    BEGIN CONCURRENT;
+      INSERT INTO t2 VALUES(1, 2);
+  }
+  sqlite3 db2 test.db
+  execsql {
+    PRAGMA journal_size_limit = 10000;
+    INSERT INTO t1 VALUES(randomblob(1000), randomblob(1000));
+  } db2
+  db2 close
+} -body {
+  execsql { COMMIT }
+} -test {
+  faultsim_test_result {0 {}} 
+  catchsql { ROLLBACK }
+  set res [catchsql { SELECT count(*) FROM t1 }]
+  if {$res!="0 9"} { error "expected {0 9} got {$res}" }
+  faultsim_integrity_check
+}
+
+finish_test
+

test/concurrent.test

@@ -0,0 +1,687 @@
+# 2015 July 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set ::testprefix concurrent
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal;
+} {wal}
+
+do_execsql_test 1.1 {
+  CREATE TABLE t1(k INTEGER PRIMARY KEY, v);
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(1, 'abcd');
+  COMMIT;
+}
+
+do_execsql_test 1.2 {
+  SELECT * FROM t1;
+} {1 abcd}
+
+do_execsql_test 1.3 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(2, 'efgh');
+  ROLLBACK;
+}
+
+do_execsql_test 1.4 {
+  SELECT * FROM t1;
+} {1 abcd}
+
+
+#-------------------------------------------------------------------------
+# CONCURRENT transactions cannot do cache spills.
+#
+foreach {tn trans spill} {
+  1 {BEGIN CONCURRENT}  0
+  2 {BEGIN}           1
+} {
+  do_test 1.5.$tn {
+    sqlite3 db2 test.db
+    set walsz [file size test.db-wal]
+
+    execsql { PRAGMA cache_size = 10 } db2
+    execsql $trans db2
+    execsql {
+      WITH cnt(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM cnt WHERE i<50)
+        INSERT INTO t1(v) SELECT randomblob(900) FROM cnt;
+    } db2
+
+    expr {[file size test.db-wal]==$walsz}
+  } [expr !$spill]
+
+  execsql ROLLBACK db2
+  db2 close
+}
+
+#-------------------------------------------------------------------------
+# CONCURRENT transactions man not be committed while there are active
+# readers.
+do_execsql_test 1.6.setup {
+  DROP TABLE t1;
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 2);
+  INSERT INTO t1 VALUES(3, 4);
+  INSERT INTO t1 VALUES(5, 6);
+}
+foreach {tn trans commit_ok} {
+  1 {BEGIN CONCURRENT}  0
+  2 {BEGIN}           1
+} {
+  do_test 1.6.$tn.1 {
+    set stmt [sqlite3_prepare db "SELECT * FROM t1" -1 dummy]
+    sqlite3_step $stmt
+  } SQLITE_ROW
+  do_test 1.6.$tn.2 {
+    execsql $trans
+    execsql { INSERT INTO t1 VALUES(7, 8) }
+  } {}
+
+  if { $commit_ok } {
+    do_test 1.6.$tn.3 { catchsql COMMIT } {0 {}}
+  } else {
+    do_test 1.6.$tn.4 { catchsql COMMIT } {/1 {cannot commit transaction .*}/}
+  }
+
+  sqlite3_finalize $stmt
+  catchsql ROLLBACK
+}
+
+#-------------------------------------------------------------------------
+# CONCURRENT transactions may not modify the db schema.
+#
+sqlite3 db2 test.db
+foreach {tn sql} {
+  1 { CREATE TABLE xx(a, b) }
+  2 { DROP TABLE t1 }
+  3 { CREATE INDEX i1 ON t1(a) }
+  4 { CREATE VIEW v1 AS SELECT * FROM t1 }
+} {
+  do_catchsql_test 1.7.0.$tn.1 "
+    BEGIN CONCURRENT;
+    $sql
+  " {0 {}}
+
+  db2 eval {INSERT INTO t1 DEFAULT VALUES}
+
+  do_catchsql_test 1.7.0.$tn.2 {
+    COMMIT
+  } {1 {database is locked}}
+
+  do_execsql_test 1.7.0.$tn.2 ROLLBACK
+
+  do_execsql_test 1.7.0.$tn.3 {
+    SELECT sql FROM sqlite_master;
+    SELECT sql FROM sqlite_temp_master;
+  } {{CREATE TABLE t1(a, b)}}
+
+  #do_execsql_test 1.7.0.$tn.3 COMMIT
+}
+
+# Except the temp db schema.
+foreach {tn sql} {
+  1 { CREATE TEMP TABLE xx(a, b) }
+  2 { DROP TABLE xx }
+  3 { CREATE TEMP TABLE yy(a, b) }
+  4 { CREATE VIEW temp.v1 AS SELECT * FROM t1 }
+  5 { CREATE INDEX yyi1 ON yy(a); }
+  6 { CREATE TABLE temp.zz(a, b) }
+} {
+  do_catchsql_test 1.7.1.$tn.1 "
+    BEGIN CONCURRENT;
+    $sql
+  " {0 {}}
+
+  do_execsql_test 1.7.1.$tn.2 COMMIT
+}
+
+
+do_execsql_test 1.7.1.x {
+  SELECT sql FROM sqlite_master;
+  SELECT sql FROM sqlite_temp_master;
+} {
+  {CREATE TABLE t1(a, b)}
+  {CREATE TABLE yy(a, b)} 
+  {CREATE VIEW v1 AS SELECT * FROM t1} 
+  {CREATE INDEX yyi1 ON yy(a)} 
+  {CREATE TABLE zz(a, b)}
+}
+
+#-------------------------------------------------------------------------
+# If an auto-vacuum database is written within an CONCURRENT transaction, it
+# is handled in the same way as for a non-CONCURRENT transaction.
+#
+reset_db
+do_execsql_test 1.8.1 {
+  PRAGMA auto_vacuum = 1;
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x, y);
+  INSERT INTO t1 VALUES('x', 'y');
+} {wal}
+
+do_execsql_test 1.8.2 {
+  BEGIN CONCURRENT;
+    SELECT * FROM t1;
+  COMMIT;
+} {x y}
+
+do_catchsql_test 1.8.3 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES('a', 'b');
+} {0 {}}
+
+do_test 1.8.4 {
+  sqlite3 db2 test.db
+  catchsql {
+    BEGIN CONCURRENT;
+      INSERT INTO t1 VALUES('c', 'd');
+  } db2
+} {1 {database is locked}}
+
+do_test 1.8.5 {
+  db eval COMMIT
+  db2 eval COMMIT
+} {}
+db close
+db2 close
+
+do_multiclient_test tn {
+
+  #-----------------------------------------------------------------------
+  # 1. Start an CONCURRENT transaction using [db1].
+  #
+  # 2. Start and then rollback a regular transaction using [db2]. This 
+  #    can be done as the ongoing [db1] transaction is CONCURRENT.
+  #
+  # 3. The [db1] transaction can now be committed, as [db2] has relinquished
+  #    the write lock.
+  #
+  do_test 2.$tn.1.1 {
+    sql1 { 
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(k INTEGER PRIMARY KEY, v);
+      INSERT INTO t1 VALUES(1, 'one');
+    }
+    sql1 { 
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(2, 'two');
+    }
+    code1 { sqlite3_get_autocommit db }
+  } 0
+
+  do_test 2.$tn.1.2 {
+    sql2 {
+      BEGIN;
+        INSERT INTO t1 VALUES(3, 'three');
+      ROLLBACK;
+    }
+  } {}
+
+  do_test 2.$tn.1.3 {
+    sql1 COMMIT
+    sql2 { SELECT * FROM t1 }
+  } {1 one 2 two}
+  
+  #-----------------------------------------------------------------------
+  # 1. Start an CONCURRENT transaction using [db1].
+  #
+  # 2. Commit a transaction using [db2].
+  #
+  # 3. Try to commit with [db1]. Check that SQLITE_BUSY_SNAPSHOT is returned,
+  #    and the transaction is not rolled back.
+  #
+  do_test 2.$tn.2.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(-1, 'hello world');
+    }
+  } {}
+
+  do_test 2.$tn.2.2 {
+    sql2 {
+      INSERT INTO t1 VALUES(3, 'three');
+    }
+  } {}
+
+  do_test 2.$tn.2.3.1 {
+    set rc [catch { sql1 COMMIT } msg]
+    list $rc $msg
+  } {1 {database is locked}}
+
+  do_test 2.$tn.2.3.2 {
+    code1 { list [sqlite3_extended_errcode db] [sqlite3_get_autocommit db] }
+  } {SQLITE_BUSY_SNAPSHOT 0}
+
+  do_test 2.$tn.2.3.3 {
+    sql1 {
+      SELECT * FROM t1;
+      ROLLBACK;
+    }
+  } {-1 {hello world} 1 one 2 two}
+  
+  #-----------------------------------------------------------------------
+  # 1. Start an CONCURRENT transaction using [db1].
+  #
+  # 2. Open a transaction using [db2].
+  #
+  # 3. Try to commit with [db1]. Check that SQLITE_BUSY is returned,
+  #    and the transaction is not rolled back.
+  #
+  # 4. Have [db2] roll its transaction back. Then check that [db1] can
+  #    commit.
+  #
+  do_test 2.$tn.3.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(4, 'four');
+    }
+  } {}
+
+  do_test 2.$tn.3.2 {
+    sql2 {
+      BEGIN;
+        INSERT INTO t1 VALUES(-1, 'xyz');
+    }
+  } {}
+
+  do_test 2.$tn.3.3.1 {
+    set rc [catch { sql1 COMMIT } msg]
+    list $rc $msg
+  } {1 {database is locked}}
+
+  do_test 2.$tn.3.3.2 {
+    code1 { list [sqlite3_extended_errcode db] [sqlite3_get_autocommit db] }
+  } {SQLITE_BUSY 0}
+
+  do_test 2.$tn.3.3.3 {
+    sql1 { SELECT * FROM t1; }
+  } {1 one 2 two 3 three 4 four}
+
+  do_test 2.$tn.3.4 {
+    sql2 ROLLBACK
+    sql1 COMMIT
+    sql1 { SELECT * FROM t1; }
+  } {1 one 2 two 3 three 4 four}
+
+  #-----------------------------------------------------------------------
+  # 1. Create a second table - t2.
+  #
+  # 2. Write to t1 with [db] and t2 with [db2].
+  #
+  # 3. See if it worked.
+  #
+  do_test 2.$tn.4.1 {
+    sql1 { CREATE TABLE t2(a, b) }
+  } {}
+  do_test 2.$tn.4.2 {
+    sql2 {
+      BEGIN CONCURRENT;
+        INSERT INTO t2 VALUES('i', 'n');
+    }
+
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(5, 'five');
+      COMMIT;
+    }
+
+    sql2 COMMIT
+  } {}
+
+  do_test 2.$tn.4.3.1 {
+    sql2 {SELECT * FROM t1}
+  } {1 one 2 two 3 three 4 four 5 five}
+  do_test 2.$tn.4.3.2 {
+    sql1 {SELECT * FROM t1}
+  } {1 one 2 two 3 three 4 four 5 five}
+
+  do_test 2.$tn.4.3.3 { sql2 {SELECT * FROM t2} } {i n}
+  do_test 2.$tn.4.3.4 { sql1 {SELECT * FROM t2} } {i n}
+
+  #-----------------------------------------------------------------------
+  # The "schema cookie" issue.
+  #
+  # 1. Begin and CONCURRENT write to "t1" using [db]
+  #
+  # 2. Create an index on t1 using [db2].
+  #
+  # 3. Attempt to commit the CONCURRENT write. This is an SQLITE_BUSY_SNAPSHOT,
+  #    even though there is no page collision.
+  #
+  do_test 2.$tn.5.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(6, 'six');
+    }
+  } {}
+
+  do_test 2.$tn.5.2 {
+    sql2 { CREATE INDEX i1 ON t1(v); }
+  } {}
+
+  do_test 2.$tn.5.3 {
+    list [catch { sql1 { COMMIT } } msg] $msg [sqlite3_errcode db]
+  } {1 {database is locked} SQLITE_BUSY_SNAPSHOT}
+
+  do_test 2.$tn.5.4 {
+    sql2 { PRAGMA integrity_check }
+  } {ok}
+  catch { sql1 ROLLBACK }
+
+  #-----------------------------------------------------------------------
+  #
+  # 1. Begin an CONCURRENT write to "t1" using [db]
+  #
+  # 2. Lots of inserts into t2. Enough to grow the db file and modify page 1.
+  #
+  # 3. Check that the CONCURRENT transaction can not be committed.
+  #
+  do_test 2.$tn.6.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(6, 'six');
+    }
+  } {}
+
+  do_test 2.$tn.6.2 {
+    sql2 { 
+      WITH src(a,b) AS (
+        VALUES(1,1) UNION ALL SELECT a+1,b+1 FROM src WHERE a<10000
+      ) INSERT INTO t2 SELECT * FROM src;
+    }
+  } {}
+
+  do_test 2.$tn.6.3 {
+    sql1 { SELECT count(*) FROM t2 }
+    list [catch { sql1 { COMMIT } } msg] $msg [sqlite3_errcode db]
+  } {1 {database is locked} SQLITE_BUSY_SNAPSHOT}
+  sql1 ROLLBACK
+
+  do_test 2.$tn.6.4 {
+    sql1 {
+      SELECT count(*) FROM t1;
+      SELECT count(*) FROM t2;
+    }
+  } {5 10001}
+
+  #-----------------------------------------------------------------------
+  # 
+  # 1. Begin an big CONCURRENT write to "t1" using [db] - large enough to
+  #    grow the db file.
+  #
+  # 2. Lots of inserts into t2. Also enough to grow the db file.
+  #
+  # 3. Check that the CONCURRENT transaction cannot be committed (due to a clash
+  #    on page 1 - the db size field).
+  #
+  do_test 2.$tn.7.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        WITH src(a,b) AS (
+          VALUES(10000,10000) UNION ALL SELECT a+1,b+1 FROM src WHERE a<20000
+        ) INSERT INTO t1 SELECT * FROM src;
+    }
+  } {}
+
+  do_test 2.$tn.7.2 {
+    sql2 { 
+      WITH src(a,b) AS (
+        VALUES(1,1) UNION ALL SELECT a+1,b+1 FROM src WHERE a<10000
+      ) INSERT INTO t2 SELECT * FROM src;
+    }
+  } {}
+
+  do_test 2.$tn.7.3 {
+    list [catch { sql1 { COMMIT } } msg] $msg [sqlite3_errcode db]
+  } {0 {} SQLITE_OK}
+
+  do_test 2.$tn.7.4 { sql3 { PRAGMA integrity_check } } ok
+}
+
+#-------------------------------------------------------------------------
+# Concurrent transactions may not modify the user_version or application_id.
+#
+reset_db
+do_execsql_test 3.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x, y);
+  INSERT INTO t1 VALUES('a', 'b');
+  PRAGMA user_version = 10;
+} {wal}
+do_execsql_test 3.1 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES('c', 'd');
+    SELECT * FROM t1;
+} {a b c d}
+do_catchsql_test 3.2 {
+  PRAGMA user_version = 11;
+} {1 {cannot modify user_version within CONCURRENT transaction}}
+do_execsql_test 3.3 {
+  PRAGMA user_version;
+  SELECT * FROM t1;
+} {10 a b c d}
+do_catchsql_test 3.4 {
+  PRAGMA application_id = 11;
+} {1 {cannot modify application_id within CONCURRENT transaction}}
+do_execsql_test 3.5 {
+  COMMIT;
+  PRAGMA user_version;
+  PRAGMA application_id;
+  SELECT * FROM t1;
+} {10 0 a b c d}
+
+#-------------------------------------------------------------------------
+# However, another transaction modifying the user_version or application_id
+# should not cause a conflict. And committing a concurrent transaction does not
+# clobber the modification - even if the concurrent transaction allocates or
+# frees database pages.
+#
+do_multiclient_test tn {
+  do_test 4.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE ttt(y UNIQUE, z UNIQUE);
+      PRAGMA user_version = 14;
+      BEGIN CONCURRENT;
+        INSERT INTO ttt VALUES('y', 'z');
+    }
+  } {wal}
+  do_test 4.$tn.2 {
+    sql2 { PRAGMA user_version = 16 }
+    sql1 COMMIT
+    sql1 { PRAGMA user_version }
+  } {16}
+
+  do_test 4.$tn.3 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO ttt VALUES(randomblob(10000), randomblob(4));
+        PRAGMA user_version;
+    }
+  } {16}
+  do_test 4.$tn.4 {
+    sql2 { PRAGMA user_version = 1234 }
+    sql1 {
+        PRAGMA user_version;
+      COMMIT;
+      PRAGMA user_version;
+      PRAGMA integrity_check;
+    }
+  } {16 1234 ok}
+
+  do_test 4.$tn.5 {
+    sql1 {
+      BEGIN CONCURRENT;
+        DELETE FROM ttt;
+        PRAGMA user_version;
+    }
+  } {1234}
+  do_test 4.$tn.4 {
+    sql2 { PRAGMA user_version = 5678 }
+    sql1 {
+        PRAGMA user_version;
+      COMMIT;
+      PRAGMA user_version;
+      PRAGMA integrity_check;
+    }
+  } {1234 5678 ok}
+}
+
+do_multiclient_test tn {
+  do_test 5.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE tt(a INTEGER PRIMARY KEY, b);
+      CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
+      INSERT INTO tt VALUES(1, randomblob(400));
+      BEGIN CONCURRENT;
+    }
+  } {wal}
+
+  do_test 5.$tn.2 {
+    sql1 { UPDATE t2 SET b=5 WHERE a=3 }
+    sql2 { INSERT INTO tt VALUES(2, randomblob(6000)) }
+  } {}
+
+  do_test 5.$tn.3 {
+    sql1 { COMMIT }
+  } {}
+}
+
+do_multiclient_test tn {
+  do_test 6.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+      CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
+      INSERT INTO t1 VALUES(1, 'one');
+      INSERT INTO t2 VALUES(2, 'two');
+    }
+  } {wal}
+
+  do_test 6.$tn.2 {
+    sql2 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t2;
+        INSERT INTO t1 VALUES(3, 'three');
+    }
+  } {2 two}
+
+  do_test 6.$tn.3 {
+    sql1 {
+      INSERT INTO t2 VALUES(3, 'three');
+    }
+  } {}
+
+  do_test 6.$tn.2 {
+    list [catch { sql2 { COMMIT } } msg] $msg
+  } {1 {database is locked}}
+}
+
+do_multiclient_test tn {
+  do_test 7.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+      WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100) 
+      INSERT INTO t1 SELECT NULL, randomblob(400) FROM s;
+
+      CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
+      WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<50000) 
+      INSERT INTO t2 SELECT NULL, randomblob(400) FROM s;
+
+      CREATE TABLE t3(a INTEGER PRIMARY KEY, b);
+      WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100) 
+      INSERT INTO t3 SELECT NULL, randomblob(400) FROM s;
+
+      CREATE TABLE t4(a INTEGER PRIMARY KEY, b);
+
+    }
+    set {} {}
+  } {}
+
+  do_test 7.$tn.2 {
+    sql2 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t1;
+        INSERT INTO t4 VALUES(1, 2);
+    }
+    set {} {}
+  } {}
+
+  do_test 7.$tn.3 {
+    sql3 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t3;
+        INSERT INTO t4 VALUES(1, 2);
+    }
+    set {} {}
+  } {}
+
+  do_test 7.$tn.4 {
+    sql1 {
+      UPDATE t1 SET b=randomblob(400);
+      UPDATE t2 SET b=randomblob(400);
+      UPDATE t3 SET b=randomblob(400);
+    }
+  } {}
+
+  do_test 7.$tn.5 {
+    csql2 { COMMIT } 
+  } {1 {database is locked}}
+
+  do_test 7.$tn.6 {
+    csql3 { COMMIT } 
+  } {1 {database is locked}}
+
+
+  csql2 ROLLBACK
+  csql3 ROLLBACK
+
+  # The following test works with $tn==1 (sql2 and sql3 use separate 
+  # processes), but is quite slow. So only run it with $tn==2 (all
+  # connections in the same process).
+  #
+  if {$tn==2} {
+    do_test 7.$tn.7 {
+      for {set i 1} {$i < 10000} {incr i} {
+        sql3 { 
+          PRAGMA wal_checkpoint;
+          BEGIN CONCURRENT;
+          SELECT * FROM t3;
+          INSERT INTO t4 VALUES(1, 2);
+        }
+
+        sql1 {
+          UPDATE t2 SET b = randomblob(400) WHERE rowid <= $i;
+          UPDATE t3 SET b = randomblob(400) WHERE rowid = 1;
+        }
+
+        if {[csql3 COMMIT]!={1 {database is locked}}} {
+          error "Failed at i=$i"
+        }
+        csql3 ROLLBACK
+      }
+    } {}
+  }
+
+}
+
+finish_test

test/concurrent2.test

@@ -0,0 +1,630 @@
+# 2015 July 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Miscellaneous tests for transactions started with BEGIN CONCURRENT. 
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+set ::testprefix concurrent2
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+do_test 0.1 {
+  llength [sqlite3_wal_info db main]
+} {2}
+
+do_multiclient_test tn {
+
+  do_test 1.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      CREATE TABLE t2(y);
+    }
+  } {wal}
+  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+
+  # Test that an CONCURRENT transaction that allocates/frees no pages does
+  # not conflict with a transaction that does allocate pages.
+  do_test 1.$tn.2  {
+    sql1 { 
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(4);
+    }
+    sql2 {
+      INSERT INTO t2 VALUES(randomblob(1500));
+    }
+    sql1 {
+      COMMIT;
+    }
+  } {}
+  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+  
+  # But that an CONCURRENT transaction does conflict with a transaction
+  # that modifies the db schema.
+  do_test 1.$tn.3  {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(5);
+    }
+    sql2 {
+      CREATE TABLE t3(z);
+    }
+    list [catch { sql1 COMMIT } msg] $msg
+  } {1 {database is locked}}
+  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+  
+  # Test that an CONCURRENT transaction that allocates at least one page 
+  # does not conflict with a transaction that allocates no pages.
+  do_test 1.$tn.4  {
+    sql1 { 
+      ROLLBACK;
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(randomblob(1500));
+    }
+    sql2 {
+      INSERT INTO t2 VALUES(8);
+    }
+    sql1 {
+      COMMIT;
+    }
+  } {}
+
+  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+}
+
+do_multiclient_test tn {
+  do_test 2.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x UNIQUE);
+      CREATE TABLE t2(y UNIQUE);
+    }
+  } {wal}
+
+  do_test 2.$tn.2  {
+    sql1 { 
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(randomblob(1500));
+    }
+    sql2 {
+      INSERT INTO t2 VALUES(randomblob(1500));
+    }
+    sql1 COMMIT
+  } {}
+
+  do_test 2.$tn.3 { sql3 { PRAGMA integrity_check } } {ok}
+
+  do_test 2.$tn.4  {
+    sql1 { 
+      BEGIN CONCURRENT;
+        DELETE FROM t1;
+    }
+    sql2 {
+      DELETE FROM t2;
+    }
+    sql1 COMMIT
+  } {}
+
+  do_test 2.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+
+  do_test 2.$tn.6 {
+    sql1 {
+      INSERT INTO t1 VALUES(randomblob(1500));
+      INSERT INTO t1 VALUES(randomblob(1500));
+      INSERT INTO t2 VALUES(randomblob(1500));
+      DELETE FROM t1 WHERE rowid=1;
+    }
+
+    sql1 {
+      BEGIN CONCURRENT;
+        DELETE FROM t1 WHERE rowid=2;
+    }
+
+    sql2 {
+      DELETE FROM t2;
+    }
+
+    sql1 COMMIT
+  } {}
+
+  do_test 2.$tn.7 { sql3 { PRAGMA integrity_check } } {ok}
+}
+
+#-------------------------------------------------------------------------
+# When an CONCURRENT transaction is opened on a database, the nFree and 
+# iTrunk header fields of the cached version of page 1 are both set 
+# to 0. This allows an CONCURRENT transaction to use its own private 
+# free-page-list, which is merged with the main database free-list when
+# the transaction is committed.
+#
+# The following tests check that nFree/iTrunk are correctly restored if
+# an CONCURRENT transaction is rolled back, and that savepoint rollbacks
+# that occur within CONCURRENT transactions do not incorrectly restore
+# these fields to their on-disk values.
+#
+reset_db
+do_execsql_test 3.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x, y);
+  INSERT INTO t1 VALUES(randomblob(1500), randomblob(1500));
+  DELETE FROM t1;
+} {wal}
+
+do_execsql_test 3.1 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(1, 2);
+  ROLLBACK;
+}
+
+do_execsql_test 3.2 { PRAGMA integrity_check } {ok}
+do_execsql_test 3.3 { PRAGMA freelist_count } {2}
+
+do_execsql_test 3.4.1 {
+  BEGIN CONCURRENT;
+    PRAGMA freelist_count;
+} {2}
+do_execsql_test 3.4.2 {
+  SAVEPOINT xyz;
+    INSERT INTO t1 VALUES(randomblob(1500), NULL);
+    PRAGMA freelist_count;
+} {0}
+do_execsql_test 3.4.3 {
+  ROLLBACK TO xyz;
+} {}
+do_execsql_test 3.4.4 { PRAGMA freelist_count } {0}
+do_execsql_test 3.4.5 { COMMIT; PRAGMA freelist_count } {2}
+do_execsql_test 3.4.6 { PRAGMA integrity_check } {ok}
+
+do_execsql_test 3.5.1 {
+  BEGIN CONCURRENT;
+    UPDATE t1 SET x=randomblob(10) WHERE y=555;
+    PRAGMA freelist_count;
+} {0}
+do_execsql_test 3.5.2 {
+  ROLLBACK;
+  PRAGMA freelist_count;
+} {2}
+do_execsql_test 3.5.3 { PRAGMA integrity_check } {ok}
+
+#-------------------------------------------------------------------------
+# Test that nothing goes wrong if an CONCURRENT transaction allocates a
+# page at the end of the file, frees it within the same transaction, and
+# then has to move the same page to avoid a conflict on COMMIT.
+#
+do_multiclient_test tn {
+  do_test 4.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      CREATE TABLE t2(x);
+    }
+  } {wal}
+
+  do_test 4.$tn.2 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(randomblob(1500));
+        INSERT INTO t1 VALUES(randomblob(1500));
+        DELETE FROM t1 WHERE rowid = 1;
+    }
+
+    sql2 {
+      INSERT INTO t2 VALUES(randomblob(1500));
+      INSERT INTO t2 VALUES(randomblob(1500));
+      INSERT INTO t2 VALUES(randomblob(1500));
+      INSERT INTO t2 VALUES(randomblob(1500));
+      DELETE FROM t2 WHERE rowid IN (1, 2);
+    }
+
+    sql1 COMMIT
+  } {}
+}
+
+#-------------------------------------------------------------------------
+#
+do_multiclient_test tn {
+  do_test 5.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      CREATE TABLE t2(x);
+      INSERT INTO t1 VALUES(randomblob(1500));
+      PRAGMA page_count;
+    }
+  } {wal 4}
+
+  do_test 5.$tn.2 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t2 VALUES(randomblob(1500));
+        PRAGMA page_count;
+    }
+  } {5}
+
+  do_test 5.$tn.3 {
+    sql2 { 
+      DELETE FROM t1;
+      PRAGMA freelist_count;
+      PRAGMA page_count;
+    }
+  } {1 4}
+
+  do_test 5.$tn.4 { sql1 COMMIT } {}
+  do_test 5.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+}
+
+#-------------------------------------------------------------------------
+#
+do_multiclient_test tn {
+  do_test 6.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      INSERT INTO t1 VALUES(randomblob(1500));
+      PRAGMA wal_checkpoint;
+    }
+  } {wal 0 5 5}
+
+  do_test 6.$tn.2 {
+    sql1 { 
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(randomblob(1500));
+        INSERT INTO t1 VALUES(randomblob(1500));
+    }
+  } {}
+
+  do_test 6.$tn.3 {
+    sql2 {
+      BEGIN;
+        INSERT INTO t1 VALUES(randomblob(1500));
+        INSERT INTO t1 VALUES(randomblob(1500));
+      COMMIT;
+    }
+  } {}
+
+  do_test 6.$tn.4 { 
+    list [catch { sql1 COMMIT } msg] $msg
+  } {1 {database is locked}}
+  do_test 6.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
+  do_test 6.$tn.5 { sql3 { SELECT count(*) from t1 } } {3}
+}
+
+#-------------------------------------------------------------------------
+# Test that if a corrupt wal-index-header is encountered when attempting
+# to commit a CONCURRENT transaction, the transaction is not committed
+# (or rolled back) and that SQLITE_BUSY_SNAPSHOT is returned to the user.
+#
+catch { db close }
+forcedelete test.db
+testvfs tvfs
+sqlite3 db test.db -vfs tvfs
+do_execsql_test 7.1 {
+  PRAGMA journal_mode = wal;
+  BEGIN;
+    CREATE TABLE t1(a, b, PRIMARY KEY(a));
+    INSERT INTO t1 VALUES(1, 2);
+    INSERT INTO t1 VALUES(3, 4);
+  COMMIT;
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(5, 6);
+    INSERT INTO t1 VALUES(7, 8);
+    SELECT * FROM t1;
+} {wal 1 2 3 4 5 6 7 8}
+
+# Corrupt the wal-index header
+incr_tvfs_hdr test.db 11 1
+
+do_catchsql_test 7.2.1 { COMMIT } {1 {database is locked}}
+do_test 7.2.2 { sqlite3_extended_errcode db } SQLITE_BUSY_SNAPSHOT
+
+do_execsql_test 7.3.1 {
+  SELECT * FROM t1;
+  ROLLBACK;
+} {1 2 3 4 5 6 7 8}
+do_execsql_test 7.3.2 {
+  SELECT * FROM t1;
+} {1 2 3 4}
+
+#-------------------------------------------------------------------------
+# Test that "PRAGMA integrity_check" works within a concurrent 
+# transaction. Within a concurrent transaction, "PRAGMA integrity_check"
+# is unable to detect unused database pages, but can detect other types
+# of corruption.
+#
+reset_db
+do_test 8.1 {
+  execsql {
+    PRAGMA journal_mode = wal;
+    CREATE TABLE kv(k INTEGER PRIMARY KEY, v UNIQUE);
+    INSERT INTO kv VALUES(NULL, randomblob(750));
+    INSERT INTO kv SELECT NULL, randomblob(750) FROM kv;
+    INSERT INTO kv SELECT NULL, randomblob(750) FROM kv;
+    INSERT INTO kv SELECT NULL, randomblob(750) FROM kv;
+    INSERT INTO kv SELECT NULL, randomblob(750) FROM kv;
+    INSERT INTO kv SELECT NULL, randomblob(750) FROM kv;
+    DELETE FROM kv WHERE rowid%2;
+  }
+  set v [db one {PRAGMA freelist_count}]
+  expr $v==33 || $v==34
+} {1}
+do_execsql_test 8.2 { PRAGMA integrity_check } ok
+do_execsql_test 8.3 { 
+  BEGIN CONCURRENT;
+    PRAGMA integrity_check;
+} {ok}
+do_execsql_test 8.4 { 
+    INSERT INTO kv VALUES(1100, 1100);
+    PRAGMA integrity_check;
+} {ok}
+do_execsql_test 8.5 { 
+  COMMIT;
+  PRAGMA integrity_check;
+} {ok}
+
+#-------------------------------------------------------------------------
+# Test that concurrent transactions do not allow foreign-key constraints
+# to be bypassed.
+#
+do_multiclient_test tn {
+  do_test 9.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE pp(i INTEGER PRIMARY KEY, j);
+      CREATE TABLE cc(a, b REFERENCES pp);
+
+      WITH seq(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM seq WHERE i<100)
+      INSERT INTO pp SELECT i, randomblob(1000) FROM seq;
+
+      PRAGMA foreign_keys = 1;
+    }
+  } {wal}
+
+
+  do_test 9.$tn.2.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO cc VALUES(42, 42);
+    }
+  } {}
+  do_test 9.$tn.2.2 {
+    sql2 { DELETE FROM pp WHERE i=42 }
+    list [catch { sql1 COMMIT } msg] $msg
+  } {1 {database is locked}}
+  do_test 9.$tn.2.3 {
+    sql1 ROLLBACK
+  } {}
+
+  do_test 9.$tn.3.1 {
+    sql1 {
+      PRAGMA foreign_keys = 0;
+      BEGIN CONCURRENT;
+        INSERT INTO cc VALUES(43, 43);
+    }
+  } {}
+  do_test 9.$tn.3.2 {
+    sql2 { DELETE FROM pp WHERE i=43 }
+    list [catch { sql1 COMMIT } msg] $msg
+  } {0 {}}
+
+  do_test 9.$tn.4.1 {
+    sql1 {
+      PRAGMA foreign_keys = on;
+      BEGIN CONCURRENT;
+        INSERT INTO cc VALUES(44, 44);
+    }
+  } {}
+  do_test 9.$tn.4.2 {
+    sql2 { DELETE FROM pp WHERE i=1 }
+    list [catch { sql1 COMMIT } msg] $msg
+  } {0 {}}
+}
+
+#-------------------------------------------------------------------------
+# Test that even if a SELECT statement appears before all writes within
+# a CONCURRENT transaction, the pages it reads are still considered when
+# considering whether or not the transaction may be committed.
+#
+do_multiclient_test tn {
+  do_test 10.$tn.1.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(a);
+      CREATE TABLE t2(b);
+      CREATE TABLE t3(c);
+      INSERT INTO t1 VALUES(1), (2), (3);
+      INSERT INTO t2 VALUES(1), (2), (3);
+      INSERT INTO t3 VALUES(1), (2), (3);
+    }
+  } {wal}
+
+  do_test 10.$tn.1.2 {
+    sql1 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t1;
+        INSERT INTO t2 VALUES(4);
+    }
+  } {1 2 3}
+
+  do_test 10.$tn.1.3 {
+    sql2 { INSERT INTO t1 VALUES(4) }
+    list [catch {sql1 COMMIT} msg] $msg
+  } {1 {database is locked}}
+  sql1 ROLLBACK
+
+  # In this case, because the "SELECT * FROM t1" is first stepped before
+  # the "BEGIN CONCURRENT", the pages it reads are not recorded by the
+  # pager object. And so the transaction can be committed. Technically
+  # this behaviour (the effect of an ongoing SELECT on a BEGIN CONCURRENT
+  # transacation) is undefined.
+  #
+  do_test 10.$tn.2.1 {
+    code1 {
+      set ::stmt [sqlite3_prepare db "SELECT * FROM t1" -1 dummy]
+      sqlite3_step $::stmt
+    }
+  } {SQLITE_ROW}
+  do_test 10.$tn.2.2 {
+    sql1 {
+      BEGIN CONCURRENT; 
+        INSERT INTO t2 VALUES(4);
+    }
+    code1 {
+      set res [list]
+      lappend res [sqlite3_column_int $::stmt 0]
+      while {[sqlite3_step $::stmt]=="SQLITE_ROW"} {
+        lappend res [sqlite3_column_int $::stmt 0]
+      }
+      sqlite3_finalize $::stmt
+      set res
+    }
+  } {1 2 3 4}
+  do_test 10.$tn.2.3 {
+    sql2 { INSERT INTO t1 VALUES(5) }
+    sql1 COMMIT
+  } {}
+
+  # More tests surrounding long-lived prepared statements and concurrent
+  # transactions.
+  do_test 10.$tn.3.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t1;
+      COMMIT;
+    }
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t2 VALUES(5);
+    }
+    sql2 {
+      INSERT INTO t1 VALUES(5);
+    }
+    sql1 COMMIT
+    sql3 {
+      SELECT * FROM t2;
+    }
+  } {1 2 3 4 5}
+  do_test 10.$tn.3.2 {
+    sql1 {
+      BEGIN CONCURRENT;
+        SELECT * FROM t1;
+      ROLLBACK;
+    }
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t2 VALUES(6);
+    }
+    sql2 {
+      INSERT INTO t1 VALUES(6);
+    }
+    sql1 COMMIT
+    sql3 { SELECT * FROM t2 }
+  } {1 2 3 4 5 6}
+  do_test 10.$tn.3.3 {
+    sql1 { BEGIN CONCURRENT }
+    code1 {
+      set ::stmt [sqlite3_prepare db "SELECT * FROM t1" -1 dummy]
+      sqlite3_step $::stmt
+    }
+    sql1 {
+      INSERT INTO t2 VALUES(7);
+      SELECT * FROM t3;
+      ROLLBACK;
+      BEGIN CONCURRENT;
+    }
+    sql2 { INSERT INTO t3 VALUES(5) }
+    code1 { sqlite3_finalize $::stmt }
+    sql1 {
+      INSERT INTO t2 VALUES(8);
+      COMMIT;
+    }
+  } {}
+}
+
+do_multiclient_test tn {
+  do_test 11.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(a);
+    }
+  } {wal}
+
+  do_test 11.$tn.2 {
+    code1 { sqlite3_wal_info db main }
+  } {0 2}
+
+  do_test 11.$tn.3 {
+    sql1 { INSERT INTO t1 VALUES(1) }
+    code1 { sqlite3_wal_info db main }
+  } {2 3}
+
+  do_test 11.$tn.4 {
+    sql2 { INSERT INTO t1 VALUES(2) }
+    code2 { sqlite3_wal_info db2 main }
+  } {3 4}
+
+  do_test 11.$tn.5 {
+    sql1 { PRAGMA wal_checkpoint }
+    sql2 { INSERT INTO t1 VALUES(3) }
+    code2 { sqlite3_wal_info db2 main }
+  } {0 1}
+}
+
+reset_db
+do_execsql_test 12.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE tx(a INTEGER PRIMARY KEY, b);
+} {wal}
+do_test 12.1 {
+  for {set i 0} {$i < 50} {incr i} {
+    execsql {
+      BEGIN CONCURRENT;
+      INSERT INTO tx(b) VALUES( randomblob( 1200 ) );
+      COMMIT;
+    }
+  }
+  execsql { PRAGMA page_size }
+} {1024}
+do_execsql_test 12.2 {
+  DELETE FROM tx;
+}
+do_test 12.3 {
+  for {set i 0} {$i < 50} {incr i} {
+    execsql {
+      BEGIN CONCURRENT;
+      INSERT INTO tx(b) VALUES( randomblob( 1200 ) );
+      COMMIT;
+    }
+  }
+  execsql { PRAGMA page_size }
+} {1024}
+do_execsql_test 12.4 {
+  DELETE FROM tx;
+}
+do_test 12.5 {
+  execsql { BEGIN CONCURRENT }
+  for {set i 0} {$i < 5000} {incr i} {
+    execsql {
+      INSERT INTO tx(b) VALUES( randomblob( 1200 ) );
+    }
+  }
+  execsql { COMMIT }
+  execsql { PRAGMA page_size }
+} {1024}
+
+
+finish_test

test/concurrent3.test

@@ -0,0 +1,231 @@
+# 2015 July 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests for transactions started with BEGIN CONCURRENT. The tests in this
+# file focus on testing that deferred page allocation works properly.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set ::testprefix concurrent3
+
+if {$AUTOVACUUM} { finish_test ; return }
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+db close
+sqlite3_shutdown
+test_sqlite3_log xLog
+proc xLog {error_code msg} {
+  # puts "$error_code: $msg"
+  # Enable the previous for debugging
+}
+reset_db
+
+proc create_schema {} {
+  db eval {
+    PRAGMA journal_mode = wal;
+
+    CREATE TABLE t1(x, y);
+    CREATE TABLE t2(x, y);
+    CREATE TABLE t3(x, y);
+    CREATE TABLE t4(x, y);
+
+    CREATE INDEX i1 ON t1(y, x);
+    CREATE INDEX i2 ON t2(y, x);
+    CREATE INDEX i3 ON t3(y, x);
+    CREATE INDEX i4 ON t4(y, x);
+  }
+}
+
+proc do_sql_op {iTbl iOp} {
+  set db "db$iTbl"
+
+  switch $iOp {
+    "i" {
+      set sql "
+        WITH cnt(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM cnt WHERE i<10)
+        INSERT INTO t$iTbl SELECT randomblob(800), randomblob(800) FROM cnt;
+      "
+    }
+
+    "d" {
+      set sql "
+        DELETE FROM t$iTbl WHERE rowid IN (
+          SELECT rowid FROM t$iTbl ORDER BY 1 ASC LIMIT 10
+        )
+      "
+    }
+
+    "D" {
+      set sql "
+        DELETE FROM t$iTbl WHERE rowid IN (
+          SELECT rowid FROM t$iTbl o WHERE (
+            SELECT count(*) FROM t$iTbl i WHERE i.rowid<o.rowid
+          ) % 2
+        )
+      "
+    }
+
+    "I" {
+      set sql "
+        INSERT INTO t$iTbl SELECT randomblob(800), randomblob(800) FROM t$iTbl;
+      "
+    }
+
+    default {
+      error "bad iOp parameter: $iOp"
+    }
+  }
+
+  $db eval $sql
+}
+
+
+set DBLIST {db1 db2 db3 db4} 
+
+create_schema
+foreach {tn oplist} {
+  1  {1i   2i   3i   4i} 
+  2  {1iii 2iii 3iii 4iii}
+  3  {1d   2d   3d   4d} 
+  .  -----------------------
+  4  {1i}
+  5  {1d 2i}
+  .  -----------------------
+  6  {1iii 2iii 3iii 4iii}
+  7  {1di  2id  3iii 4ddd}
+  8  {1iii 2iii 3iii 4iii}
+  9  {1D  2II}
+  10 {1I  2D  3I  4D}
+  11 {1III 3dddddd 4III}
+} {
+  if {[string range $oplist 0 0]=="-"} {
+    reset_db
+    create_schema
+    continue
+  }
+  foreach db $DBLIST { sqlite3 $db test.db }
+
+  do_test 1.$tn {
+    foreach db $DBLIST { $db eval "BEGIN CONCURRENT" } 
+
+    foreach op $oplist {
+      set iTbl [string range $op 0 0]
+      foreach char [split [string range $op 1 end] {}] {
+        do_sql_op $iTbl $char
+      }
+    }
+
+    foreach db $DBLIST { $db eval "COMMIT" }
+    db eval {PRAGMA integrity_check}
+  } {ok}
+
+  foreach db $DBLIST { 
+    $db close 
+  }
+}
+
+#-------------------------------------------------------------------------
+#
+proc create_schema2 {} {
+  db eval {
+    PRAGMA journal_mode = wal;
+    CREATE TABLE t1(x INTEGER PRIMARY KEY, y);
+    CREATE INDEX i1 ON t1(y);
+  }
+}
+
+proc randint {nMax} {
+  db eval {SELECT abs(random() % $nMax)}
+}
+
+proc do_sql_op2 {db iOp} {
+  switch -- $iOp {
+    i {
+      # Insert 1 rows.
+      set r [randint 1000000000]
+      set ::rows($r) 1
+      #puts "insert row $r"
+      $db eval { INSERT OR IGNORE INTO t1 VALUES($r, randomblob(50)); }
+    }
+
+    d {
+      # Insert 1 row
+      set keys [array names ::rows]
+      set r [randint [llength $keys]]
+      set rowid [lindex $keys $r]
+      $db eval { DELETE FROM t1 WHERE x=$rowid }
+      unset ::rows($rowid)
+    }
+  }
+}
+
+foreach {tn nRepeat oplist} {
+  - - ----------------------------
+  1 100 { 1iiiiiiiiii }
+  2 100 { 1i 2d }
+  3 100 { 1d 2i }
+  4  50 { 1d 2i 3d }
+  5 500 { 1i 2i 3i 4i }
+  6 500 { 1i 2d 3d 4d }
+} {
+  if {[string range $oplist 0 0]=="-"} {
+    array unset rows
+    reset_db
+    create_schema2
+    continue
+  }
+
+  foreach db $DBLIST { 
+    sqlite3 $db test.db 
+    set stats($db,0) 0
+    set stats($db,1) 0
+  }
+  array unset used
+
+  do_test 2.$tn {
+
+    for {set i 0} {$i < $nRepeat} {incr i} {
+      foreach db $DBLIST { $db eval "BEGIN CONCURRENT" } 
+
+      foreach op $oplist {
+        set iDb [string range $op 0 0]
+        set used(db$iDb) 1
+        foreach char [split [string range $op 1 end] {}] {
+          do_sql_op2 "db$iDb" $char
+        }
+      }
+
+      foreach db $DBLIST { 
+        set rc [catch { $db eval COMMIT } msg]
+        if {$rc} { $db eval ROLLBACK }
+        incr stats($db,$rc)
+      }
+      set res [db eval {PRAGMA integrity_check}]
+      if {$res != "ok"} { puts "after $db $rc: $res" ; after 1000000 }
+    }
+  } {}
+
+  foreach db $DBLIST { 
+    $db close 
+  }
+  # foreach k [lsort [array names used]] {
+  #   puts "$k: $stats($k,0) committed, $stats($k,1) rolled back"
+  # }
+}
+
+
+
+finish_test

test/concurrent4.test

@@ -0,0 +1,194 @@
+# 2017 May 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Miscellaneous tests for transactions started with BEGIN CONCURRENT. 
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+set ::testprefix concurrent4
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x PRIMARY KEY, y UNIQUE);
+  WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s;
+  DELETE FROM t1 WHERE rowid<2;
+} {wal}
+
+do_execsql_test 1.1 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1(rowid, x, y) VALUES(1000, randomblob(3000), randomblob(3000));
+    SAVEPOINT abc;
+    DELETE FROM t1 WHERE rowid = 1000;
+}
+
+do_execsql_test 1.2 { ROLLBACK TO abc }
+do_execsql_test 1.3 { COMMIT }
+do_execsql_test 1.4 { PRAGMA integrity_check } {ok}
+
+do_multiclient_test tn {
+  do_test 2.$tn.1 {
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(a, b);
+      WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100)
+        INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s;
+
+      CREATE TABLE t2(a, b);
+      WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100)
+        INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM s;
+    }
+
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(randomblob(3000), randomblob(3000));
+    }
+  } {}
+
+  do_test 2.$tn.2 {
+    sql2 {
+      WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10)
+        INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM s;
+    }
+    sql2 {
+      DELETE FROM t2 WHERE rowid<10;
+    }
+  } {}
+
+  do_test 2.$tn.3 {
+    sql1 {
+      COMMIT;
+      PRAGMA integrity_check;
+    }
+  } {ok}
+  do_test 2.$tn.4 {
+    sql2 {
+      PRAGMA integrity_check;
+    }
+  } {ok}
+}
+
+reset_db
+do_execsql_test 3.1 {
+  PRAGMA page_size = 1024;
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t2(x);
+  INSERT INTO t2 VALUES(randomblob(5000));
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+  INSERT INTO t1 VALUES(25, randomblob(104));
+  DELETE FROM t2;
+} {wal}
+
+do_execsql_test 3.2 {
+  BEGIN CONCURRENT;
+    REPLACE INTO t1 VALUES(25, randomblob(1117));
+  COMMIT;
+} {}
+
+#-------------------------------------------------------------------------
+# Test the effect of BEGIN CONCURRENT transactions that consist entirely
+# of read-only statements.
+#
+reset_db
+do_execsql_test 4.0 {
+  PRAGMA page_size = 1024;
+  PRAGMA journal_mode = wal;
+  
+  CREATE TABLE t4(a, b);
+  INSERT INTO t4 VALUES(1, 2);
+  INSERT INTO t4 VALUES(3, 4);
+} {wal}
+ 
+sqlite3 db2 test.db
+do_test 4.1.1 {
+  db eval {
+    BEGIN CONCURRENT;
+      INSERT INTO t4 VALUES(5, 6);
+  }
+
+  db2 eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+    ROLLBACK;
+  }
+} {1 2 3 4}
+
+do_test 4.1.2 {
+  db eval { COMMIT }
+  db2 eval { SELECT * FROM t4 }
+} {1 2 3 4 5 6}
+
+do_test 4.2.1 {
+  db eval {
+    BEGIN CONCURRENT;
+      INSERT INTO t4 VALUES(7, 8);
+  }
+
+  db2 eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+    COMMIT;
+  }
+} {1 2 3 4 5 6}
+
+do_test 4.2.2 {
+  db eval { COMMIT }
+  db2 eval { SELECT * FROM t4 }
+} {1 2 3 4 5 6 7 8}
+
+do_test 4.3 {
+  db2 eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+  }
+
+  db eval {
+    BEGIN CONCURRENT;
+      INSERT INTO t4 VALUES(9, 10);
+    COMMIT;
+  }
+  db2 eval {
+    SELECT * FROM t4;
+    COMMIT;
+  }
+} {1 2 3 4 5 6 7 8}
+
+set sz [file size test.db-wal]
+do_test 4.4.1 {
+  db eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+      SELECT * FROM sqlite_master;
+  }
+
+  db eval COMMIT
+  file size test.db-wal
+} $sz
+do_test 4.4.2 {
+  db eval {
+    BEGIN CONCURRENT;
+      SELECT * FROM t4;
+      SELECT * FROM sqlite_master;
+    ROLLBACK;
+  }
+  file size test.db-wal
+} $sz
+
+finish_test
+

test/concurrent5.test

@@ -0,0 +1,133 @@
+# 2017 May 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+set ::testprefix concurrent5
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+db close
+sqlite3_shutdown
+test_sqlite3_log [list lappend ::log]
+set ::log [list]
+
+sqlite3 db test.db
+
+proc do_test_conflict_msg {tn msg} {
+  set msg "cannot commit CONCURRENT transaction - [string trim $msg]"
+  uplevel [list do_test $tn {lindex $::log end} $msg]
+}
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x, y);
+  CREATE TABLE t2(c);
+  WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s;
+} {wal}
+
+sqlite3 db2 test.db
+
+do_test 1.1.1 {
+  set ::log [list]
+  db2 eval {
+    BEGIN CONCURRENT;
+      SELECT count(*) FROM t1;
+      INSERT INTO t2 VALUES(10);
+  }
+
+  db eval {
+    INSERT INTO t1 VALUES(randomblob(200), randomblob(200));
+  }
+
+  catchsql COMMIT db2
+} {1 {database is locked}}
+do_test_conflict_msg 1.1.2 {
+  conflict at page 2 (read-only page; part of db table t1; content=0500000063021100...)
+}
+
+do_test 1.2.1 {
+  set ::log [list]
+  db2 eval {
+    ROLLBACK;
+    BEGIN CONCURRENT;
+      INSERT INTO t1 VALUES(11, 12);
+  }
+
+  db eval {
+    INSERT INTO t1 VALUES(12, 11);
+  }
+
+  catchsql COMMIT db2
+} {1 {database is locked}}
+
+do_test_conflict_msg 1.2.2 {
+  conflict at page 105 (read/write page; part of db table t1; content=0D00000002026100...)
+}
+
+do_test 1.3.1 {
+  set ::log [list]
+  db2 eval {
+    ROLLBACK;
+    BEGIN CONCURRENT;
+      INSERT INTO t2 VALUES('x');
+  }
+
+  db eval {
+    INSERT INTO t2 VALUES('y');
+  }
+
+  catchsql COMMIT db2
+} {1 {database is locked}}
+
+do_test_conflict_msg 1.3.2 {
+  conflict at page 3 (read/write page; part of db table t2; content=0D0000000103FB00...)
+}
+
+do_test 1.4.1 {
+  set ::log [list]
+
+  execsql {
+    ROLLBACK;
+    CREATE TABLE t3(a INTEGER PRIMARY KEY, b INTEGER);
+    CREATE INDEX i3 ON t3(b);
+
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<5000
+    ) INSERT INTO t3 SELECT i, i FROM s;
+
+    BEGIN CONCURRENT;
+      INSERT INTO t3 VALUES(0, 5001);
+  } db2
+
+  execsql { INSERT INTO t3 VALUES(NULL, 5002) } db
+  catchsql COMMIT db2
+} {1 {database is locked}}
+
+do_test_conflict_msg 1.3.2 {
+  conflict at page 211 (read/write page; part of db index t3.i3; content=0A0310006300D800...)
+}
+
+db close
+db2 close
+sqlite3_shutdown
+test_sqlite3_log 
+sqlite3_initialize
+finish_test
+

test/concurrent6.test

@@ -0,0 +1,60 @@
+# 2017 May 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+set ::testprefix concurrent6
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+sqlite3 db2 test.db
+
+do_execsql_test 1.0 {
+  PRAGMA page_size = 1024;
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(x);
+  CREATE TABLE t3(x);
+  CREATE TABLE t4(x);
+
+  INSERT INTO t1 VALUES(zeroblob(1500));
+} {wal}
+
+do_execsql_test -db db2 1.1 {
+  BEGIN CONCURRENT;
+    INSERT INTO t3 VALUES(zeroblob(4000));
+    DELETE FROM t1;
+}
+
+do_execsql_test 1.2 {
+  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) 
+  INSERT INTO t2 SELECT zeroblob(1000) FROM s;
+
+  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) 
+  INSERT INTO t4  SELECT zeroblob(1000) FROM s;
+
+  DELETE FROM t4;
+}
+
+do_execsql_test -db db2 1.3 {
+  COMMIT;
+}
+
+
+finish_test
+

test/concurrent7.test

@@ -0,0 +1,52 @@
+# 2018 Jan 5
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix concurrent7
+
+sqlite3 db2 test.db
+
+do_execsql_test 1 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(x);
+} {wal}
+
+do_execsql_test -db db2 2 {
+  SELECT * FROM t1;
+}
+
+do_execsql_test 3 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(randomblob(1500));
+    INSERT INTO t1 VALUES(randomblob(1500));
+    DELETE FROM t1 WHERE rowid = 1;
+}
+
+do_execsql_test -db db2 4 {
+  INSERT INTO t2 VALUES(randomblob(1500));
+  INSERT INTO t2 VALUES(randomblob(1500));
+  INSERT INTO t2 VALUES(randomblob(1500));
+  INSERT INTO t2 VALUES(randomblob(1500));
+  DELETE FROM t2 WHERE rowid IN (1, 2);
+}
+
+do_execsql_test 5 {
+  COMMIT;
+  PRAGMA integrity_check;
+} {ok}
+
+finish_test
+
+

test/concurrent8.test

@@ -0,0 +1,109 @@
+# 2020 July 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix concurrent8
+
+source $testdir/lock_common.tcl
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+do_multiclient_test tn {
+
+  do_test 1.$tn.0 {
+    sql1 {
+      CREATE TABLE t1(x, y);
+      PRAGMA journal_mode = wal;
+    }
+  } {wal}
+
+  do_test 1.$tn.1 {
+    sql1 {
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(1, 1);
+    }
+  } {}
+
+  do_test 1.$tn.2 {
+    sql2 {
+      CREATE TABLE t2(a, b);
+    }
+  } {}
+
+  do_test 1.$tn.3 {
+    list [catch { sql1 { COMMIT } } msg] $msg
+  } {1 {database is locked}}
+
+  do_test 1.$tn.4 {
+    code1 { db errorcode }
+  } {517}  ;# SQLITE_BUSY_SNAPSHOT
+
+  do_test 1.$tn.5 {
+    sql1 {
+      ROLLBACK;
+      BEGIN CONCURRENT;
+        CREATE TABLE t3(a, b);
+      COMMIT;
+    }
+  } {}
+
+  do_test 1.$tn.6 {
+    set nPg [sql1 {PRAGMA page_count}]
+    sql1 "BEGIN CONCURRENT"
+    for {set i 0} {$i<250} {incr i} {
+      sql1 "CREATE TABLE z$i (a, b, c)"
+    }
+    sql1 "COMMIT"
+    set nPg2 [sql1 {PRAGMA page_count}]
+    expr $nPg2>$nPg
+  } {1}
+
+  do_test 1.$tn.7 {
+    sql2 { PRAGMA integrity_check }
+  } {ok}
+
+  do_test 1.$tn.8 {
+    sql1 {
+      BEGIN CONCURRENT;
+        CREATE TABLE t4(a, b);
+    }
+    sql2 {
+      INSERT INTO t1 VALUES(2, 2);
+    }
+    list [catch { sql1 COMMIT } msg] $msg
+  } {1 {database is locked}}
+  sql1 ROLLBACK
+
+  do_test 1.$tn.9 {
+    sql1 {
+      BEGIN CONCURRENT;
+        CREATE TEMP TABLE t5(a, b);
+        INSERT INTO t2 VALUES('x', 'x');
+    }
+    sql2 {
+      INSERT INTO t1 VALUES(3, 3);
+      CREATE TEMP TABLE t1(x, y);
+    }
+    sql1 COMMIT 
+  } {}
+}
+
+
+
+finish_test
+
+

test/corruptA.test

@@ -47,7 +47,7 @@ db close
 forcecopy test.db test.db-template
 
 set unreadable_version 02
-ifcapable wal { set unreadable_version 03 }
+ifcapable wal { set unreadable_version 04 }
 do_test corruptA-2.1 {
   forcecopy test.db-template test.db
   hexio_write test.db 19 $unreadable_version   ;# the read format number

test/corruptN.test

@@ -141,16 +141,6 @@ do_test 2.0 {
 | end c-b92b.txt.db
 }]} {}
 
-# This test only works with the legacy RC4 PRNG
-if 0 {
-  prng_seed 0 db
-  do_catchsql_test 2.1 {
-  SELECT count(*) FROM sqlite_schema;
-  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000)
-  INSERT INTO t1(a) SELECT randomblob(null) FROM c;
-  } {1 {database disk image is malformed}}
-}
-
 reset_db
 if {![info exists ::G(perm:presql)]} {
   do_execsql_test 3.0 {

test/parser1.test

@@ -1,103 +0,0 @@
-# 2014-08-24
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-# This file implements regression tests for SQLite library.
-# The focus of this script is testing details of the SQL language parser.
-# 
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-do_catchsql_test parser1-1.1 {
-  CREATE TABLE t1(
-    a TEXT PRIMARY KEY,
-    b TEXT,
-    FOREIGN KEY(b COLLATE nocase DESC) REFERENCES t1(a COLLATE binary ASC)
-  );
-} {1 {syntax error after column name "b"}}
-
-
-# Verify that a legacy schema in the sqlite_master file is allowed to have
-# COLLATE, ASC, and DESC keywords on the id list of a FK constraint, and that
-# those keywords are silently ignored.
-#
-sqlite3_db_config db DEFENSIVE 0
-do_execsql_test parser1-1.2 {
-  CREATE TABLE t1(
-    a TEXT PRIMARY KEY,
-    b TEXT,
-    FOREIGN KEY(b) REFERENCES t1(a)
-  );
-  INSERT INTO t1 VALUES('abc',NULL),('xyz','abc');
-  PRAGMA writable_schema=on;
-  UPDATE sqlite_master SET sql='CREATE TABLE t1(
-    a TEXT PRIMARY KEY,
-    b TEXT,
-    FOREIGN KEY(b COLLATE nocase) REFERENCES t1(a)
-  )' WHERE name='t1';
-  SELECT name FROM sqlite_master WHERE sql LIKE '%collate%';
-} {t1}
-sqlite3 db2 test.db
-do_test parser1-1.3 {
-  sqlite3 db2 test.db
-  db2 eval {SELECT * FROM t1 ORDER BY 1}
-} {abc {} xyz abc}
-db2 close
-
-do_execsql_test parser1-1.4 {
-  UPDATE sqlite_master SET sql='CREATE TABLE t1(
-    a TEXT PRIMARY KEY,
-    b TEXT,
-    FOREIGN KEY(b ASC) REFERENCES t1(a)
-  )' WHERE name='t1';
-  SELECT name FROM sqlite_master WHERE sql LIKE '%ASC%';
-} {t1}
-sqlite3 db2 test.db
-do_test parser1-1.5 {
-  sqlite3 db2 test.db
-  db2 eval {SELECT * FROM t1 ORDER BY 1}
-} {abc {} xyz abc}
-db2 close
-
-do_catchsql_test parser1-2.1 {
-  WITH RECURSIVE
-    c(x COLLATE binary) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<5)
-  SELECT x FROM c;
-} {1 {syntax error after column name "x"}}
-do_catchsql_test parser1-2.2 {
-  WITH RECURSIVE
-    c(x ASC) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<5)
-  SELECT x FROM c;
-} {1 {syntax error after column name "x"}}
-
-# Verify that the comma between multiple table constraints is
-# optional.
-#
-# The missing comma is technically a syntax error.  But we have to support
-# it because there might be legacy databases that omit the commas in their
-# sqlite_master tables.
-#
-do_execsql_test parser1-3.1 {
-  CREATE TABLE t300(id INTEGER PRIMARY KEY);
-  CREATE TABLE t301(
-    id INTEGER PRIMARY KEY,
-    c1 INTEGER NOT NULL,
-    c2 INTEGER NOT NULL,
-    c3 BOOLEAN NOT NULL DEFAULT 0,
-    FOREIGN KEY(c1) REFERENCES t300(id) ON DELETE CASCADE ON UPDATE RESTRICT
-        /* no comma */
-    FOREIGN KEY(c2) REFERENCES t300(id) ON DELETE CASCADE ON UPDATE RESTRICT
-        /* no comma */
-    UNIQUE(c1, c2)
-  );
-  PRAGMA foreign_key_list(t301);
-} {0 0 t300 c2 id RESTRICT CASCADE NONE 1 0 t300 c1 id RESTRICT CASCADE NONE}
-
-finish_test

test/permutations.test

@@ -89,7 +89,6 @@ foreach f [glob $testdir/*.test] { lappend alltests [file tail $f] }
 foreach f [glob -nocomplain            \
     $testdir/../ext/rtree/*.test       \
     $testdir/../ext/fts5/test/*.test   \
-    $testdir/../ext/expert/*.test      \
     $testdir/../ext/lsm1/test/*.test   \
 ] {
   lappend alltests $f 
@@ -188,7 +187,7 @@ test_suite "veryquick" -prefix "" -description {
   that test malloc and IO errors are omitted.
 } -files [
   test_set $allquicktests -exclude *malloc* *ioerr* *fault* *bigfile* *_err* \
-      *fts5corrupt* *fts5big* *fts5aj*
+      *fts5corrupt* *fts5big* *fts5aj* wal2big.test
 ]
 
 test_suite "shell" -prefix "" -description {
@@ -458,18 +457,28 @@ lappend ::testsuitelist xxx
 test_suite "coverage-wal" -description {
   Coverage tests for file wal.c.
 } -files {
-  wal.test wal2.test wal3.test wal4.test wal5.test
-  wal64k.test wal6.test wal7.test wal8.test wal9.test
-  walbak.test walbig.test walblock.test walcksum.test walcrash2.test
-  walcrash3.test walcrash4.test walcrash.test walfault.test walhook.test
-  walmode.test walnoshm.test waloverwrite.test walpersist.test 
-  walprotocol2.test walprotocol.test walro2.test walrofault.test 
-  walro.test walshared.test walslow.test walvfs.test
-  walfault2.test
-  nockpt.test
+wal2big.test wal2recover.test wal2rewrite.test
+wal2simple.test wal2snapshot.test wal2.test
+wal3.test wal4.test wal5.test
+wal64k.test wal6.test wal7.test wal8.test wal9.test
+walbak.test walbig.test walblock.test walcksum.test 
+walfault.test walhook.test walmode.test walnoshm.test
+waloverwrite.test walpersist.test walprotocol2.test
+walprotocol.test walro2.test walrofault.test walro.test
+walshared.test walslow.test wal.test
+wal2savepoint.test wal2lock.test wal2recover2.test
 
+  wal2concurrent.test
+  concurrent.test concurrent2.test concurrent3.test
+  concurrent4.test concurrent5.test concurrent6.test
+  concurrent7.test
+  concfault.test concfault2.test
+
+  walvfs.test walfault2.test nockpt.test
   snapshot2.test snapshot3.test snapshot4.test
   snapshot_fault.test snapshot.test snapshot_up.test
+  walcrash2.test walcrash3.test walcrash4.test walcrash.test
+  wal2fault.test
 } 
 
 test_suite "coverage-pager" -description {
@@ -643,6 +652,16 @@ test_suite "onefile" -description {
   rollback.test  select1.test  select2.test  select3.test
 }
 
+# Run some tests using the "unix-excl" VFS.
+#
+test_suite "unix-excl" -description {
+  Run some tests using the "unix-excl" VFS
+} -initialize {
+  set ::G(perm:sqlite3_args) [list -vfs unix-excl]
+} -files {
+  shmlock.test
+}
+
 # Run some tests using UTF-16 databases.
 #
 test_suite "utf16" -description {
@@ -1028,6 +1047,23 @@ test_suite "wal" -description {
   fts3c.test   fts3d.test   fts3e.test   fts3query.test 
 }
 
+test_suite "wal2" -description {
+  Run tests with journal_mode=WAL2
+} -initialize {
+  set ::G(savepoint6_iterations) 100
+} -shutdown {
+  unset -nocomplain ::G(savepoint6_iterations)
+} -files {
+  savepoint.test     savepoint2.test     savepoint6.test
+  trans.test         avtrans.test
+
+  fts3aa.test  fts3ab.test  fts3ac.test  fts3ad.test
+  fts3ae.test  fts3af.test  fts3ag.test  fts3ah.test
+  fts3ai.test  fts3aj.test  fts3ak.test  fts3al.test
+  fts3am.test  fts3an.test  fts3ao.test  fts3b.test
+  fts3c.test   fts3d.test   fts3e.test   fts3query.test 
+}
+
 test_suite "rtree" -description {
   All R-tree related tests. Provides coverage of source file rtree.c.
 } -files [glob -nocomplain $::testdir/../ext/rtree/*.test]

test/rdonly.test

@@ -41,7 +41,7 @@ do_test rdonly-1.1.1 {
   sqlite3_db_readonly db main
 } {0}
 
-# Changes the write version from 1 to 3.  Verify that the database
+# Changes the write version from 1 to 4.  Verify that the database
 # can be read but not written.
 #
 do_test rdonly-1.2 {
@@ -49,7 +49,7 @@ do_test rdonly-1.2 {
   hexio_get_int [hexio_read test.db 18 1]
 } 1
 do_test rdonly-1.3 {
-  hexio_write test.db 18 03
+  hexio_write test.db 18 04
   sqlite3 db test.db
   execsql {
     SELECT * FROM t1;
@@ -83,7 +83,7 @@ do_test rdonly-1.5 {
 # the database is read-only until after it is locked.
 #
 set ro_version 02
-ifcapable wal { set ro_version 03 }
+ifcapable wal { set ro_version 04 }
 do_test rdonly-1.6 {
   hexio_write test.db 18 $ro_version     ; # write-version
   hexio_write test.db 24 11223344        ; # change-counter

test/savepoint.test

@@ -28,6 +28,7 @@ do_test savepoint-1.1 {
     RELEASE sp1;
   }
 } {}
+wal_check_journal_mode savepoint-1.1
 do_test savepoint-1.2 {
   execsql {
     SAVEPOINT sp1;
@@ -805,7 +806,8 @@ do_test savepoint-11.6 {
 integrity_check savepoint-11.7
 do_test savepoint-11.8 {
   execsql { ROLLBACK }
-  execsql { PRAGMA wal_checkpoint }
+  db close
+  sqlite3 db test.db
   file size test.db
 } {8192}
 

test/savepoint6.test

@@ -15,6 +15,10 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
 proc sql {zSql} {
+  if {0 && $::debug_op} {
+    puts stderr "$zSql ;"
+    flush stderr
+  }
   uplevel db eval [list $zSql]
   #puts stderr "$zSql ;"
 }
@@ -67,11 +71,13 @@ proc x_to_y {x} {
 #   delete_rows XVALUES
 #
 proc savepoint {zName} {
+  if {$::debug_op} { puts stderr "savepoint $zName" ; flush stderr }
   catch { sql "SAVEPOINT $zName" }
   lappend ::lSavepoint [list $zName [array get ::aEntry]]
 }
 
 proc rollback {zName} {
+  if {$::debug_op} { puts stderr "rollback $zName" ; flush stderr }
   catch { sql "ROLLBACK TO $zName" }
   for {set i [expr {[llength $::lSavepoint]-1}]} {$i>=0} {incr i -1} {
     set zSavepoint [lindex $::lSavepoint $i 0]
@@ -89,6 +95,7 @@ proc rollback {zName} {
 }
 
 proc release {zName} {
+  if {$::debug_op} { puts stderr "release $zName" ; flush stderr }
   catch { sql "RELEASE $zName" }
   for {set i [expr {[llength $::lSavepoint]-1}]} {$i>=0} {incr i -1} {
     set zSavepoint [lindex $::lSavepoint $i 0]
@@ -104,6 +111,7 @@ proc release {zName} {
 }
 
 proc insert_rows {lX} {
+  if {$::debug_op} { puts stderr "insert_rows $lX" ; flush stderr }
   foreach x $lX {
     set y [x_to_y $x]
 
@@ -116,6 +124,7 @@ proc insert_rows {lX} {
 }
 
 proc delete_rows {lX} {
+  if {$::debug_op} { puts stderr "delete_rows $lX" ; flush stderr }
   foreach x $lX {
     # Update database [db]
     sql "DELETE FROM t1 WHERE x = $x"
@@ -164,6 +173,11 @@ proc random_integers {nRes nRange} {
 } 
 #-------------------------------------------------------------------------
 
+set ::debug_op 0
+proc debug_ops {} {
+  set ::debug_op 1
+}
+
 proc database_op {} {
   set i [expr int(rand()*2)] 
   if {$i==0} {
@@ -185,9 +199,6 @@ proc savepoint_op {} {
   set C [lindex $cmds [expr int(rand()*6)]]
   set N [lindex $names [expr int(rand()*5)]]
 
-  #puts stderr "   $C $N ;  "
-  #flush stderr
-
   $C $N
   return ok
 }

test/tester.tcl

@@ -612,6 +612,7 @@ proc reset_db {} {
   forcedelete test.db
   forcedelete test.db-journal
   forcedelete test.db-wal
+  forcedelete test.db-wal2
   sqlite3 db ./test.db
   set ::DB [sqlite3_connection_pointer db]
   if {[info exists ::SETUP_SQL]} {
@@ -2241,17 +2242,32 @@ proc drop_all_indexes {{db db}} {
 #     Returns true if this test should be run in WAL mode. False otherwise.
 #
 proc wal_is_wal_mode {} {
-  expr {[permutation] eq "wal"}
+  if {[permutation] eq "wal"} { return 1 }
+  if {[permutation] eq "wal2"} { return 2 }
+  return 0
 }
 proc wal_set_journal_mode {{db db}} {
-  if { [wal_is_wal_mode] } {
-    $db eval "PRAGMA journal_mode = WAL"
+  switch -- [wal_is_wal_mode] {
+    0 {
+    }
+
+    1 {
+      $db eval "PRAGMA journal_mode = WAL"
+    }
+
+    2 {
+      $db eval "PRAGMA journal_mode = WAL2"
+    }
   }
 }
 proc wal_check_journal_mode {testname {db db}} {
   if { [wal_is_wal_mode] } {
     $db eval { SELECT * FROM sqlite_master }
-    do_test $testname [list $db eval "PRAGMA main.journal_mode"] {wal}
+    set expected "wal"
+    if {[wal_is_wal_mode]==2} {
+      set expected "wal2"
+    }
+    do_test $testname [list $db eval "PRAGMA main.journal_mode"] $expected
   }
 }
 

test/threadtest3.c

@@ -1494,6 +1494,8 @@ static void dynamic_triggers(int nMs){
 #include "tt3_stress.c"
 #include "tt3_shared.c"
 
+#include "tt3_bcwal2.c"
+
 int main(int argc, char **argv){
   struct ThreadTest {
     void (*xTest)(int);   /* Routine for running this test */
@@ -1518,6 +1520,8 @@ int main(int argc, char **argv){
     { stress1,             "stress1", 10000 },
     { stress2,             "stress2", 60000 },
     { shared1,             "shared1", 10000 },
+
+    { bcwal2_1,            "bcwal2_1", 100000 },
   };
   static char *substArgv[] = { 0, "*", 0 };
   int i, iArg;

test/tt3_bcwal2.c

@@ -0,0 +1,122 @@
+/*
+** 2011-02-02
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the test program "threadtest3". Despite being a C
+** file it is not compiled separately, but included by threadtest3.c using
+** the #include directive normally used with header files.
+**
+** This file contains the implementation of test cases:
+**
+**     bcwal2_1
+*/
+
+static char *bcwal2_1_checkpointer(int iTid, void *pArg){
+  Error err = {0};                /* Error code and message */
+  Sqlite db = {0};                /* SQLite database connection */
+  int nIter = 0;
+
+  opendb(&err, &db, "test.db", 0);
+  while( !timetostop(&err) ){
+    sql_script(&err, &db, "PRAGMA wal_checkpoint;");
+    nIter++;
+  }
+  closedb(&err, &db);
+
+  print_and_free_err(&err);
+  return sqlite3_mprintf("%d iterations", nIter);
+}
+
+static char *bcwal2_1_integrity(int iTid, void *pArg){
+  Error err = {0};                /* Error code and message */
+  Sqlite db = {0};                /* SQLite database connection */
+  int nIter = 0;
+
+  opendb(&err, &db, "test.db", 0);
+  while( !timetostop(&err) ){
+    // integrity_check(&err, &db);
+    sql_script(&err, &db, "SELECT * FROM t1;");
+    nIter++;
+  }
+  closedb(&err, &db);
+
+  print_and_free_err(&err);
+  return sqlite3_mprintf("%d integrity-checks", nIter);
+}
+
+static char *bcwal2_1_writer(int iTid, void *pArg){
+  Error err = {0};                /* Error code and message */
+  Sqlite db = {0};                /* SQLite database connection */
+  int nWrite = 0;                 /* Writes so far */
+  int nBusy = 0;                  /* Busy errors so far */
+  sqlite3_mutex *pMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_APP1);
+
+  opendb(&err, &db, "test.db", 0);
+  while( !timetostop(&err) ){
+
+    sql_script(&err, &db,
+        "PRAGMA wal_autocheckpoint = 0;"
+        "BEGIN CONCURRENT;"
+        "  REPLACE INTO t1 VALUES( abs(random() % 100000), "
+        "     hex(randomblob( abs( random() % 200 ) + 50 ))"
+        "  );"
+    );
+
+    if( err.rc==SQLITE_OK ){
+      sqlite3_mutex_enter(pMutex);
+      sql_script(&err, &db, "COMMIT");
+      sqlite3_mutex_leave(pMutex);
+      if( err.rc==SQLITE_OK ){
+        nWrite++;
+      }else{
+        clear_error(&err, SQLITE_BUSY);
+        sql_script(&err, &db, "ROLLBACK");
+        nBusy++;
+      }
+
+      assert( err.rc!=SQLITE_OK || sqlite3_get_autocommit(db.db)==1 );
+    }
+  }
+  closedb(&err, &db);
+
+  print_and_free_err(&err);
+  return sqlite3_mprintf("%d successful writes, %d busy", nWrite, nBusy);
+}
+
+static void bcwal2_1(int nMs){
+  Error err = {0};
+  Sqlite db = {0};
+  Threadset threads = {0};
+
+  opendb(&err, &db, "test.db", 1);
+  sql_script(&err, &db, 
+      "PRAGMA page_size = 1024;"
+      "PRAGMA journal_mode = wal2;"
+      "CREATE TABLE t1(ii INTEGER PRIMARY KEY, tt TEXT);"
+      "CREATE INDEX t1tt ON t1(tt);"
+  );
+
+  setstoptime(&err, nMs);
+
+  launch_thread(&err, &threads, bcwal2_1_writer, 0);
+  launch_thread(&err, &threads, bcwal2_1_writer, 0);
+  launch_thread(&err, &threads, bcwal2_1_writer, 0);
+  launch_thread(&err, &threads, bcwal2_1_integrity, 0);
+  launch_thread(&err, &threads, bcwal2_1_checkpointer, 0);
+
+  join_all_threads(&err, &threads);
+
+  /* Do a final integrity-check on the db */
+  integrity_check(&err, &db);
+  closedb(&err, &db);
+
+  print_and_free_err(&err);
+}
+

test/uri.test

@@ -282,11 +282,11 @@ ifcapable wal {
       INSERT INTO t2 VALUES('x', 'y');
     }
     lsort [array names ::T1]
-  } {test.db1 test.db1-journal test.db1-wal}
+  } {test.db1 test.db1-journal test.db1-wal test.db1-wal2}
   
   do_test 5.1.2 {
     lsort [array names ::T2]
-  } {test.db2 test.db2-journal test.db2-wal}
+  } {test.db2 test.db2-journal test.db2-wal test.db2-wal2}
   db close
   
   tvfs1 delete

test/wal.test

@@ -1175,7 +1175,7 @@ foreach {tn pgsz works} {
   9  32768    1
  10  65536    1
  11 131072    0
- 11   1016    0
+ 12   1016    0
 } {
 
   if {$::SQLITE_MAX_PAGE_SIZE < $pgsz} {
@@ -1185,14 +1185,14 @@ foreach {tn pgsz works} {
   for {set pg 1} {$pg <= 3} {incr pg} {
     forcecopy testX.db test.db
     forcedelete test.db-wal
-  
+
     # Check that the database now exists and consists of three pages. And
     # that there is no associated wal file.
     #
     do_test wal-18.2.$tn.$pg.1 { file exists test.db-wal } 0
     do_test wal-18.2.$tn.$pg.2 { file exists test.db } 1
     do_test wal-18.2.$tn.$pg.3 { file size test.db } [expr 1024*3]
-  
+
     do_test wal-18.2.$tn.$pg.4 {
 
       # Create a wal file that contains a single frame (database page
@@ -1224,16 +1224,16 @@ foreach {tn pgsz works} {
       puts -nonewline $fd $framehdr
       puts -nonewline $fd $framebody
       close $fd
-  
+
       file size test.db-wal
     } [wal_file_size 1 $pgsz]
-  
+
     do_test wal-18.2.$tn.$pg.5 {
       sqlite3 db test.db
       set rc [catch { db one {PRAGMA integrity_check} } msg]
       expr { $rc!=0 || $msg!="ok" }
     } $works
-  
+ 
     db close
   }
 }

test/wal2.test

@@ -35,43 +35,6 @@ proc cond_incr_sync_count {adj} {
   }
 }
 
-proc set_tvfs_hdr {file args} {
-
-  # Set $nHdr to the number of bytes in the wal-index header:
-  set nHdr 48
-  set nInt [expr {$nHdr/4}]
-
-  if {[llength $args]>2} {
-    error {wrong # args: should be "set_tvfs_hdr fileName ?val1? ?val2?"}
-  }
-
-  set blob [tvfs shm $file]
-  if {$::tcl_platform(byteOrder)=="bigEndian"} {set fmt I} {set fmt i}
-
-  if {[llength $args]} {
-    set ia [lindex $args 0]
-    set ib $ia
-    if {[llength $args]==2} {
-      set ib [lindex $args 1]
-    }
-    binary scan $blob a[expr $nHdr*2]a* dummy tail
-    set blob [binary format ${fmt}${nInt}${fmt}${nInt}a* $ia $ib $tail]
-    tvfs shm $file $blob
-  }
-
-  binary scan $blob ${fmt}${nInt} ints
-  return $ints
-}
-
-proc incr_tvfs_hdr {file idx incrval} {
-  set ints [set_tvfs_hdr $file]
-  set v [lindex $ints $idx]
-  incr v $incrval
-  lset ints $idx $v
-  set_tvfs_hdr $file $ints
-}
-
-
 #-------------------------------------------------------------------------
 # Test case wal2-1.*:
 #

test/wal2big.test

@@ -0,0 +1,71 @@
+# 2017 September 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+
+set testprefix wal2big
+ifcapable !wal {finish_test ; return }
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1b ON t1(b);
+  CREATE INDEX t1c ON t1(c);
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_size_limit = 10000000;
+
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200000
+  )
+  INSERT INTO t1 SELECT random(), random(), random() FROM s;
+} {wal2 10000000}
+
+do_execsql_test 1.1 {
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200000
+  )
+  INSERT INTO t1 SELECT random(), random(), random() FROM s;
+}
+
+do_test 1.2 {
+  list [expr [file size test.db-wal]>10000000] \
+       [expr [file size test.db-wal2]>10000000]
+} {1 1}
+
+do_test 1.3 {
+  sqlite3 db2 test.db
+  execsql {
+    SELECT count(*) FROM t1;
+    PRAGMA integrity_check;
+  } db2
+} {400000 ok}
+
+do_test 1.4 {
+  db2 close
+  forcecopy test.db test.db2
+  forcecopy test.db-wal test.db2-wal
+  forcecopy test.db-wal2 test.db2-wal2
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT count(*) FROM t1;
+    PRAGMA integrity_check;
+  }
+} {400000 ok}
+
+finish_test

test/wal2concurrent.test

@@ -0,0 +1,164 @@
+# 2018 December 6
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set ::testprefix wal2concurrent
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+
+#-------------------------------------------------------------------------
+# Warm-body test.
+#
+foreach tn {1 2} {
+  reset_db
+  sqlite3 db2 test.db
+  do_execsql_test 1.0 {
+    PRAGMA page_size = 1024;
+    CREATE TABLE t1(x);
+    CREATE TABLE t2(y);
+    PRAGMA journal_size_limit = 5000;
+    PRAGMA journal_mode = wal2;
+  } {5000 wal2}
+  
+  do_execsql_test 1.1 {
+    INSERT INTO t1 VALUES(1);
+    BEGIN CONCURRENT;
+      INSERT INTO t1 VALUES(2);
+  } {}
+  
+  do_test 1.2 {
+    execsql { 
+      PRAGMA journal_size_limit = 5000;
+      INSERT INTO t1 VALUES(3) 
+    } db2
+    catchsql { COMMIT }
+  } {1 {database is locked}}
+  
+  do_catchsql_test 1.3 { COMMIT   } {1 {database is locked}}
+  do_catchsql_test 1.4 { ROLLBACK } {0 {}}
+  
+  do_test 1.5 {
+    list [file size test.db-wal] [file size test.db-wal2]
+  } {2128 0}
+  
+  do_execsql_test 1.6 {
+    BEGIN CONCURRENT;
+      INSERT INTO t1 VALUES(2);
+  } {}
+  
+  do_test 1.7 {
+    execsql { INSERT INTO t2 VALUES(randomblob(4000)) } db2
+    list [file size test.db-wal] [file size test.db-wal2]
+  } {7368 0}
+  
+  if {$tn==1} {
+    do_test 1.8 {
+      execsql { 
+        INSERT INTO t2 VALUES(1);
+        INSERT INTO t1 VALUES(5);
+      } db2
+      list [file size test.db-wal] [file size test.db-wal2]
+    } {7368 2128}
+
+    do_catchsql_test 1.9  { COMMIT   } {1 {database is locked}}
+    do_catchsql_test 1.10 { ROLLBACK } {0 {}}
+    db close
+    sqlite3 db test.db
+    do_execsql_test 1.11 { SELECT * FROM t1 } {1 3 5}
+    do_execsql_test 1.12 { SELECT count(*) FROM t2 } {2}
+  } else {
+    do_test 1.8 {
+      execsql { 
+        INSERT INTO t2 VALUES(1);
+      } db2
+      list [file size test.db-wal] [file size test.db-wal2]
+    } {7368 1080}
+
+    do_catchsql_test 1.9  { COMMIT   } {0 {}}
+    db close
+    sqlite3 db test.db
+    do_execsql_test 1.11 { SELECT * FROM t1 } {1 3 2}
+    do_execsql_test 1.12 { SELECT count(*) FROM t2 } {2}
+
+    do_test 1.13 {
+      list [file size test.db-wal] [file size test.db-wal2]
+    } {7368 2128}
+  }
+}
+
+do_multiclient_test tn {
+  do_test 2.$tn.1 {
+    sql1 {
+      PRAGMA auto_vacuum = OFF;
+      CREATE TABLE t1(x UNIQUE);
+      CREATE TABLE t2(x UNIQUE);
+      PRAGMA journal_mode = wal2;
+      PRAGMA journal_size_limit = 15000;
+    }
+  } {wal2 15000}
+
+  do_test 2.$tn.2 {
+    sql1 {
+      WITH s(i) AS (
+        SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<=10
+      )
+      INSERT INTO t1 SELECT randomblob(800) FROM s;
+    }
+  } {}
+
+  do_test 2.$tn.3 {
+    sql1 { DELETE FROM t1 WHERE (rowid%4)==0 }
+    list [expr [file size test.db-wal]>15000] \
+         [expr [file size test.db-wal2]>15000]
+  } {1 0}
+
+  do_test 2.$tn.4 {
+    sql1 { PRAGMA wal_checkpoint; }
+    sql1 { 
+      BEGIN CONCURRENT;
+        INSERT INTO t1 VALUES(randomblob(800));
+    }
+  } {}
+
+  do_test 2.$tn.5 {
+    sql2 { 
+      PRAGMA journal_size_limit = 15000;
+      INSERT INTO t2 VALUES(randomblob(800));
+      INSERT INTO t2 VALUES(randomblob(800));
+      INSERT INTO t2 VALUES(randomblob(800));
+      INSERT INTO t2 VALUES(randomblob(800));
+      INSERT INTO t2 VALUES(randomblob(800));
+      DELETE FROM t2;
+    }
+    list [expr [file size test.db-wal]>15000] \
+         [expr [file size test.db-wal2]>15000]
+  } {1 1}
+
+  do_test 2.$tn.6 {
+    sql1 { 
+        INSERT INTO t1 VALUES(randomblob(800));
+      COMMIT;
+      PRAGMA integrity_check;
+    }
+  } {ok}
+}
+
+
+
+finish_test
+

test/wal2fault.test

@@ -0,0 +1,52 @@
+# 2010 May 03
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+source $testdir/lock_common.tcl
+
+ifcapable !wal {finish_test ; return }
+set testprefix wal2fault
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(x,y);
+  PRAGMA journal_mode = wal2;
+  WITH s(i) AS ( SELECT 100 UNION ALL SELECT i-1 FROM s WHERE (i-1)>0 )
+  INSERT INTO t1 SELECT i, randomblob(i) FROM s;
+  WITH s(i) AS ( SELECT 100 UNION ALL SELECT i-1 FROM s WHERE (i-1)>0 )
+  INSERT INTO t1 SELECT i, randomblob(i) FROM s;
+} {wal2}
+
+do_test 1.1 {
+  expr [file size test.db-wal]>10000
+} {1}
+faultsim_save_and_close
+
+do_faultsim_test 1 -prep {
+  faultsim_restore_and_reopen
+  execsql {
+    PRAGMA journal_size_limit = 10000;
+    SELECT count(*) FROM sqlite_master;
+  }
+} -body {
+  execsql {
+    INSERT INTO t1 VALUES(1, 2);
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+finish_test

test/wal2lock.test

@@ -0,0 +1,106 @@
+# 2018 December 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+
+set testprefix wal2lock
+ifcapable !wal {finish_test ; return }
+
+db close
+testvfs tvfs 
+sqlite3 db test.db -vfs tvfs
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal2;
+  CREATE TABLE y1(y, yy);
+  CREATE INDEX y1y ON y1(y);
+  CREATE INDEX y1yy ON y1(yy);
+  INSERT INTO y1 VALUES(1, 2), (3, 4), (5, 6);
+} {wal2}
+
+tvfs script vfs_callback
+tvfs filter xShmLock
+
+set ::lock [list]
+proc vfs_callback {func file name lock} {
+  lappend ::lock $lock
+  return SQLITE_OK
+}
+
+do_execsql_test 1.1.1 {
+  SELECT * FROM y1
+} {1 2 3 4 5 6}
+do_test 1.1.2 {
+  set ::lock
+} {{4 1 lock shared} {4 1 unlock shared}}
+
+set ::bFirst 1
+proc vfs_callback {func file name lock} {
+  if {$::bFirst} {
+    set ::bFirst 0
+    return SQLITE_BUSY
+  }
+  return SQLITE_OK
+}
+do_execsql_test 1.2 {
+  SELECT * FROM y1
+} {1 2 3 4 5 6}
+
+set ::bFirst 1
+proc vfs_callback {func file name lock} {
+  if {$::bFirst} {
+    set ::bFirst 0
+    return SQLITE_IOERR
+  }
+  return SQLITE_OK
+}
+do_catchsql_test 1.3 {
+  SELECT * FROM y1
+} {1 {disk I/O error}}
+
+puts "# Warning: This next test case causes SQLite to call xSleep(1) 100 times."
+puts "# Normally this equates to a delay of roughly 10 seconds, but if SQLite"
+puts "# is built on unix without HAVE_USLEEP defined, it may be much longer."
+proc vfs_callback {func file name lock} { return SQLITE_BUSY }
+do_catchsql_test 1.4 {
+  SELECT * FROM y1
+} {1 {locking protocol}}
+proc vfs_callback {func file name lock} { return SQLITE_OK }
+
+sqlite3 db2 test.db -vfs tvfs
+set ::bFirst 1
+
+proc vfs_callback {func file name lock} {
+  if {$::bFirst} {
+    set ::bFirst 0
+    db2 eval { INSERT INTO y1 VALUES(7, 8) }
+  }
+}
+
+do_execsql_test 1.5.1 {
+  SELECT * FROM y1
+} {1 2 3 4 5 6 7 8}
+do_execsql_test 1.5.2 {
+  SELECT * FROM y1
+} {1 2 3 4 5 6 7 8}
+
+db close
+db2 close
+tvfs delete
+finish_test

test/wal2openclose.test

@@ -0,0 +1,81 @@
+# 2017 September 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+
+set testprefix wal2openclose
+ifcapable !wal {finish_test ; return }
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  PRAGMA journal_mode = wal2;
+  PRAGMA wal_autocheckpoint = 0;
+  PRAGMA journal_size_limit = 75000;
+} {wal2 0 75000}
+
+do_test 1.1 {
+  for {set ii 1} {$ii <= 200} {incr ii} {
+    execsql {
+      INSERT INTO t1 VALUES($ii, $ii, $ii);
+    }
+  }
+  expr ([file size test.db-wal2] - 75000) > 30000
+} {1}
+
+do_test 1.2 {
+  db close
+  list [file exists test.db-wal] [file exists test.db-wal2]
+} {0 0}
+
+sqlite3 db test.db
+do_execsql_test 1.3 {
+  SELECT sum(c) FROM t1
+} {20100}
+db close
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE TABLE t1(a, b, c);
+  PRAGMA journal_mode = wal2;
+  INSERT INTO t1 VALUES(1, 2, 3);
+} {wal2}
+db_save_and_close
+
+db_restore_and_reopen
+do_execsql_test 2.1 {
+  SELECT * FROM t1;
+} {1 2 3}
+
+do_test 2.2 {
+  sqlite3 db2 test.db
+  db2 eval {INSERT INTO t1 VALUES(4, 5, 6)}
+  db2 close
+} {}
+
+breakpoint
+db close
+sqlite3 db test.db
+do_execsql_test 2.2 {
+  SELECT * FROM t1;
+} {1 2 3 4 5 6}
+
+
+
+finish_test

test/wal2recover.test

@@ -0,0 +1,265 @@
+# 2018 December 13
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+
+set testprefix wal2recover
+ifcapable !wal {finish_test ; return }
+
+proc db_copy {from to} {
+  forcecopy $from $to
+  forcecopy ${from}-wal ${to}-wal
+  forcecopy ${from}-wal2 ${to}-wal2
+}
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1b ON t1(b);
+  CREATE INDEX t1c ON t1(c);
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_size_limit = 15000;
+  PRAGMA wal_autocheckpoint = 0;
+} {wal2 15000 0}
+
+do_test 1.1 {
+  for {set i 1} {$i <= 1000} {incr i} {
+    execsql { INSERT INTO t1 VALUES(random(), random(), random()) }
+    db_copy test.db test.db2
+    sqlite3 db2 test.db
+    set res [execsql {
+      SELECT count(*) FROM t1;
+      PRAGMA integrity_check;
+    } db2]
+    db2 close
+    if {$res != [list $i ok]} {
+      error "failure on iteration $i"
+    }
+  }
+  set {} {}
+} {}
+
+#--------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE TABLE t1(x UNIQUE);
+  CREATE TABLE t2(x UNIQUE);
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_size_limit = 10000;
+  PRAGMA wal_autocheckpoint = 0;
+  BEGIN;
+    INSERT INTO t1 VALUES(randomblob(4000));
+    INSERT INTO t1 VALUES(randomblob(4000));
+    INSERT INTO t1 VALUES(randomblob(4000));
+  COMMIT;
+  BEGIN;
+    INSERT INTO t2 VALUES(randomblob(4000));
+    INSERT INTO t2 VALUES(randomblob(4000));
+    INSERT INTO t2 VALUES(randomblob(4000));
+  COMMIT;
+} {wal2 10000 0}
+do_test 2.0.1 {
+  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
+} {5120 28328 28328}
+
+# Test recovery with both wal files intact.
+#
+do_test 2.1 {
+  db_copy test.db test.db2
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT count(*) FROM t1;
+    SELECT count(*) FROM t2;
+    PRAGMA integrity_check;
+  } db2
+} {3 3 ok}
+
+do_test 2.2 {
+  db2 close
+  db_copy test.db test.db2
+  hexio_write test.db2-wal 16 12345678
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT count(*) FROM t1;
+    SELECT count(*) FROM t2;
+  } db2
+} {0 3}
+
+do_test 2.3 {
+  db2 close
+  db_copy test.db test.db2
+  hexio_write test.db2-wal2 16 12345678
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT count(*) FROM t1;
+    SELECT count(*) FROM t2;
+    PRAGMA integrity_check;
+  } db2
+} {3 0 ok}
+
+do_test 2.4 {
+  db2 close
+  db_copy test.db test.db2
+  forcecopy test.db-wal test.db2-wal2
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT count(*) FROM t1;
+    SELECT count(*) FROM t2;
+    PRAGMA integrity_check;
+  } db2
+} {3 0 ok}
+
+do_test 2.5 {
+  db2 close
+  db_copy test.db test.db2
+  forcecopy test.db-wal  test.db2-wal2
+  forcecopy test.db-wal2  test.db2-wal
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT count(*) FROM t1;
+    SELECT count(*) FROM t2;
+    PRAGMA integrity_check;
+  } db2
+} {3 3 ok}
+
+do_test 2.6 {
+  db2 close
+  db_copy test.db test.db2
+  forcecopy test.db-wal test.db2-wal2
+  close [open test.db-wal w]
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT count(*) FROM t1;
+    SELECT count(*) FROM t2;
+    PRAGMA integrity_check;
+  } db2
+} {3 0 ok}
+
+do_test 2.7 {
+  db2 close
+  db_copy test.db test.db2
+  forcedelete test.db2-wal
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT count(*) FROM t1;
+    SELECT count(*) FROM t2;
+    PRAGMA integrity_check;
+  } db2
+} {0 0 ok}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE t1(a TEXT, b TEXT, c TEXT);
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1b ON t1(b);
+  CREATE INDEX t1c ON t1(c);
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_size_limit = 10000;
+  PRAGMA wal_autocheckpoint = 0;
+  PRAGMA cache_size = 5;
+} {wal2 10000 0}
+
+do_execsql_test 3.1 {
+  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200)
+  INSERT INTO t1 SELECT i, i, i FROM s;
+
+  INSERT INTO t1 VALUES(201, 201, 201);
+} {}
+
+do_test 3.2 {
+  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
+} {5120 15752 4224}
+
+do_test 3.3 {
+  forcecopy test.db test.db2
+  forcecopy test.db-wal test.db2-wal
+  forcecopy test.db-wal2 test.db2-wal2
+  sqlite3 db2 test.db2
+  execsql {
+    PRAGMA journal_size_limit = 10000;
+    PRAGMA wal_autocheckpoint = 0;
+    PRAGMA cache_size = 5;
+    BEGIN;
+      WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200)
+      INSERT INTO t1 SELECT i, i, i FROM s;
+  } db2
+  list [file size test.db2] [file size test.db2-wal] [file size test.db2-wal2]
+} {5120 15752 23088}
+
+do_test 3.4 {
+  set fd [open test.db2-shm]
+  fconfigure $fd -encoding binary -translation binary
+  set data [read $fd]
+  close $fd
+
+  set fd [open test.db-shm w]
+  fconfigure $fd -encoding binary -translation binary
+  puts -nonewline $fd $data
+  close $fd
+
+  execsql {
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 10)
+      INSERT INTO t1 SELECT i, i, i FROM s;
+    SELECT count(*) FROM t1;
+    PRAGMA integrity_check;
+  }
+} {211 ok}
+
+do_test 3.5 {
+  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
+} {5120 15752 18896}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 4.0 {
+  PRAGMA journal_mode = wal2;
+  CREATE TABLE xyz(x, y, z);
+  INSERT INTO xyz VALUES('x', 'y', 'z');
+} {wal2}
+db close
+do_test 4.1 {
+  close [open test.db-wal w]
+  file mkdir test.db-wal2
+  sqlite3 db test.db
+  catchsql { SELECT * FROM xyz }
+} {1 {unable to open database file}}
+db close
+file delete test.db-wal2
+
+do_test 4.2 {
+  sqlite3 db test.db
+  execsql { 
+    INSERT INTO xyz VALUES('a', 'b', 'c');
+  }
+  forcecopy test.db test.db2
+  forcecopy test.db-wal test.db2-wal
+  forcedelete test.db2-wal2
+  file mkdir test.db2-wal2
+  sqlite3 db2 test.db2
+  catchsql { SELECT * FROM xyz } db2
+} {1 {unable to open database file}}
+db2 close
+file delete test.db2-wal2
+
+
+finish_test
+

test/wal2recover2.test

@@ -0,0 +1,312 @@
+# 2018 December 13
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+
+set testprefix wal2recover2
+ifcapable !wal {finish_test ; return }
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(x);
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 )
+    INSERT INTO t1 SELECT i FROM s;
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 )
+    INSERT INTO t2 SELECT i FROM s;
+
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_size_limit = 10000;
+} {wal2 10000}
+
+set ::L 1125750
+set ::M 1126500
+set ::H 1127250
+
+do_execsql_test 1.1 {
+  UPDATE t1 SET x=x+1;
+  UPDATE t2 SET x=x+1 WHERE rowid<=750;
+
+  SELECT sum(x) FROM t1;
+  SELECT sum(x) FROM t2;
+} [list $H $M]
+
+do_test 1.2 {
+  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
+} {31744 14704 7368}
+
+proc cksum {zIn data} {
+  if {[string length $zIn]==0} {
+    set s0 0
+    set s1 0
+  } else {
+    set s0 [hexio_get_int [string range $zIn 0 7]]
+    set s1 [hexio_get_int [string range $zIn 8 15]]
+  }
+  set n [expr [string length $data] / 8]
+
+  for {set i 0} {$i < $n} {incr i 2} {
+    set x0 [hexio_get_int -l [string range $data [expr $i*8]   [expr $i*8+7]]]
+    set x1 [hexio_get_int -l [string range $data [expr $i*8+8] [expr $i*8+8+7]]]
+
+    set s0 [expr ($s0 + $x0 + $s1) & 0xFFFFFFFF]
+    set s1 [expr ($s1 + $x1 + $s0) & 0xFFFFFFFF]
+  }
+
+  return "[hexio_render_int32 $s0][hexio_render_int32 $s1]"
+}
+
+proc fix_wal_cksums {file} {
+  # Fix the checksum on the wal header.
+  set data [hexio_read $file 0 32]
+  set cksum [cksum {} [string range $data 0 47]]
+  set salt [hexio_read $file 16 8]
+  hexio_write $file 24 $cksum
+
+  # Fix the checksums for all pages in the wal file.
+  set pgsz [hexio_get_int [hexio_read $file 8 4]]
+  set sz [file size $file]
+  for {set off 32} {$off < $sz} {incr off [expr $pgsz+24]} {
+    set e [hexio_read $file $off 8]
+    set cksum [cksum $cksum $e]
+
+    set p [hexio_read $file [expr $off+24] $pgsz]
+    set cksum [cksum $cksum $p]
+
+    hexio_write $file [expr $off+8] $salt
+    hexio_write $file [expr $off+16] $cksum
+  }
+}
+
+proc wal_incr_hdrfield {file field} {
+  switch -- $field {
+    nCkpt { set offset 12 }
+    salt0 { set offset 16 }
+    salt1 { set offset 20 }
+    default {
+      error "unknown field $field - should be \"nCkpt\", \"salt0\" or \"salt1\""
+    }
+  }
+
+  # Increment the value in the wal header.
+  set v [hexio_get_int [hexio_read $file $offset 4]]
+  incr v
+  hexio_write $file $offset [hexio_render_int32 $v]
+  
+  # Fix various checksums
+  fix_wal_cksums $file
+}
+
+proc wal_set_nckpt {file val} {
+  # Increment the value in the wal header.
+  hexio_write $file 12 [hexio_render_int32 $val]
+  
+  # Fix various checksums
+  fix_wal_cksums $file
+}
+
+proc wal_set_follow {file prevfile} {
+  set pgsz [hexio_get_int [hexio_read $prevfile 8 4]]
+  set sz [file size $prevfile]
+  set cksum [hexio_read $prevfile [expr $sz-$pgsz-8] 8]
+
+  hexio_write $file 16 $cksum
+  fix_wal_cksums $file
+}
+
+foreach {tn file field} {
+  1 test.db2-wal    salt0
+  2 test.db2-wal    salt1
+  3 test.db2-wal    nCkpt
+  4 test.db2-wal2   salt0
+  5 test.db2-wal2   salt1
+  6 test.db2-wal2   nCkpt
+} {
+  do_test 1.3.$tn {
+    forcecopy test.db test.db2
+    forcecopy test.db-wal test.db2-wal
+    forcecopy test.db-wal2 test.db2-wal2
+    wal_incr_hdrfield $file $field
+    sqlite3 db2 test.db2
+    execsql {
+      SELECT sum(x) FROM t1;
+      SELECT sum(x) FROM t2;
+    } db2
+  } [list $H $L]
+  db2 close
+}
+
+do_test 1.4 {
+  forcecopy test.db test.db2
+  forcecopy test.db-wal2 test.db2-wal
+  forcedelete test.db2-wal2
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT sum(x) FROM t1;
+    SELECT sum(x) FROM t2;
+  } db2
+} [list $L $M]
+
+do_test 1.5 {
+  forcecopy test.db test.db2
+  forcecopy test.db-wal2 test.db2-wal
+  forcecopy test.db-wal test.db2-wal2
+  sqlite3 db2 test.db2
+  execsql {
+    SELECT sum(x) FROM t1;
+    SELECT sum(x) FROM t2;
+  } db2
+} [list $H $M]
+
+foreach {tn file field} {
+  1 test.db2-wal    salt0
+  2 test.db2-wal    salt1
+  3 test.db2-wal2   salt0
+  4 test.db2-wal2   salt1
+} {
+  do_test 1.6.$tn {
+    forcecopy test.db test.db2
+    forcecopy test.db-wal2 test.db2-wal
+    forcecopy test.db-wal test.db2-wal2
+    wal_incr_hdrfield $file $field
+    sqlite3 db2 test.db2
+    execsql {
+      SELECT sum(x) FROM t1;
+      SELECT sum(x) FROM t2;
+    } db2
+  } [list $H $L]
+  db2 close
+}
+
+foreach {tn nCkpt1 nCkpt2 res} [list \
+  1   2 1   "$H $M"                  \
+  2   2 2   "$L $M"                  \
+  3   3 1   "$H $L"                  \
+  4   15 14 "$H $M"                  \
+  5   0 15  "$H $M"                  \
+  6   1 15  "$L $M"                  \
+] {
+  do_test 1.7.$tn {
+    forcecopy test.db test.db2
+    forcecopy test.db-wal2 test.db2-wal
+    forcecopy test.db-wal test.db2-wal2
+
+    wal_set_nckpt test.db2-wal2 $nCkpt2
+    wal_set_nckpt test.db2-wal  $nCkpt1
+    wal_set_follow test.db2-wal test.db2-wal2
+
+
+    sqlite3 db2 test.db2
+    execsql {
+      SELECT sum(x) FROM t1;
+      SELECT sum(x) FROM t2;
+    } db2
+  } $res
+  db2 close
+}
+
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 1.8.1 {
+  PRAGMA autovacuum = 0;
+  PRAGMA page_size = 4096;
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(x);
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 )
+    INSERT INTO t1 SELECT i FROM s;
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 )
+    INSERT INTO t2 SELECT i FROM s;
+
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_size_limit = 10000;
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 )
+    INSERT INTO t2 SELECT i FROM s;
+} {wal2 10000}
+
+do_test 1.8.2 {
+  list [file size test.db-wal] [file size test.db-wal2]
+} {24752 0}
+
+do_execsql_test 1.8.3 { PRAGMA user_version = 123 }
+do_test 1.8.4 {
+  list [file size test.db-wal] [file size test.db-wal2]
+} {24752 4152}
+
+do_test 1.8.5 {
+  hexio_write test.db-wal2 [expr 56+16] 0400
+  fix_wal_cksums test.db-wal2
+} {}
+
+do_test 1.8.6 {
+  forcecopy test.db test.db2
+  forcecopy test.db-wal test.db2-wal
+  forcecopy test.db-wal2 test.db2-wal2
+  sqlite3 db2 test.db2
+  catchsql { SELECT * FROM sqlite_master } db2
+} {1 {malformed database schema (?)}}
+db2 close
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1b ON t1(b);
+  CREATE INDEX t1c ON t1(c);
+  PRAGMA journal_mode = wal2;
+
+  INSERT INTO t1 VALUES(randomblob(50), randomblob(50), randomblob(50));
+  INSERT INTO t1 VALUES(randomblob(50), randomblob(50), randomblob(50));
+  INSERT INTO t1 VALUES(randomblob(50), randomblob(50), randomblob(50));
+  PRAGMA journal_size_limit = 5000;
+  INSERT INTO t1 VALUES(randomblob(50), randomblob(50), randomblob(50));
+  INSERT INTO t1 VALUES(randomblob(50), randomblob(50), randomblob(50));
+  INSERT INTO t1 VALUES(randomblob(50), randomblob(50), randomblob(50));
+  INSERT INTO t1 VALUES(randomblob(50), randomblob(50), randomblob(50));
+  INSERT INTO t1 VALUES(randomblob(50), randomblob(50), randomblob(50));
+} {wal2 5000}
+
+do_test 2.1 {
+  forcecopy test.db test.db2
+  forcecopy test.db-wal2 test.db2-wal
+  forcecopy test.db-wal test.db2-wal2
+  
+  hexio_write test.db2-wal 5000 1234567890
+} {5}
+
+do_test 2.2 {
+  sqlite3 db2 test.db2
+  breakpoint
+  execsql {
+    SELECT count(*) FROM t1;
+    PRAGMA integrity_check
+  } db2
+} {4 ok}
+
+do_test 2.3 {
+  execsql {
+    INSERT INTO t1 VALUES(randomblob(50), randomblob(50), randomblob(50));
+    SELECT count(*) FROM t1;
+    PRAGMA integrity_check
+  } db2
+} {5 ok}
+
+
+finish_test

test/wal2recover3.test

@@ -0,0 +1,52 @@
+# 2022 June 28
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+
+set testprefix wal2recover3
+ifcapable !wal {finish_test ; return }
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(x);
+  PRAGMA journal_mode = wal2;
+  PRAGMA wal_autocheckpoint = 0;
+  PRAGMA journal_size_limit = 10000;
+} {wal2 0 10000}
+
+do_execsql_test 1.1 {
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 )
+    INSERT INTO t1 SELECT i FROM s;
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 )
+    INSERT INTO t2 SELECT i FROM s;
+}
+
+db_save_and_close
+set fd [open sv_test.db-wal2 r+]
+seek $fd 4000
+puts -nonewline $fd 0
+close $fd
+
+db_restore_and_reopen
+do_execsql_test 1.2 {
+  SELECT sql FROM sqlite_schema;
+} {{CREATE TABLE t1(x)} {CREATE TABLE t2(x)}}
+
+finish_test
+

test/wal2rewrite.test

@@ -0,0 +1,92 @@
+# 2017 September 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+
+set testprefix wal2rewrite
+ifcapable !wal {finish_test ; return }
+
+proc filesize {filename} {
+  if {[file exists $filename]} {
+    return [file size $filename]
+  }
+  return 0
+}
+
+foreach {tn jrnlmode} {
+  1 wal
+  2 wal2
+} {
+  reset_db
+  execsql "PRAGMA journal_mode = $jrnlmode"
+  do_execsql_test $tn.1 {
+    PRAGMA journal_size_limit = 10000;
+    PRAGMA cache_size = 5;
+    PRAGMA wal_autocheckpoint = 10;
+  
+    CREATE TABLE t1(a INTEGER PRIMARY KEY, b INTEGER, c BLOB);
+    CREATE INDEX t1b ON t1(b);
+    CREATE INDEX t1c ON t1(c);
+  
+    WITH s(i) AS (
+      SELECT 1 UNION SELECT i+1 FROM s WHERE i<10
+    )
+    INSERT INTO t1 SELECT i, i, randomblob(800) FROM s;
+  } {10000 10}
+  
+  for {set i 0} {$i < 4} {incr i} {
+    do_execsql_test $tn.$i.1 {
+      UPDATE t1 SET c=randomblob(800) WHERE (b%10)==5 AND ($i%2)
+    }
+    do_execsql_test $tn.$i.2 {
+      BEGIN;
+      UPDATE t1 SET b=b+10, c=randomblob(800);
+      UPDATE t1 SET b=b+10, c=randomblob(800);
+      UPDATE t1 SET b=b+10, c=randomblob(800);
+      UPDATE t1 SET b=b+10, c=randomblob(800);
+      UPDATE t1 SET b=b+10, c=randomblob(800);
+      UPDATE t1 SET b=b+10, c=randomblob(800);
+      UPDATE t1 SET b=b+10, c=randomblob(800);
+      UPDATE t1 SET b=b+10, c=randomblob(800);
+      UPDATE t1 SET b=b+10, c=randomblob(800);
+      UPDATE t1 SET b=b+10, c=randomblob(800);
+    }
+    execsql COMMIT
+
+    do_test $tn.$i.3 { expr [filesize test.db-wal]  < 100000 } 1
+    do_test $tn.$i.4 { expr [filesize test.db-wal2] < 100000 } 1
+
+    set sum [db eval {SELECT sum(b), md5sum(c) FROM t1}]
+
+    do_test $tn.$i.5 {
+      foreach f [glob -nocomplain test.db2*] {forcedelete $f}
+      foreach f [glob -nocomplain test.db*] {
+        forcecopy $f [string map {test.db test.db2} $f]
+      }
+
+      sqlite3 db2 test.db2
+      db2 eval {SELECT sum(b), md5sum(c) FROM t1}
+    } $sum
+    db2 close
+  }
+}
+    
+
+
+finish_test

test/wal2rollback.test

@@ -0,0 +1,62 @@
+# 2017 September 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+
+set testprefix wal2rollback
+ifcapable !wal {finish_test ; return }
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE TABLE t2(a, b, c);
+  CREATE INDEX i1 ON t1(a);
+  CREATE INDEX i2 ON t1(b);
+  PRAGMA journal_mode = wal2;
+  PRAGMA cache_size = 5;
+  PRAGMA journal_size_limit = 10000;
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s LIMIT 1000
+  )
+  INSERT INTO t1 SELECT i, i, randomblob(200) FROM s;
+} {wal2 10000}
+
+do_test 1.1 {
+  expr [file size test.db-wal] > 10000
+} 1
+
+do_test 1.2 {
+  execsql {
+    BEGIN;
+      UPDATE t1 SET b=b+1;
+      INSERT INTO t2 VALUES(1,2,3);
+  }
+  expr [file size test.db-wal2] > 10000
+} {1}
+
+breakpoint
+do_execsql_test 1.3 {
+  ROLLBACK;
+  SELECT * FROM t2;
+  SELECT count(*) FROM t1 WHERE a=b;
+  PRAGMA integrity_check;
+} {1000 ok}
+
+
+
+finish_test

test/wal2savepoint.test

@@ -0,0 +1,74 @@
+# 2018 December 13
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+
+set testprefix wal2savepoint
+ifcapable !wal {finish_test ; return }
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1b ON t1(b);
+  CREATE INDEX t1c ON t1(c);
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_size_limit = 15000;
+  PRAGMA wal_autocheckpoint = 0;
+  PRAGMA cache_size = 5;
+} {wal2 15000 0}
+
+do_execsql_test 1.1 {
+  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200)
+  INSERT INTO t1 SELECT random(), random(), random() FROM s;
+} {}
+
+do_test 1.2 {
+  list [file size test.db] [file size test.db-wal2] \
+       [expr [file size test.db-wal]>20000]
+} {5120 0 1}
+
+do_execsql_test 1.3 {
+  BEGIN;
+    SAVEPOINT abc;
+      WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 100)
+      INSERT INTO t1 SELECT random(), random(), random() FROM s;
+    ROLLBACK TO abc;
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 10)
+    INSERT INTO t1 SELECT random(), random(), random() FROM s;
+  COMMIT;
+  SELECT count(*) FROM t1;
+  PRAGMA integrity_check;
+} {210 ok}
+
+do_execsql_test 1.4 {
+  BEGIN;
+    SAVEPOINT abc;
+      WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 100)
+      INSERT INTO t1 SELECT random(), random(), random() FROM s;
+    ROLLBACK TO abc;
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 10)
+    INSERT INTO t1 SELECT random(), random(), random() FROM s;
+  COMMIT;
+  SELECT count(*) FROM t1;
+  PRAGMA integrity_check;
+} {220 ok}
+
+
+finish_test
+

test/wal2simple.test

@@ -0,0 +1,475 @@
+# 2017 September 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+
+set testprefix wal2simple
+ifcapable !wal {finish_test ; return }
+
+#-------------------------------------------------------------------------
+# The following tests verify that a client can switch in and out of wal
+# and wal2 mode. But that it is not possible to change directly from wal
+# to wal2, or from wal2 to wal mode.
+#
+do_execsql_test 1.1.0 {
+  PRAGMA journal_mode = wal2
+} {wal2}
+execsql { SELECT * FROM sqlite_master} 
+do_execsql_test 1.x {
+  PRAGMA journal_mode;
+  PRAGMA main.journal_mode;
+} {wal2 wal2}
+db close
+do_test 1.1.1 { file size test.db } {1024}
+do_test 1.1.2 { hexio_read test.db 18 2 } 0303
+
+sqlite3 db test.db
+do_execsql_test 1.2.0 {
+  SELECT * FROM sqlite_master;
+  PRAGMA journal_mode = delete;
+} {delete}
+db close
+do_test 1.2.1 { file size test.db } {1024}
+do_test 1.2.2 { hexio_read test.db 18 2 } 0101
+
+sqlite3 db test.db
+do_execsql_test 1.3.0 {
+  SELECT * FROM sqlite_master;
+  PRAGMA journal_mode = wal;
+} {wal}
+db close
+do_test 1.3.1 { file size test.db } {1024}
+do_test 1.3.2 { hexio_read test.db 18 2 } 0202
+
+sqlite3 db test.db
+do_catchsql_test 1.4.0 {
+  PRAGMA journal_mode = wal2;
+} {1 {cannot change from wal to wal2 mode}}
+do_execsql_test 1.4.1 {
+  PRAGMA journal_mode = wal;
+  PRAGMA journal_mode = delete;
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_mode = wal2;
+} {wal delete wal2 wal2}
+do_catchsql_test 1.4.2 {
+  PRAGMA journal_mode = wal;
+} {1 {cannot change from wal2 to wal mode}}
+db close
+do_test 1.4.3 { hexio_read test.db 18 2 } 0303
+
+#-------------------------------------------------------------------------
+# Test that recovery in wal2 mode works.
+#
+forcedelete test.db test.db-wal test.db-wal2
+reset_db
+do_execsql_test 2.0 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_size_limit = 5000;
+} {wal2 5000}
+
+proc wal_hook {DB nm nFrame} { $DB eval { PRAGMA wal_checkpoint } }
+db wal_hook {wal_hook db}
+
+for {set i 1} {$i <= 200} {incr i} {
+  execsql { INSERT INTO t1 VALUES(NULL, randomblob(100)) }
+  set res [db eval { SELECT sum(a), md5sum(b) FROM t1 }]
+
+  do_test 2.1.$i {
+    foreach f [glob -nocomplain test.db2*] { forcedelete $f }
+    forcecopy test.db      test.db2
+    forcecopy test.db-wal  test.db2-wal
+    forcecopy test.db-wal2 test.db2-wal2
+
+    sqlite3 db2 test.db2
+    db2 eval { SELECT sum(a), md5sum(b) FROM t1 }
+  } $res
+
+  db2 close
+}
+
+#-------------------------------------------------------------------------
+
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE t1(x BLOB, y INTEGER PRIMARY KEY);
+  CREATE INDEX i1 ON t1(x);
+  PRAGMA cache_size = 5;
+  PRAGMA journal_mode = wal2;
+} {wal2}
+
+do_test 3.1 {
+  execsql BEGIN
+  for {set i 1} {$i < 1000} {incr i} {
+    execsql { INSERT INTO t1 VALUES(randomblob(800), $i) }
+  }
+  execsql COMMIT
+} {}
+
+do_execsql_test 3.2 {
+  PRAGMA integrity_check;
+} {ok}
+
+#-------------------------------------------------------------------------
+catch { db close }
+foreach f [glob -nocomplain test.db*] { forcedelete $f }
+reset_db
+do_execsql_test 4.0 {
+  CREATE TABLE t1(x, y);
+  PRAGMA journal_mode = wal2;
+} {wal2}
+
+do_execsql_test 4.1 {
+  SELECT * FROM t1;
+} {}
+
+do_execsql_test 4.2 {
+  INSERT INTO t1 VALUES(1, 2);
+} {}
+
+do_execsql_test 4.3 {
+  SELECT * FROM t1;
+} {1 2}
+
+do_test 4.4 {
+  sqlite3 db2 test.db
+  execsql { SELECT * FROM t1 } db2
+} {1 2}
+
+do_test 4.5 {
+  lsort [glob test.db*]
+} {test.db test.db-shm test.db-wal test.db-wal2}
+
+do_test 4.6 {
+  db close
+  db2 close
+  sqlite3 db test.db
+  execsql { SELECT * FROM t1 }
+} {1 2}
+
+do_execsql_test 4.7 {
+  PRAGMA journal_size_limit = 4000;
+  INSERT INTO t1 VALUES(3, 4);
+  INSERT INTO t1 VALUES(5, 6);
+  INSERT INTO t1 VALUES(7, 8);
+  INSERT INTO t1 VALUES(9, 10);
+  INSERT INTO t1 VALUES(11, 12);
+  INSERT INTO t1 VALUES(13, 14);
+  INSERT INTO t1 VALUES(15, 16);
+  INSERT INTO t1 VALUES(17, 18);
+  SELECT * FROM t1;
+} {4000 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18}
+
+do_test 4.8 {
+  sqlite3 db2 test.db
+  execsql { SELECT * FROM t1 } db2
+} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18}
+
+do_test 4.9 {
+  db close
+  db2 close
+  lsort [glob test.db*]
+} {test.db}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 5.0 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
+  CREATE INDEX i1 ON t1(b, c);
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_size_limit = 4000;
+} {wal2 4000}
+
+proc wal_hook {DB nm nFrame} {
+  $DB eval { PRAGMA wal_checkpoint }
+}
+db wal_hook [list wal_hook db]
+
+
+foreach js {4000 8000 12000} {
+  foreach NROW [list 100 200 300 400 500 600 1000] {
+    do_test 5.$js.$NROW.1 {
+      db eval "DELETE FROM t1"
+      db eval "PRAGMA journal_size_limit = $js"
+      set nTotal 0
+      for {set i 0} {$i < $NROW} {incr i} {
+        db eval { INSERT INTO t1 VALUES($i, $i, randomblob(abs(random()%50))) }
+        incr nTotal $i
+      }
+      set {} {}
+    } {}
+
+    do_test 5.$js.$NROW.2 {
+      sqlite3 db2 test.db
+      db2 eval { 
+        PRAGMA integrity_check;
+        SELECT count(*), sum(b) FROM t1;
+      }
+    } [list ok $NROW $nTotal]
+
+    db2 close
+  }
+}
+
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 6.0 {
+  CREATE TABLE tx(x);
+  PRAGMA journal_mode = wal2;
+  PRAGMA journal_size_limit = 3500;
+} {wal2 3500}
+
+do_test 6.1 {
+  for {set i 0} {$i < 10} {incr i} {
+    execsql "CREATE TABLE t$i (x);"
+  }
+} {}
+
+do_test 6.2.1 {
+  foreach f [glob -nocomplain test.db2*] { forcedelete $f }
+  forcecopy test.db-wal2 test.db2-wal2
+  sqlite3 db2 test.db2
+  db2 eval { SELECT * FROM sqlite_master }
+} {}
+do_test 6.2.2 {
+  db2 eval {
+    PRAGMA journal_mode = wal2;
+    SELECT * FROM sqlite_master;
+  }
+} {wal2}
+
+do_test 6.3.1 {
+  db2 close
+  foreach f [glob -nocomplain test.db2*] { forcedelete $f }
+  forcecopy test.db-wal2 test.db2-wal2
+  forcecopy test.db test.db2
+  sqlite3 db2 test.db2
+  db2 eval { SELECT * FROM sqlite_master }
+} {table tx tx 2 {CREATE TABLE tx(x)}}
+do_test 6.3.2 {
+  db2 eval {
+    PRAGMA journal_mode = wal2;
+    SELECT * FROM sqlite_master;
+  }
+} {wal2 table tx tx 2 {CREATE TABLE tx(x)}}
+
+do_test 6.4.1 {
+  db2 close
+  foreach f [glob -nocomplain test.db2*] { forcedelete $f }
+  forcecopy test.db-wal2 test.db2-wal2
+  forcecopy test.db-wal test.db2-wal
+  sqlite3 db2 test.db2
+  db2 eval { SELECT * FROM sqlite_master }
+} {}
+do_test 6.4.2 {
+  db2 eval {
+    PRAGMA journal_mode = wal2;
+    SELECT * FROM sqlite_master;
+  }
+} {wal2}
+db2 close
+
+#-------------------------------------------------------------------------
+reset_db
+sqlite3 db2 test.db
+do_execsql_test 7.0 {
+  PRAGMA journal_size_limit = 10000;
+  PRAGMA journal_mode = wal2;
+  PRAGMA wal_autocheckpoint = 0;
+  BEGIN;
+    CREATE TABLE t1(a);
+    INSERT INTO t1 VALUES( randomblob(8000) );
+  COMMIT;
+} {10000 wal2 0}
+
+do_test 7.1 {
+  list [file size test.db-wal] [file size test.db-wal2]
+} {9464 0}
+
+# Connection db2 is holding a PART1 lock. 
+#
+#   7.2.2: Test that the PART1 does not prevent db from switching to the
+#          other wal file.
+#
+#   7.2.3: Test that the PART1 does prevent a checkpoint of test.db-wal.
+#
+#   7.2.4: Test that after the PART1 is released the checkpoint is possible.
+#
+do_test 7.2.1 {
+  execsql {
+    BEGIN;
+      SELECT count(*) FROM t1;
+  } db2
+} {1}
+do_test 7.2.2 {
+  execsql {
+    INSERT INTO t1 VALUES( randomblob(800) );
+    INSERT INTO t1 VALUES( randomblob(800) );
+  }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {13656 3176 1024}
+do_test 7.2.3 {
+  execsql { PRAGMA wal_checkpoint }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {13656 3176 1024}
+do_test 7.2.4 {
+  execsql { END } db2
+  execsql { PRAGMA wal_checkpoint }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {13656 3176 11264}
+
+# Connection db2 is holding a PART2_FULL1 lock. 
+#
+#   7.3.2: Test that the lock does not prevent checkpointing.
+#
+#   7.3.3: Test that the lock does prevent the writer from overwriting 
+#          test.db-wal.
+#
+#   7.3.4: Test that after the PART2_FULL1 is released the writer can
+#          switch wal files and overwrite test.db-wal
+#
+db close
+db2 close
+sqlite3 db test.db
+sqlite3 db2 test.db
+do_test 7.3.1 {
+  execsql {
+    PRAGMA wal_autocheckpoint = 0;
+    PRAGMA journal_size_limit = 10000;
+    INSERT INTO t1 VALUES(randomblob(10000));
+    INSERT INTO t1 VALUES(randomblob(500));
+  }
+  execsql {
+    BEGIN;
+      SELECT count(*) FROM t1;
+  } db2
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 3176 12288}
+do_test 7.3.2 {
+  execsql { PRAGMA wal_checkpoint }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 3176 22528}
+do_test 7.3.3 {
+  execsql { 
+    INSERT INTO t1 VALUES(randomblob(10000));
+    INSERT INTO t1 VALUES(randomblob(500));
+  }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 18896 22528}
+do_test 7.3.4 {
+  execsql END db2
+  execsql { INSERT INTO t1 VALUES(randomblob(5000)); }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 18896 22528}
+
+# Connection db2 is holding a PART2 lock. 
+#
+#   7.4.2: Test that the lock does not prevent writer switching to test.db-wal.
+#
+#   7.3.3: Test that the lock does prevent checkpointing of test.db-wal2.
+#
+#   7.3.4: Test that after the PART2 is released test.db-wal2 can be
+#          checkpointed.
+#
+db close
+db2 close
+breakpoint
+sqlite3 db test.db
+sqlite3 db2 test.db
+do_test 7.4.1 {
+  execsql {
+    PRAGMA wal_autocheckpoint = 0;
+    PRAGMA journal_size_limit = 10000;
+    INSERT INTO t1 VALUES(randomblob(10000));
+    INSERT INTO t1 VALUES(randomblob(10000));
+    PRAGMA wal_checkpoint;
+  }
+  execsql {
+    BEGIN;
+      SELECT count(*) FROM t1;
+  } db2
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 12608 50176}
+do_test 7.4.2 {
+  execsql { 
+    INSERT INTO t1 VALUES(randomblob(5000));
+  }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 12608 50176}
+do_test 7.4.3 {
+  execsql { PRAGMA wal_checkpoint }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 12608 50176}
+do_test 7.4.4 {
+  execsql END db2
+  execsql { PRAGMA wal_checkpoint }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 12608 60416}
+
+# Connection db2 is holding a PART1_FULL2 lock. 
+#
+#   7.5.2: Test that the lock does not prevent a checkpoint of test.db-wal2.
+#
+#   7.5.3: Test that the lock does prevent the writer from overwriting
+#          test.db-wal2.
+#
+#   7.5.4: Test that after the PART1_FULL2 lock is released, the writer
+#          can switch to test.db-wal2.
+#
+db close
+db2 close
+sqlite3 db test.db
+sqlite3 db2 test.db
+do_test 7.5.1 {
+  execsql {
+    PRAGMA wal_autocheckpoint = 0;
+    PRAGMA journal_size_limit = 10000;
+    INSERT INTO t1 VALUES(randomblob(10000));
+    INSERT INTO t1 VALUES(randomblob(10000));
+    PRAGMA wal_checkpoint;
+    INSERT INTO t1 VALUES(randomblob(5000));
+  }
+  execsql {
+    BEGIN;
+      SELECT count(*) FROM t1;
+  } db2
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 12608 76800}
+do_test 7.5.2 {
+  execsql { PRAGMA wal_checkpoint }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {12608 12608 87040}
+do_test 7.5.3.1 {
+  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {14704 12608 87040}
+do_test 7.5.3.2 {
+  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {22040 12608 87040}
+do_test 7.5.4 {
+  execsql END db2
+  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
+  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
+} {22040 12608 87040}
+
+
+finish_test
+

test/wal2snapshot.test

@@ -0,0 +1,94 @@
+# 2018 December 5
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA journal_mode=WAL2" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+set testprefix wal2snapshot
+ifcapable !wal {finish_test ; return }
+ifcapable !snapshot {finish_test; return}
+
+foreach {tn mode} {1 wal 2 wal2} {
+  reset_db
+  do_execsql_test $tn.1 "PRAGMA journal_mode = $mode" $mode
+
+  do_execsql_test $tn.2 {
+    CREATE TABLE t1(a, b);
+    INSERT INTO t1 VALUES(1, 2);
+    INSERT INTO t1 VALUES(3, 4);
+    BEGIN;
+  }
+
+  # Check that sqlite3_snapshot_get() is an error for a wal2 db.
+  #
+  if {$tn==1} {
+    do_test 1.3 {
+      set S [sqlite3_snapshot_get db main]
+      sqlite3_snapshot_free $S
+    } {}
+  } else {
+    do_test 2.3 {
+      list [catch { sqlite3_snapshot_get db main } msg] $msg
+    } {1 SQLITE_ERROR}
+  }
+  
+  # Check that sqlite3_snapshot_recover() is an error for a wal2 db.
+  #
+  do_execsql_test $tn.4 COMMIT
+  if {$tn==1} {
+    do_test 1.5 {
+      sqlite3_snapshot_recover db main
+    } {}
+  } else {
+    do_test 2.5 {
+      list [catch { sqlite3_snapshot_recover db main } msg] $msg
+    } {1 SQLITE_ERROR}
+  }
+ 
+  # Check that sqlite3_snapshot_open() is an error for a wal2 db.
+  #
+  if {$tn==1} {
+    do_test 1.6 {
+      execsql BEGIN
+      set SNAPSHOT [sqlite3_snapshot_get_blob db main]
+      sqlite3_snapshot_open_blob db main $SNAPSHOT
+      execsql COMMIT
+    } {}
+  } else {
+
+    do_test 2.6.1 {
+      execsql BEGIN
+      set res [
+        list [catch { sqlite3_snapshot_open_blob db main $SNAPSHOT } msg] $msg
+      ]
+      execsql COMMIT
+      set res
+    } {1 SQLITE_ERROR}
+    do_test 2.6.2 {
+      execsql BEGIN
+      execsql {SELECT * FROM sqlite_master}
+      set res [
+        list [catch { sqlite3_snapshot_open_blob db main $SNAPSHOT } msg] $msg
+      ]
+      execsql COMMIT
+      set res
+    } {1 SQLITE_ERROR}
+  }
+}
+
+
+finish_test
+
+

test/wal_common.tcl

@@ -90,4 +90,44 @@ proc wal_fix_walindex_cksum {hdrvar} {
   lset hdr 11 $c2
 }
 
+# This command assumes that $file is the name of a database file opened
+# in wal mode using a [testvfs] VFS. It returns a list of the 12 32-bit
+# integers that make up the wal-index-header for the named file.
+#
+proc set_tvfs_hdr {file args} {
+
+  # Set $nHdr to the number of bytes in the wal-index header:
+  set nHdr 48
+  set nInt [expr {$nHdr/4}]
+
+  if {[llength $args]>2} {
+    error {wrong # args: should be "set_tvfs_hdr fileName ?val1? ?val2?"}
+  }
+
+  set blob [tvfs shm $file]
+  if {$::tcl_platform(byteOrder)=="bigEndian"} {set fmt I} {set fmt i}
+
+  if {[llength $args]} {
+    set ia [lindex $args 0]
+    set ib $ia
+    if {[llength $args]==2} {
+      set ib [lindex $args 1]
+    }
+    binary scan $blob a[expr $nHdr*2]a* dummy tail
+    set blob [binary format ${fmt}${nInt}${fmt}${nInt}a* $ia $ib $tail]
+    tvfs shm $file $blob
+  }
+
+  binary scan $blob ${fmt}${nInt} ints
+  return $ints
+}
+
+proc incr_tvfs_hdr {file idx incrval} {
+  set ints [set_tvfs_hdr $file]
+  set v [lindex $ints $idx]
+  incr v $incrval
+  lset ints $idx $v
+  set_tvfs_hdr $file $ints
+}
+
 

test/walprotocol2.test

@@ -85,7 +85,7 @@ proc lock_callback {method filename handle lock} {
     db2 eval { INSERT INTO x VALUES('x') }
   }
 }
-db timeout 10
+db timeout 1100
 do_catchsql_test 2.4 {
   BEGIN EXCLUSIVE;
 } {0 {}}

test/walrofault.test

@@ -55,6 +55,4 @@ do_faultsim_test 1 -faults oom* -prep {
   faultsim_test_result {0 {hello world ! world hello}}
 }
 
-
-
 finish_test

tool/mkpragmatab.tcl

@@ -107,6 +107,12 @@ set pragma_def {
   IF:   !defined(SQLITE_OMIT_FLAG_PRAGMAS)
   IF:   defined(SQLITE_DEBUG)
 
+  NAME: noop_update
+  TYPE: FLAG
+  ARG:  SQLITE_NoopUpdate
+  IF:   !defined(SQLITE_OMIT_FLAG_PRAGMAS)
+  IF:   defined(SQLITE_ENABLE_NOOP_UPDATE)
+
   NAME: ignore_check_constraints
   TYPE: FLAG
   ARG:  SQLITE_IgnoreChecks

tool/tserver.c

@@ -0,0 +1,643 @@
+/*
+** 2017 June 7
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** Simple multi-threaded server used for informal testing of concurrency
+** between connections in different threads. Listens for tcp/ip connections
+** on port 9999 of the 127.0.0.1 interface only. To build:
+**
+**   gcc -g $(TOP)/tool/tserver.c sqlite3.o -lpthread -o tserver
+**
+** To run using "x.db" as the db file:
+**
+**   ./tserver x.db
+**
+** To connect, open a client socket on port 9999 and start sending commands.
+** Commands are either SQL - which must be terminated by a semi-colon, or
+** dot-commands, which must be terminated by a newline. If an SQL statement
+** is seen, it is prepared and added to an internal list.
+**
+** Dot-commands are:
+**
+**   .list                    Display all SQL statements in the list.
+**   .quit                    Disconnect.
+**   .run                     Run all SQL statements in the list.
+**   .repeats N               Configure the number of repeats per ".run".
+**   .seconds N               Configure the number of seconds to ".run" for.
+**   .mutex_commit            Add a "COMMIT" protected by a g.commit_mutex
+**                            to the current SQL.
+**   .stop                    Stop the tserver process - exit(0).
+**   .checkpoint N
+**   .integrity_check
+**
+** Example input:
+**
+**   BEGIN;
+**     INSERT INTO t1 VALUES(randomblob(10), randomblob(100));
+**     INSERT INTO t1 VALUES(randomblob(10), randomblob(100));
+**     INSERT INTO t1 VALUES(randomblob(10), randomblob(100));
+**   COMMIT;
+**   .repeats 100000
+**   .run
+**
+*/
+#define TSERVER_PORTNUMBER 9999
+
+#include <arpa/inet.h>
+#include <assert.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+#include "sqlite3.h"
+
+#define TSERVER_DEFAULT_CHECKPOINT_THRESHOLD 3900
+
+/* Global variables */
+struct TserverGlobal {
+  char *zDatabaseName;             /* Database used by this server */
+  char *zVfs;
+  sqlite3_mutex *commit_mutex;
+  sqlite3 *db;                     /* Global db handle */
+
+  /* The following use native pthreads instead of a portable interface. This
+  ** is because a condition variable, as well as a mutex, is required.  */
+  pthread_mutex_t ckpt_mutex;
+  pthread_cond_t ckpt_cond;
+  int nThreshold;                  /* Checkpoint when wal is this large */
+  int bCkptRequired;               /* True if wal checkpoint is required */
+  int nRun;                        /* Number of clients in ".run" */
+  int nWait;                       /* Number of clients waiting on ckpt_cond */
+};
+
+static struct TserverGlobal g = {0};
+
+typedef struct ClientSql ClientSql;
+struct ClientSql {
+  sqlite3_stmt *pStmt;
+  int flags;
+};
+
+#define TSERVER_CLIENTSQL_MUTEX     0x0001
+#define TSERVER_CLIENTSQL_INTEGRITY 0x0002
+
+typedef struct ClientCtx ClientCtx;
+struct ClientCtx {
+  sqlite3 *db;                    /* Database handle for this client */
+  int fd;                         /* Client fd */
+  int nRepeat;                    /* Number of times to repeat SQL */
+  int nSecond;                    /* Number of seconds to run for */
+  ClientSql *aPrepare;            /* Array of prepared statements */
+  int nPrepare;                   /* Valid size of apPrepare[] */
+  int nAlloc;                     /* Allocated size of apPrepare[] */
+
+  int nClientThreshold;           /* Threshold for checkpointing */
+  int bClientCkptRequired;        /* True to do a checkpoint */
+};
+
+static int is_eol(int i){
+  return (i=='\n' || i=='\r');
+}
+static int is_whitespace(int i){
+  return (i==' ' || i=='\t' || is_eol(i));
+}
+
+/*
+** Implementation of SQL scalar function usleep().
+*/
+static void usleepFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  int nUs;
+  sqlite3_vfs *pVfs = (sqlite3_vfs*)sqlite3_user_data(context);
+  assert( argc==1 );
+  nUs = sqlite3_value_int64(argv[0]);
+  pVfs->xSleep(pVfs, nUs);
+}
+
+static void trim_string(const char **pzStr, int *pnStr){
+  const char *zStr = *pzStr;
+  int nStr = *pnStr;
+
+  while( nStr>0 && is_whitespace(zStr[0]) ){
+    zStr++;
+    nStr--;
+  }
+  while( nStr>0 && is_whitespace(zStr[nStr-1]) ){
+    nStr--;
+  }
+
+  *pzStr = zStr;
+  *pnStr = nStr;
+}
+
+static int send_message(ClientCtx *p, const char *zFmt, ...){
+  char *zMsg;
+  va_list ap;                             /* Vararg list */
+  va_start(ap, zFmt);
+  int res = -1;
+
+  zMsg = sqlite3_vmprintf(zFmt, ap);
+  if( zMsg ){
+    res = write(p->fd, zMsg, strlen(zMsg));
+  }
+  sqlite3_free(zMsg);
+  va_end(ap);
+
+  return (res<0);
+}
+
+static int handle_some_sql(ClientCtx *p, const char *zSql, int nSql){
+  const char *zTail = zSql;
+  int nTail = nSql;
+  int rc = SQLITE_OK;
+
+  while( rc==SQLITE_OK ){
+    if( p->nPrepare>=p->nAlloc ){
+      int nByte = (p->nPrepare+32) * sizeof(ClientSql);
+      ClientSql *aNew = sqlite3_realloc(p->aPrepare, nByte);
+      if( aNew ){
+        memset(&aNew[p->nPrepare], 0, sizeof(ClientSql)*32);
+        p->aPrepare = aNew;
+        p->nAlloc = p->nPrepare+32;
+      }else{
+        rc = SQLITE_NOMEM;
+        break;
+      }
+    }
+    rc = sqlite3_prepare_v2(
+        p->db, zTail, nTail, &p->aPrepare[p->nPrepare].pStmt, &zTail
+    );
+    if( rc!=SQLITE_OK ){
+      send_message(p, "error - %s (eec=%d)\n", sqlite3_errmsg(p->db),
+          sqlite3_extended_errcode(p->db)
+      );
+      rc = 1;
+      break;
+    }
+    if( p->aPrepare[p->nPrepare].pStmt==0 ){
+      break;
+    }
+    p->nPrepare++;
+    nTail = nSql - (zTail-zSql);
+    rc = send_message(p, "ok (%d SQL statements)\n", p->nPrepare);
+  }
+
+  return rc;
+}
+
+/*
+** Return a micro-seconds resolution timer.
+*/
+static sqlite3_int64 get_timer(void){
+  struct timeval t;
+  gettimeofday(&t, 0);
+  return (sqlite3_int64)t.tv_usec + ((sqlite3_int64)t.tv_sec * 1000000);
+}
+
+static void clear_sql(ClientCtx *p){
+  int j;
+  for(j=0; j<p->nPrepare; j++){
+    sqlite3_finalize(p->aPrepare[j].pStmt);
+  }
+  p->nPrepare = 0;
+}
+
+/*
+** The sqlite3_wal_hook() callback used by all client database connections.
+*/
+static int clientWalHook(void *pArg, sqlite3 *db, const char *zDb, int nFrame){
+  if( g.nThreshold>0 ){
+    if( nFrame>=g.nThreshold ){
+      g.bCkptRequired = 1;
+    }
+  }else{
+    ClientCtx *pCtx = (ClientCtx*)pArg;
+    if( pCtx->nClientThreshold && nFrame>=pCtx->nClientThreshold ){
+      pCtx->bClientCkptRequired = 1;
+    }
+  }
+  return SQLITE_OK;
+}
+
+static int handle_run_command(ClientCtx *p){
+  int i, j;
+  int nBusy = 0;
+  sqlite3_int64 t0 = get_timer();
+  sqlite3_int64 t1 = t0;
+  sqlite3_int64 tCommit = 0;
+  int nT1 = 0;
+  int nTBusy1 = 0;
+  int rc = SQLITE_OK;
+
+  pthread_mutex_lock(&g.ckpt_mutex);
+  g.nRun++;
+  pthread_mutex_unlock(&g.ckpt_mutex);
+
+  for(j=0; (p->nRepeat<=0 || j<p->nRepeat) && rc==SQLITE_OK; j++){
+    sqlite3_int64 t2;
+
+    for(i=0; i<p->nPrepare && rc==SQLITE_OK; i++){
+      sqlite3_stmt *pStmt = p->aPrepare[i].pStmt;
+
+      /* If the MUTEX flag is set, grab g.commit_mutex before executing
+      ** the SQL statement (which is always "COMMIT" in this case). */
+      if( p->aPrepare[i].flags & TSERVER_CLIENTSQL_MUTEX ){
+        sqlite3_mutex_enter(g.commit_mutex);
+        tCommit -= get_timer();
+      }
+
+      /* Execute the statement */
+      if( p->aPrepare[i].flags & TSERVER_CLIENTSQL_INTEGRITY ){
+        sqlite3_step(pStmt);
+        if( sqlite3_stricmp("ok", (const char*)sqlite3_column_text(pStmt, 0)) ){
+          send_message(p, "error - integrity_check failed: %s\n", 
+              sqlite3_column_text(pStmt, 0)
+          );
+        }
+        sqlite3_reset(pStmt);
+      }
+      while( sqlite3_step(pStmt)==SQLITE_ROW );
+      rc = sqlite3_reset(pStmt);
+
+      /* Relinquish the g.commit_mutex mutex if required. */
+      if( p->aPrepare[i].flags & TSERVER_CLIENTSQL_MUTEX ){
+        tCommit += get_timer();
+        sqlite3_mutex_leave(g.commit_mutex);
+      }
+
+      if( (rc & 0xFF)==SQLITE_BUSY ){
+        if( sqlite3_get_autocommit(p->db)==0 ){
+          sqlite3_exec(p->db, "ROLLBACK", 0, 0, 0);
+        }
+        nBusy++;
+        rc = SQLITE_OK;
+        break;
+      }
+      else if( rc!=SQLITE_OK ){
+        send_message(p, "error - %s (eec=%d)\n", sqlite3_errmsg(p->db),
+            sqlite3_extended_errcode(p->db)
+        );
+      }
+    }
+
+    t2 = get_timer();
+    if( t2>=(t1+1000000) ){
+      sqlite3_int64 nUs = (t2 - t1);
+      sqlite3_int64 nDone = (j+1 - nBusy - nT1);
+
+      rc = send_message(
+          p, "(%d done @ %lld per second, %d busy)\n", 
+          (int)nDone, (1000000*nDone + nUs/2) / nUs, nBusy - nTBusy1
+      );
+      t1 = t2;
+      nT1 = j+1 - nBusy;
+      nTBusy1 = nBusy;
+      if( p->nSecond>0 && ((sqlite3_int64)p->nSecond*1000000)<=t1-t0 ) break;
+    }
+
+    /* Global checkpoint handling. */
+    if( g.nThreshold>0 ){
+      pthread_mutex_lock(&g.ckpt_mutex);
+      if( rc==SQLITE_OK && g.bCkptRequired ){
+        if( g.nWait==g.nRun-1 ){
+          /* All other clients are already waiting on the condition variable.
+          ** Run the checkpoint, signal the condition and move on.  */
+          rc = sqlite3_wal_checkpoint(p->db, "main");
+          g.bCkptRequired = 0;
+          pthread_cond_broadcast(&g.ckpt_cond);
+        }else{
+          assert( g.nWait<g.nRun-1 );
+          g.nWait++;
+          pthread_cond_wait(&g.ckpt_cond, &g.ckpt_mutex);
+          g.nWait--;
+        }
+      }
+      pthread_mutex_unlock(&g.ckpt_mutex);
+    }
+
+    if( rc==SQLITE_OK && p->bClientCkptRequired ){
+      rc = sqlite3_wal_checkpoint(p->db, "main");
+      if( rc==SQLITE_BUSY ) rc = SQLITE_OK;
+      assert( rc==SQLITE_OK );
+      p->bClientCkptRequired = 0;
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    int nMs = (get_timer() - t0) / 1000;
+    send_message(p, "ok (%d/%d SQLITE_BUSY)\n", nBusy, j);
+    if( p->nRepeat<=0 ){
+      send_message(p, "### ok %d busy %d ms %d commit-ms %d\n", 
+          j-nBusy, nBusy, nMs, (int)(tCommit / 1000)
+      );
+    }
+  }
+  clear_sql(p);
+
+  pthread_mutex_lock(&g.ckpt_mutex);
+  g.nRun--;
+  pthread_mutex_unlock(&g.ckpt_mutex);
+
+  return rc;
+}
+
+static int handle_dot_command(ClientCtx *p, const char *zCmd, int nCmd){
+  int n;
+  int rc = 0;
+  const char *z = &zCmd[1];
+  const char *zArg;
+  int nArg;
+
+  assert( zCmd[0]=='.' );
+  for(n=0; n<(nCmd-1); n++){
+    if( is_whitespace(z[n]) ) break;
+  }
+
+  zArg = &z[n];
+  nArg = nCmd-n;
+  trim_string(&zArg, &nArg);
+
+  if( n>=1 && n<=4 && 0==strncmp(z, "list", n) ){
+    int i;
+    for(i=0; rc==0 && i<p->nPrepare; i++){
+      const char *zSql = sqlite3_sql(p->aPrepare[i].pStmt);
+      int nSql = strlen(zSql);
+      trim_string(&zSql, &nSql);
+      rc = send_message(p, "%d: %.*s\n", i, nSql, zSql);
+    }
+  }
+
+  else if( n>=1 && n<=4 && 0==strncmp(z, "quit", n) ){
+    rc = -1;
+  }
+
+  else if( n>=2 && n<=7 && 0==strncmp(z, "repeats", n) ){
+    if( nArg ){
+      p->nRepeat = strtol(zArg, 0, 0);
+      if( p->nRepeat>0 ) p->nSecond = 0;
+    }
+    rc = send_message(p, "ok (repeat=%d)\n", p->nRepeat);
+  }
+
+  else if( n>=2 && n<=3 && 0==strncmp(z, "run", n) ){
+    rc = handle_run_command(p);
+  }
+
+  else if( n>=2 && n<=7 && 0==strncmp(z, "seconds", n) ){
+    if( nArg ){
+      p->nSecond = strtol(zArg, 0, 0);
+      if( p->nSecond>0 ) p->nRepeat = 0;
+    }
+    rc = send_message(p, "ok (seconds=%d)\n", p->nSecond);
+  }
+
+  else if( n>=1 && n<=12 && 0==strncmp(z, "mutex_commit", n) ){
+    rc = handle_some_sql(p, "COMMIT;", 7);
+    if( rc==SQLITE_OK ){
+      p->aPrepare[p->nPrepare-1].flags |= TSERVER_CLIENTSQL_MUTEX;
+    }
+  }
+
+  else if( n>=1 && n<=10 && 0==strncmp(z, "checkpoint", n) ){
+    if( nArg ){
+      p->nClientThreshold = strtol(zArg, 0, 0);
+    }
+    rc = send_message(p, "ok (checkpoint=%d)\n", p->nClientThreshold);
+  }
+
+  else if( n>=2 && n<=4 && 0==strncmp(z, "stop", n) ){
+    sqlite3_close(g.db);
+    exit(0);
+  }
+
+  else if( n>=2 && n<=15 && 0==strncmp(z, "integrity_check", n) ){
+    rc = handle_some_sql(p, "PRAGMA integrity_check;", 23);
+    if( rc==SQLITE_OK ){
+      p->aPrepare[p->nPrepare-1].flags |= TSERVER_CLIENTSQL_INTEGRITY;
+    }
+  }
+
+  else{
+    send_message(p, 
+        "unrecognized dot command: %.*s\n"
+        "should be \"list\", \"run\", \"repeats\", \"mutex_commit\", "
+        "\"checkpoint\", \"integrity_check\" or \"seconds\"\n", n, z
+    );
+    rc = 1;
+  }
+
+  return rc;
+}
+
+static void *handle_client(void *pArg){
+  char zCmd[32*1024];             /* Read buffer */
+  int nCmd = 0;                   /* Valid bytes in zCmd[] */
+  int res;                        /* Result of read() call */
+  int rc = SQLITE_OK;
+
+  ClientCtx ctx;
+  memset(&ctx, 0, sizeof(ClientCtx));
+
+  ctx.fd = (int)(intptr_t)pArg;
+  ctx.nRepeat = 1;
+  rc = sqlite3_open_v2(g.zDatabaseName, &ctx.db,
+      SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, g.zVfs
+  );
+  if( rc!=SQLITE_OK ){
+    fprintf(stderr, "sqlite3_open(): %s\n", sqlite3_errmsg(ctx.db));
+    return 0;
+  }
+  sqlite3_create_function(
+      ctx.db, "usleep", 1, SQLITE_UTF8, (void*)sqlite3_vfs_find(0), 
+      usleepFunc, 0, 0
+  );
+
+  /* Register the wal-hook with the new client connection */
+  sqlite3_wal_hook(ctx.db, clientWalHook, (void*)&ctx);
+
+  while( rc==SQLITE_OK ){
+    int i;
+    int iStart;
+    int nConsume;
+    res = read(ctx.fd, &zCmd[nCmd], sizeof(zCmd)-nCmd-1);
+    if( res<=0 ) break;
+    nCmd += res;
+    if( nCmd>=sizeof(zCmd)-1 ){
+      fprintf(stderr, "oversized (>32KiB) message\n");
+      res = 0;
+      break;
+    }
+    zCmd[nCmd] = '\0';
+
+    do {
+      nConsume = 0;
+
+      /* Gobble up any whitespace */
+      iStart = 0;
+      while( is_whitespace(zCmd[iStart]) ) iStart++;
+
+      if( zCmd[iStart]=='.' ){
+        /* This is a dot-command. Search for end-of-line. */
+        for(i=iStart; i<nCmd; i++){
+          if( is_eol(zCmd[i]) ){
+            rc = handle_dot_command(&ctx, &zCmd[iStart], i-iStart);
+            nConsume = i+1;
+            break;
+          }
+        }
+      }else{
+
+        int iSemi;
+        char c = 0;
+        for(iSemi=iStart; iSemi<nCmd; iSemi++){
+          if( zCmd[iSemi]==';' ){
+            c = zCmd[iSemi+1];
+            zCmd[iSemi+1] = '\0';
+            break;
+          }
+        }
+
+        if( iSemi<nCmd ){
+          if( sqlite3_complete(zCmd) ){
+            rc = handle_some_sql(&ctx, zCmd, iSemi+1);
+            nConsume = iSemi+1;
+          }
+
+          if( c ){
+            zCmd[iSemi+1] = c;
+          }
+        }
+      }
+
+      if( nConsume>0 ){
+        nCmd = nCmd-nConsume;
+        if( nCmd>0 ){
+          memmove(zCmd, &zCmd[nConsume], nCmd);
+        }
+      }
+    }while( rc==SQLITE_OK && nConsume>0 );
+  }
+
+  fprintf(stdout, "Client %d disconnects (rc=%d)\n", ctx.fd, rc);
+  fflush(stdout);
+  close(ctx.fd);
+  clear_sql(&ctx);
+  sqlite3_free(ctx.aPrepare);
+  sqlite3_close(ctx.db);
+  return 0;
+} 
+
+static void usage(const char *zExec){
+  fprintf(stderr, "Usage: %s ?-vfs VFS? DATABASE\n", zExec);
+  exit(1);
+}
+
+int main(int argc, char *argv[]) {
+  int sfd;
+  int rc;
+  int yes = 1;
+  struct sockaddr_in server;
+  int i;
+
+  /* Ignore SIGPIPE. Otherwise the server exits if a client disconnects
+  ** abruptly.  */
+  signal(SIGPIPE, SIG_IGN);
+
+  g.nThreshold = TSERVER_DEFAULT_CHECKPOINT_THRESHOLD;
+  if( (argc%2) ) usage(argv[0]);
+  for(i=1; i<(argc-1); i+=2){
+    int n = strlen(argv[i]);
+    if( n>=2 && 0==sqlite3_strnicmp("-walautocheckpoint", argv[i], n) ){
+      g.nThreshold = strtol(argv[i+1], 0, 0);
+    }else 
+    if( n>=2 && 0==sqlite3_strnicmp("-vfs", argv[i], n) ){
+      g.zVfs = argv[i+1];
+    }
+  }
+  g.zDatabaseName = argv[argc-1];
+
+  g.commit_mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST);
+  pthread_mutex_init(&g.ckpt_mutex, 0);
+  pthread_cond_init(&g.ckpt_cond, 0);
+
+  rc = sqlite3_open_v2(g.zDatabaseName, &g.db,
+      SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, g.zVfs
+  );
+  if( rc!=SQLITE_OK ){
+    fprintf(stderr, "sqlite3_open(): %s\n", sqlite3_errmsg(g.db));
+    return 1;
+  }
+
+  rc = sqlite3_exec(g.db, "SELECT * FROM sqlite_master", 0, 0, 0);
+  if( rc!=SQLITE_OK ){
+    fprintf(stderr, "sqlite3_exec(): %s\n", sqlite3_errmsg(g.db));
+    return 1;
+  }
+
+  sfd = socket(AF_INET, SOCK_STREAM, 0);
+  if( sfd<0 ){
+    fprintf(stderr, "socket() failed\n");
+    return 1;
+  }
+
+  rc = setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));
+  if( rc<0 ){
+    perror("setsockopt");
+    return 1;
+  }
+
+  memset(&server, 0, sizeof(server));
+  server.sin_family = AF_INET;
+  server.sin_addr.s_addr = inet_addr("127.0.0.1");
+  server.sin_port = htons(TSERVER_PORTNUMBER);
+
+  rc = bind(sfd, (struct sockaddr *)&server, sizeof(struct sockaddr));
+  if( rc<0 ){
+    fprintf(stderr, "bind() failed\n");
+    return 1;
+  }
+
+  rc = listen(sfd, 8);
+  if( rc<0 ){
+    fprintf(stderr, "listen() failed\n");
+    return 1;
+  }
+
+  while( 1 ){
+    pthread_t tid;
+    int cfd = accept(sfd, NULL, NULL);
+    if( cfd<0 ){
+      perror("accept()");
+      return 1;
+    }
+
+    fprintf(stdout, "Client %d connects\n", cfd);
+    fflush(stdout);
+    rc = pthread_create(&tid, NULL, handle_client, (void*)(intptr_t)cfd);
+    if( rc!=0 ){
+      perror("pthread_create()");
+      return 1;
+    }
+
+    pthread_detach(tid);
+  }
+
+  return 0;
+}

tool/tserver_test.tcl

@@ -0,0 +1,304 @@
+#!/usr/bin/tclsh
+#
+# This script is used to run the performance test cases described in
+# README-server-edition.html.
+#
+
+
+package require sqlite3
+
+# Default values for command line switches:
+set O(-database)  ""
+set O(-rows)      [expr 5000000]
+set O(-mode)      wal2
+set O(-tserver)   "./tserver"
+set O(-seconds)   20
+set O(-writers)   1
+set O(-readers)   0
+set O(-integrity) 0
+set O(-verbose)   0
+set O(-external)   0
+
+
+proc error_out {err} {
+  puts stderr $err
+  exit -1
+}
+
+proc usage {} {
+  puts stderr "Usage: $::argv0 ?OPTIONS?"
+  puts stderr ""
+  puts stderr "Where OPTIONS are:"
+  puts stderr "  -database <database file>             (default: test.db)"
+  puts stderr "  -mode <mode>                          (default: wal2)"
+  puts stderr "  -rows <number of rows>                (default: 5000000)"
+  puts stderr "  -tserver <path to tserver executable> (default: ./tserver)"
+  puts stderr "  -seconds <time to run for in seconds> (default: 20)"
+  puts stderr "  -writers <number of writer clients>   (default: 1)"
+  puts stderr "  -readers <number of reader clients>   (default: 0)"
+  puts stderr "  -integrity <number of IC clients>     (default: 0)"
+  puts stderr "  -verbose 0|1                          (default: 0)"
+  puts stderr "  -external 0|1                         (default: 0)"
+  exit -1
+}
+
+for {set i 0} {$i < [llength $argv]} {incr i} {
+  set opt ""
+  set arg [lindex $argv $i]
+  set n [expr [string length $arg]-1]
+  foreach k [array names ::O] {
+    if {[string range $k 0 $n]==$arg} {
+      if {$opt==""} {
+        set opt $k
+      } else {
+        error_out "ambiguous option: $arg ($k or $opt)"
+      }
+    }
+  }
+  if {$opt==""} { usage }
+  if {$i==[llength $argv]-1} {
+    error_out "option requires an argument: $opt"
+  }
+  incr i
+  set val [lindex $argv $i]
+  switch -- $opt {
+    -mode {
+      if {$val != "wal" && $val != "wal2"} {
+        set xyz "\"wal\" or \"wal2\""
+        error_out "Found \"$val\" - expected $xyz"
+      }
+    }
+  }
+  set O($opt) [lindex $argv $i]
+}
+if {$O(-database)==""} {
+  set O(-database) "test.db"
+}
+
+set O(-rows) [expr $O(-rows)]
+
+#--------------------------------------------------------------------------
+# Create and populate the required test database, if it is not already 
+# present in the file-system.
+#
+proc create_test_database {} {
+  global O
+
+  if {$O(-external)} return
+
+  if {[file exists $O(-database)]} {
+    sqlite3 db $O(-database)
+
+    # Check the schema looks Ok.
+    set s [db one {
+      SELECT group_concat(name||pk, '.') FROM pragma_table_info('t1');
+    }]
+    if {$s != "a1.b0.c0.d0"} {
+      error_out "Database $O(-database) exists but is not usable (schema)"
+    }
+
+    # Check that the row count matches.
+    set n [db one { SELECT count(*) FROM t1 }]
+    if {$n != $O(-rows)} {
+      error_out "Database $O(-database) exists but is not usable (row-count)"
+    }
+    db close
+  } else {
+    catch { file delete -force $O(-database)-journal }
+    catch { file delete -force $O(-database)-wal }
+
+    if {$O(-verbose)} {
+      puts "Building database $O(-database)..."
+    }
+
+    sqlite3 db $O(-database)
+    db eval {
+      CREATE TABLE t1(
+        a INTEGER PRIMARY KEY, 
+        b BLOB(16), 
+        c BLOB(16), 
+        d BLOB(400)
+      );
+      CREATE INDEX i1 ON t1(b);
+      CREATE INDEX i2 ON t1(c);
+
+      WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$O(-rows))
+      INSERT INTO t1 
+      SELECT i-1, randomblob(16), randomblob(16), randomblob(400) FROM s;
+    }
+    db close
+  }
+
+  switch -- $O(-mode) {
+    wal {
+      sqlite3 db $O(-database)
+      db eval {PRAGMA journal_mode = delete}
+      db eval {PRAGMA journal_mode = wal}
+      db close
+    }
+
+    wal2 {
+      sqlite3 db $O(-database)
+      db eval {PRAGMA journal_mode = delete}
+      db eval {PRAGMA journal_mode = wal2}
+      db close
+    }
+  }
+}
+
+#-------------------------------------------------------------------------
+# Functions to start and stop the tserver process:
+#
+#   tserver_start
+#   tserver_stop
+#
+set ::tserver {}
+proc tserver_start {} {
+  global O
+  if {$O(-external)} return
+  set cmd "|$O(-tserver) -vfs unix-excl "
+  if {$O(-mode)=="wal2"} {
+    append cmd " -walautocheckpoint 0 "
+  }
+  append cmd "$O(-database)"
+  set ::tserver [open $cmd]
+  fconfigure $::tserver -blocking 0
+  fileevent $::tserver readable tserver_data
+}
+
+proc tserver_data {} {
+  global O
+  if {[eof $::tserver]} {
+    error_out "tserver has exited"
+  }
+  set line [gets $::tserver]
+  if {$line != "" && $O(-verbose)} {
+    puts "tserver: $line"
+  }
+}
+
+proc tserver_stop {} {
+  global O
+  if {$O(-external)} return
+  close $::tserver
+  set fd [socket localhost 9999]
+  puts $fd ".stop"
+  close $fd
+}
+#-------------------------------------------------------------------------
+
+set ::nClient 0
+set ::client_output [list]
+
+proc client_data {name fd} {
+  global O
+  if {[eof $fd]} {
+    incr ::nClient -1
+    close $fd
+    return
+  }
+  set str [gets $fd]
+  if {[string trim $str]!=""} {
+    if {[string range $str 0 3]=="### "} {
+      lappend ::client_output [concat [list name $name] [lrange $str 1 end]]
+    } 
+    if {$O(-verbose)} {
+      puts "$name: $str"
+    }
+  }
+}
+
+proc client_launch {name script} {
+  global O
+  set fd [socket localhost 9999]
+  fconfigure $fd -blocking 0
+  puts $fd "PRAGMA synchronous = OFF;"
+  puts $fd ".repeat 1"
+  puts $fd ".run"
+  puts $fd $script
+  puts $fd ".seconds $O(-seconds)"
+  puts $fd ".run"
+  puts $fd ".quit"
+  flush $fd
+  incr ::nClient
+  fileevent $fd readable [list client_data $name $fd]
+}
+
+proc client_wait {} {
+  while {$::nClient>0} {vwait ::nClient}
+}
+
+proc script_writer {bCkpt} {
+  global O
+
+  set commit ".mutex_commit"
+  set begin "BEGIN CONCURRENT;"
+  set ckpt ""
+  if {$bCkpt} {
+    set ckpt ".checkpoint 2000"
+  }
+
+  set tail "randomblob(16), randomblob(16), randomblob(400));"
+  return [subst -nocommands {
+    $ckpt
+    $begin
+      REPLACE INTO t1 VALUES(abs(random() % $O(-rows)), $tail
+    $commit
+  }]
+}
+
+proc script_reader {} {
+  global O
+
+  return [subst -nocommands {
+    BEGIN;
+      SELECT * FROM t1 WHERE a>abs((random()%$O(-rows))) LIMIT 10;
+    END;
+  }]
+}
+
+proc script_integrity {} {
+  global O
+
+  return {
+    .integrity_check
+  }
+}
+
+
+create_test_database
+tserver_start
+
+for {set i 0} {$i < $O(-writers)} {incr i} {
+  client_launch w.$i [script_writer [expr {$i==0 && $O(-mode)=="wal2"}]]
+}
+for {set i 0} {$i < $O(-readers)} {incr i} {
+  client_launch r.$i [script_reader]
+}
+for {set i 0} {$i < $O(-integrity)} {incr i} {
+  client_launch i.$i [script_integrity]
+}
+client_wait
+
+set name(w) "Writers"
+set name(r) "Readers"
+set name(i) "Integrity"
+foreach r $::client_output {
+  array set a $r
+  set type [string range $a(name) 0 0]
+  incr x($type.ok) $a(ok);
+  incr x($type.busy) $a(busy);
+  incr x($type.n) 1
+  set t($type) 1
+}
+
+foreach type [array names t] {
+  set nTPS [expr $x($type.ok) / $O(-seconds)]
+  set nC [expr $nTPS / $x($type.n)]
+  set nTotal [expr $x($type.ok) + $x($type.busy)]
+  set bp [format %.2f [expr $x($type.busy) * 100.0 / $nTotal]]
+  puts "$name($type): $nTPS transactions/second ($nC per client) ($bp% busy)"
+}
+
+tserver_stop
+