Enhance the scrub utility program so that it does a FULL checkpoint prior to

starting the backup, to ensure that the database file content matches what needs
to be backed up without having to look at the WAL file.

FossilOrigin-Name: ab1c5ce50f139070d7a322f43132a93c8af2ed68

Makefile.in

@@ -589,6 +589,10 @@ sqlite3$(TEXE):	$(TOP)/src/shell.c sqlite3.c
 sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS)
 
+scrub$(TEXE):	$(TOP)/ext/misc/scrub.c sqlite3.o
+	$(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \
+		$(TOP)/ext/misc/scrub.c sqlite3.o $(TLIBS)
+
 srcck1$(BEXE):	$(TOP)/tool/srcck1.c
 	$(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c
 

Makefile.msc

@@ -1455,6 +1455,9 @@ $(SQLITE3EXE):	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_S
 sqldiff.exe:	$(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
+scrub.exe:	$(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
 srcck1.exe:	$(TOP)\tool\srcck1.c
 	$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c
 

ext/misc/scrub.c

@@ -0,0 +1,594 @@
+/*
+** 2016-05-05
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file implements a utility function (and a utility program) that
+** makes a copy of an SQLite database while simultaneously zeroing out all
+** deleted content.
+**
+** Normally (when PRAGMA secure_delete=OFF, which is the default) when SQLite
+** deletes content, it does not overwrite the deleted content but rather marks
+** the region of the file that held that content as being reusable.  This can
+** cause deleted content to recoverable from the database file.  This stale
+** content is removed by the VACUUM command, but VACUUM can be expensive for
+** large databases.  When in PRAGMA secure_delete=ON mode, the deleted content
+** is zeroed, but secure_delete=ON has overhead as well.
+**
+** This utility attempts to make a copy of a complete SQLite database where
+** all of the deleted content is zeroed out in the copy, and it attempts to
+** do so while being faster than running VACUUM.
+**
+** Usage:
+**
+**   int sqlite3_scrub_backup(
+**       const char *zSourceFile,   // Source database filename
+**       const char *zDestFile,     // Destination database filename
+**       char **pzErrMsg            // Write error message here
+**   );
+**
+** Simply call the API above specifying the filename of the source database
+** and the name of the backup copy.  The source database must already exist
+** and can be in active use. (A read lock is held during the backup.)  The
+** destination file should not previously exist.  If the pzErrMsg parameter
+** is non-NULL and if an error occurs, then an error message might be written
+** into memory obtained from sqlite3_malloc() and *pzErrMsg made to point to
+** that error message.  But if the error is an OOM, the error might not be
+** reported.  The routine always returns non-zero if there is an error.
+**
+** If compiled with -DSCRUB_STANDALONE then a main() procedure is added and
+** this file becomes a standalone program that can be run as follows:
+**
+**      ./sqlite3scrub SOURCE DEST
+*/
+#include "sqlite3.h"
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+
+typedef struct ScrubState ScrubState;
+typedef unsigned char u8;
+typedef unsigned short u16;
+typedef unsigned int u32;
+
+
+/* State information for a scrub-and-backup operation */
+struct ScrubState {
+  const char *zSrcFile;    /* Name of the source file */
+  const char *zDestFile;   /* Name of the destination file */
+  int rcErr;               /* Error code */
+  char *zErr;              /* Error message text */
+  sqlite3 *dbSrc;          /* Source database connection */
+  sqlite3_file *pSrc;      /* Source file handle */
+  sqlite3 *dbDest;         /* Destination database connection */
+  sqlite3_file *pDest;     /* Destination file handle */
+  u32 szPage;              /* Page size */
+  u32 szUsable;            /* Usable bytes on each page */
+  u32 nPage;               /* Number of pages */
+  u8 *page1;               /* Content of page 1 */
+};
+
+/* Store an error message */
+static void scrubBackupErr(ScrubState *p, const char *zFormat, ...){
+  va_list ap;
+  sqlite3_free(p->zErr);
+  va_start(ap, zFormat);
+  p->zErr = sqlite3_vmprintf(zFormat, ap);
+  va_end(ap);
+  if( p->rcErr==0 ) p->rcErr = SQLITE_ERROR;
+}
+
+/* Allocate memory to hold a single page of content */
+static u8 *scrubBackupAllocPage(ScrubState *p){
+  u8 *pPage;
+  if( p->rcErr ) return 0;
+  pPage = sqlite3_malloc( p->szPage );
+  if( pPage==0 ) p->rcErr = SQLITE_NOMEM;
+  return pPage;
+}
+
+/* Read a page from the source database into memory.  Use the memory
+** provided by pBuf if not NULL or allocate a new page if pBuf==NULL.
+*/
+static u8 *scrubBackupRead(ScrubState *p, int pgno, u8 *pBuf){
+  int rc;
+  sqlite3_int64 iOff;
+  u8 *pOut = pBuf;
+  if( p->rcErr ) return 0;
+  if( pOut==0 ){
+    pOut = scrubBackupAllocPage(p);
+    if( pOut==0 ) return 0;
+  }
+  iOff = (pgno-1)*(sqlite3_int64)p->szPage;
+  rc = p->pSrc->pMethods->xRead(p->pSrc, pOut, p->szPage, iOff);
+  if( rc!=SQLITE_OK ){
+    if( pBuf==0 ) sqlite3_free(pOut);
+    pOut = 0;
+    scrubBackupErr(p, "read failed for page %d", pgno);
+    p->rcErr = SQLITE_IOERR;
+  }
+  return pOut;  
+}
+
+/* Write a page to the destination database */
+static void scrubBackupWrite(ScrubState *p, int pgno, const u8 *pData){
+  int rc;
+  sqlite3_int64 iOff;
+  if( p->rcErr ) return;
+  iOff = (pgno-1)*(sqlite3_int64)p->szPage;
+  rc = p->pDest->pMethods->xWrite(p->pDest, pData, p->szPage, iOff);
+  if( rc!=SQLITE_OK ){
+    scrubBackupErr(p, "write failed for page %d", pgno);
+    p->rcErr = SQLITE_IOERR;
+  }
+}
+
+/* Prepare a statement against the "db" database. */
+static sqlite3_stmt *scrubBackupPrepare(
+  ScrubState *p,      /* Backup context */
+  sqlite3 *db,        /* Database to prepare against */
+  const char *zSql    /* SQL statement */
+){
+  sqlite3_stmt *pStmt;
+  if( p->rcErr ) return 0;
+  p->rcErr = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+  if( p->rcErr ){
+    scrubBackupErr(p, "SQL error \"%s\" on \"%s\"",
+                   sqlite3_errmsg(db), zSql);
+    sqlite3_finalize(pStmt);
+    return 0;
+  }
+  return pStmt;
+}
+
+
+/* Open the source database file */
+static void scrubBackupOpenSrc(ScrubState *p){
+  sqlite3_stmt *pStmt;
+  int rc;
+  /* Open the source database file */
+  p->rcErr = sqlite3_open_v2(p->zSrcFile, &p->dbSrc,
+                 SQLITE_OPEN_READWRITE |
+                 SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0);
+  if( p->rcErr ){
+    scrubBackupErr(p, "cannot open source database: %s",
+                      sqlite3_errmsg(p->dbSrc));
+    return;
+  }
+  p->rcErr = sqlite3_exec(p->dbSrc, "SELECT 1 FROM sqlite_master; BEGIN;",
+                          0, 0, 0);
+  if( p->rcErr ){
+    scrubBackupErr(p,
+       "cannot start a read transaction on the source database: %s",
+       sqlite3_errmsg(p->dbSrc));
+    return;
+  }
+  rc = sqlite3_wal_checkpoint_v2(p->dbSrc, "main", SQLITE_CHECKPOINT_FULL,
+                                 0, 0);
+  if( rc ){
+    scrubBackupErr(p, "cannot checkpoint the source database");
+    return;
+  }
+  pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_size");
+  if( pStmt==0 ) return;
+  rc = sqlite3_step(pStmt);
+  if( rc==SQLITE_ROW ){
+    p->szPage = sqlite3_column_int(pStmt, 0);
+  }else{
+    scrubBackupErr(p, "unable to determine the page size");
+  }
+  sqlite3_finalize(pStmt);
+  if( p->rcErr ) return;
+  pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_count");
+  if( pStmt==0 ) return;
+  rc = sqlite3_step(pStmt);
+  if( rc==SQLITE_ROW ){
+    p->nPage = sqlite3_column_int(pStmt, 0);
+  }else{
+    scrubBackupErr(p, "unable to determine the size of the source database");
+  }
+  sqlite3_finalize(pStmt);
+  sqlite3_file_control(p->dbSrc, "main", SQLITE_FCNTL_FILE_POINTER, &p->pSrc);
+  if( p->pSrc==0 || p->pSrc->pMethods==0 ){
+    scrubBackupErr(p, "cannot get the source file handle");
+    p->rcErr = SQLITE_ERROR;
+  }
+}
+
+/* Create and open the destination file */
+static void scrubBackupOpenDest(ScrubState *p){
+  sqlite3_stmt *pStmt;
+  int rc;
+  char *zSql;
+  if( p->rcErr ) return;
+  p->rcErr = sqlite3_open_v2(p->zDestFile, &p->dbDest,
+                 SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE |
+                 SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0);
+  if( p->rcErr ){
+    scrubBackupErr(p, "cannot open destination database: %s",
+                      sqlite3_errmsg(p->dbDest));
+    return;
+  }
+  zSql = sqlite3_mprintf("PRAGMA page_size(%u);", p->szPage);
+  if( zSql==0 ){
+    p->rcErr = SQLITE_NOMEM;
+    return;
+  }
+  p->rcErr = sqlite3_exec(p->dbDest, zSql, 0, 0, 0);
+  sqlite3_free(zSql);
+  if( p->rcErr ){
+    scrubBackupErr(p,
+       "cannot set the page size on the destination database: %s",
+       sqlite3_errmsg(p->dbDest));
+    return;
+  }
+  sqlite3_exec(p->dbDest, "PRAGMA journal_mode=OFF;", 0, 0, 0);
+  p->rcErr = sqlite3_exec(p->dbDest, "BEGIN EXCLUSIVE;", 0, 0, 0);
+  if( p->rcErr ){
+    scrubBackupErr(p,
+       "cannot start a write transaction on the destination database: %s",
+       sqlite3_errmsg(p->dbDest));
+    return;
+  }
+  pStmt = scrubBackupPrepare(p, p->dbDest, "PRAGMA page_count;");
+  if( pStmt==0 ) return;
+  rc = sqlite3_step(pStmt);
+  if( rc!=SQLITE_ROW ){
+    scrubBackupErr(p, "cannot measure the size of the destination");
+  }else if( sqlite3_column_int(pStmt, 0)>1 ){
+    scrubBackupErr(p, "destination database is not empty - holds %d pages",
+                   sqlite3_column_int(pStmt, 0));
+  }
+  sqlite3_finalize(pStmt);
+  sqlite3_file_control(p->dbDest, "main", SQLITE_FCNTL_FILE_POINTER, &p->pDest);
+  if( p->pDest==0 || p->pDest->pMethods==0 ){
+    scrubBackupErr(p, "cannot get the destination file handle");
+    p->rcErr = SQLITE_ERROR;
+  }
+}
+
+/* Read a 32-bit big-endian integer */
+static u32 scrubBackupInt32(const u8 *a){
+  u32 v = a[3];
+  v += ((u32)a[2])<<8;
+  v += ((u32)a[1])<<16;
+  v += ((u32)a[0])<<24;
+  return v;
+}
+
+/* Read a 16-bit big-endian integer */
+static u32 scrubBackupInt16(const u8 *a){
+  return (a[0]<<8) + a[1];
+}
+
+/*
+** Read a varint.  Put the value in *pVal and return the number of bytes.
+*/
+static int scrubBackupVarint(const u8 *z, sqlite3_int64 *pVal){
+  sqlite3_int64 v = 0;
+  int i;
+  for(i=0; i<8; i++){
+    v = (v<<7) + (z[i]&0x7f);
+    if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; }
+  }
+  v = (v<<8) + (z[i]&0xff);
+  *pVal = v;
+  return 9;
+}
+
+/*
+** Return the number of bytes in a varint.
+*/
+static int scrubBackupVarintSize(const u8 *z){
+  int i;
+  for(i=0; i<8; i++){
+    if( (z[i]&0x80)==0 ){ return i+1; }
+  }
+  return 9;
+}
+
+/*
+** Copy the freelist trunk page given, and all its descendents,
+** zeroing out as much as possible in the process.
+*/
+static void scrubBackupFreelist(ScrubState *p, int pgno, u32 nFree){
+  u8 *a, *aBuf;
+  u32 n, mx;
+
+  if( p->rcErr ) return;
+  aBuf = scrubBackupAllocPage(p);
+  if( aBuf==0 ) return;
+ 
+  while( pgno && nFree){
+    a = scrubBackupRead(p, pgno, aBuf);
+    if( a==0 ) break;
+    n = scrubBackupInt32(&a[4]);
+    mx = p->szUsable/4 - 2;
+    if( n<mx ){
+      memset(&a[n*4+8], 0, 4*(mx-n));
+    }
+    scrubBackupWrite(p, pgno, a);
+    pgno = scrubBackupInt32(a);
+#if 0
+    /* There is really no point in copying the freelist leaf pages.
+    ** Simply leave them uninitialized in the destination database.  The
+    ** OS filesystem should zero those pages for us automatically.
+    */
+    for(i=0; i<n && nFree; i++){
+      u32 iLeaf = scrubBackupInt32(&a[i*4+8]);
+      if( aZero==0 ){
+        aZero = scrubBackupAllocPage(p);
+        if( aZero==0 ){ pgno = 0; break; }
+        memset(aZero, 0, p->szPage);
+      }
+      scrubBackupWrite(p, iLeaf, aZero);
+      nFree--;
+    }
+#endif
+  }
+  sqlite3_free(aBuf);
+}
+
+/*
+** Copy an overflow chain from source to destination.  Zero out any
+** unused tail at the end of the overflow chain.
+*/
+static void scrubBackupOverflow(ScrubState *p, int pgno, u32 nByte){
+  u8 *a, *aBuf;
+
+  aBuf = scrubBackupAllocPage(p);
+  if( aBuf==0 ) return;
+  while( nByte>0 && pgno!=0 ){
+    a = scrubBackupRead(p, pgno, aBuf);
+    if( a==0 ) break;
+    if( nByte >= (p->szUsable)-4 ){
+      nByte -= (p->szUsable) - 4;
+    }else{
+      u32 x = (p->szUsable - 4) - nByte;
+      u32 i = p->szUsable - x;
+      memset(&a[i], 0, x);
+      nByte = 0;
+    }
+    scrubBackupWrite(p, pgno, a);
+    pgno = scrubBackupInt32(a);
+  }
+  sqlite3_free(aBuf);      
+}
+   
+
+/*
+** Copy B-Tree page pgno, and all of its children, from source to destination.
+** Zero out deleted content during the copy.
+*/
+static void scrubBackupBtree(ScrubState *p, int pgno, int iDepth){
+  u8 *a;
+  u32 i, n, pc;
+  u32 nCell;
+  u32 nPrefix;
+  u32 szHdr;
+  u32 iChild;
+  u8 *aTop;
+  u8 *aCell;
+  u32 x, y;
+  int ln = 0;
+
+  
+  if( p->rcErr ) return;
+  if( iDepth>50 ){
+    scrubBackupErr(p, "corrupt: b-tree too deep at page %d", pgno);
+    return;
+  }
+  if( pgno==1 ){
+    a = p->page1;
+  }else{
+    a = scrubBackupRead(p, pgno, 0);
+    if( a==0 ) return;
+  }
+  nPrefix = pgno==1 ? 100 : 0;
+  aTop = &a[nPrefix];
+  szHdr = 8 + 4*(aTop[0]==0x02 || aTop[0]==0x05);
+  aCell = aTop + szHdr;
+  nCell = scrubBackupInt16(&aTop[3]);
+
+  /* Zero out the gap between the cell index and the start of the
+  ** cell content area */
+  x = scrubBackupInt16(&aTop[5]);  /* First byte of cell content area */
+  if( x>p->szUsable ){ ln=__LINE__; goto btree_corrupt; }
+  y = szHdr + nPrefix + nCell*2;
+  if( y>x ){ ln=__LINE__; goto btree_corrupt; }
+  if( y<x ) memset(a+y, 0, x-y);  /* Zero the gap */
+
+  /* Zero out all the free blocks */  
+  pc = scrubBackupInt16(&aTop[1]);
+  if( pc>0 && pc<x ){ ln=__LINE__; goto btree_corrupt; }
+  while( pc ){
+    if( pc>(p->szUsable)-4 ){ ln=__LINE__; goto btree_corrupt; }
+    n = scrubBackupInt16(&a[pc+2]);
+    if( pc+n>(p->szUsable) ){ ln=__LINE__; goto btree_corrupt; }
+    if( n>4 ) memset(&a[pc+4], 0, n-4);
+    x = scrubBackupInt16(&a[pc]);
+    if( x<pc+4 && x>0 ){ ln=__LINE__; goto btree_corrupt; }
+    pc = x;
+  }
+
+  /* Write this one page */
+  scrubBackupWrite(p, pgno, a);
+
+  /* Walk the tree and process child pages */
+  for(i=0; i<nCell; i++){
+    u32 X, M, K, nLocal;
+    sqlite3_int64 P;
+    pc = scrubBackupInt16(&aCell[i*2]);
+    if( pc <= szHdr ){ ln=__LINE__; goto btree_corrupt; }
+    if( pc > p->szUsable-3 ){ ln=__LINE__; goto btree_corrupt; }
+    if( aTop[0]==0x05 || aTop[0]==0x02 ){
+      if( pc+4 > p->szUsable ){ ln=__LINE__; goto btree_corrupt; }
+      iChild = scrubBackupInt32(&a[pc]);
+      pc += 4;
+      scrubBackupBtree(p, iChild, iDepth+1);
+      if( aTop[0]==0x05 ) continue;
+    }
+    pc += scrubBackupVarint(&a[pc], &P);
+    if( pc >= p->szUsable ){ ln=__LINE__; goto btree_corrupt; }
+    if( aTop[0]==0x0d ){
+      X = p->szUsable - 35;
+    }else{
+      X = ((p->szUsable - 12)*64/255) - 23;
+    }
+    if( P<=X ){
+      /* All content is local.  No overflow */
+      continue;
+    }
+    M = ((p->szUsable - 12)*32/255)-23;
+    K = M + ((P-M)%(p->szUsable-4));
+    if( aTop[0]==0x0d ){
+      pc += scrubBackupVarintSize(&a[pc]);
+      if( pc > (p->szUsable-4) ){ ln=__LINE__; goto btree_corrupt; }
+    }
+    nLocal = K<=X ? K : M;
+    if( pc+nLocal > p->szUsable-4 ){ ln=__LINE__; goto btree_corrupt; }
+    iChild = scrubBackupInt32(&a[pc+nLocal]);
+    scrubBackupOverflow(p, iChild, P-nLocal);
+  }
+
+  /* Walk the right-most tree */
+  if( aTop[0]==0x05 || aTop[0]==0x02 ){
+    iChild = scrubBackupInt32(&aTop[8]);
+    scrubBackupBtree(p, iChild, iDepth+1);
+  }
+
+  /* All done */
+  if( pgno>1 ) sqlite3_free(a);
+  return;
+
+btree_corrupt:
+  scrubBackupErr(p, "corruption on page %d of source database (errid=%d)",
+                 pgno, ln);
+  if( pgno>1 ) sqlite3_free(a);  
+}
+
+/*
+** Copy all ptrmap pages from source to destination.
+** This routine is only called if the source database is in autovacuum
+** or incremental vacuum mode.
+*/
+static void scrubBackupPtrmap(ScrubState *p){
+  u32 pgno = 2;
+  u32 J = p->szUsable/5;
+  u32 iLock = (1073742335/p->szPage)+1;
+  u8 *a, *pBuf;
+  if( p->rcErr ) return;
+  pBuf = scrubBackupAllocPage(p);
+  if( pBuf==0 ) return;
+  while( pgno<=p->nPage ){
+    a = scrubBackupRead(p, pgno, pBuf);
+    if( a==0 ) break;
+    scrubBackupWrite(p, pgno, a);
+    pgno += J+1;
+    if( pgno==iLock ) pgno++;
+  }
+  sqlite3_free(pBuf);
+}
+
+int sqlite3_scrub_backup(
+  const char *zSrcFile,    /* Source file */
+  const char *zDestFile,   /* Destination file */
+  char **pzErr             /* Write error here if non-NULL */
+){
+  ScrubState s;
+  u32 n, i;
+  sqlite3_stmt *pStmt;
+
+  memset(&s, 0, sizeof(s));
+  s.zSrcFile = zSrcFile;
+  s.zDestFile = zDestFile;
+
+  /* Open both source and destination databases */
+  scrubBackupOpenSrc(&s);
+  scrubBackupOpenDest(&s);
+
+  /* Read in page 1 */
+  s.page1 = scrubBackupRead(&s, 1, 0);
+  if( s.page1==0 ) goto scrub_abort;
+  s.szUsable = s.szPage - s.page1[20];
+
+  /* Copy the freelist */    
+  n = scrubBackupInt32(&s.page1[36]);
+  i = scrubBackupInt32(&s.page1[32]);
+  if( n ) scrubBackupFreelist(&s, i, n);
+
+  /* Copy ptrmap pages */
+  n = scrubBackupInt32(&s.page1[52]);
+  if( n ) scrubBackupPtrmap(&s);
+
+  /* Copy all of the btrees */
+  scrubBackupBtree(&s, 1, 0);
+  pStmt = scrubBackupPrepare(&s, s.dbSrc,
+       "SELECT rootpage FROM sqlite_master WHERE coalesce(rootpage,0)>0");
+  if( pStmt==0 ) goto scrub_abort;
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    i = (u32)sqlite3_column_int(pStmt, 0);
+    scrubBackupBtree(&s, i, 0);
+  }
+  sqlite3_finalize(pStmt);
+
+scrub_abort:    
+  /* Close the destination database without closing the transaction. If we
+  ** commit, page zero will be overwritten. */
+  sqlite3_close(s.dbDest);
+
+  /* But do close out the read-transaction on the source database */
+  sqlite3_exec(s.dbSrc, "COMMIT;", 0, 0, 0);
+  sqlite3_close(s.dbSrc);
+  sqlite3_free(s.page1);
+  if( pzErr ){
+    *pzErr = s.zErr;
+  }else{
+    sqlite3_free(s.zErr);
+  }
+  return s.rcErr;
+}   
+
+#ifdef SCRUB_STANDALONE
+/* Error and warning log */
+static void errorLogCallback(void *pNotUsed, int iErr, const char *zMsg){
+  const char *zType;
+  switch( iErr&0xff ){
+    case SQLITE_WARNING: zType = "WARNING";  break;
+    case SQLITE_NOTICE:  zType = "NOTICE";   break;
+    default:             zType = "ERROR";    break;
+  }
+  fprintf(stderr, "%s: %s\n", zType, zMsg);
+}
+
+/* The main() routine when this utility is run as a stand-alone program */
+int main(int argc, char **argv){
+  char *zErr = 0;
+  int rc;
+  if( argc!=3 ){
+    fprintf(stderr,"Usage: %s SOURCE DESTINATION\n", argv[0]);
+    exit(1);
+  }
+  sqlite3_config(SQLITE_CONFIG_LOG, errorLogCallback, 0);
+  rc = sqlite3_scrub_backup(argv[1], argv[2], &zErr);
+  if( rc==SQLITE_NOMEM ){
+    fprintf(stderr, "%s: out of memory\n", argv[0]);
+    exit(1);
+  }
+  if( zErr ){
+    fprintf(stderr, "%s: %s\n", argv[0], zErr);
+    sqlite3_free(zErr);
+    exit(1);
+  }
+  return 0;
+}
+#endif

main.mk

@@ -487,6 +487,9 @@ sqldiff$(EXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
 	$(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \
 		$(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB)
 
+scrub$(EXE):	$(TOP)/ext/misc/scrub.c sqlite3.o
+	$(TCC) -I. -DSCRUB_STANDALONE -o scrub$(EXE) $(TOP)/ext/misc/scrub.c sqlite3.o $(THREADLIB)
+
 srcck1$(EXE):	$(TOP)/tool/srcck1.c
 	$(BCC) -o srcck1$(EXE) $(TOP)/tool/srcck1.c
 

manifest

@@ -1,8 +1,8 @@
-C For\sin-memory\sdatabases,\sit\sdoes\snot\smatter\sif\spcache\sentries\sare\smarked\n"clean"\sor\s"writable".
-D 2016-05-11T23:54:14.261
-F Makefile.in 9eda6e1c90d05c199c3ec8a7069b0682ad307657
+C Enhance\sthe\sscrub\sutility\sprogram\sso\sthat\sit\sdoes\sa\sFULL\scheckpoint\sprior\sto\nstarting\sthe\sbackup,\sto\sensure\sthat\sthe\sdatabase\sfile\scontent\smatches\swhat\sneeds\nto\sbe\sbacked\sup\swithout\shaving\sto\slook\sat\sthe\sWAL\sfile.
+D 2016-05-17T21:17:51.530
+F Makefile.in f59e0763ff448719fc1bd25513882b0567286317
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc db82b35aef27f412fef14d8534afc022138bcdfd
+F Makefile.msc 306d73e854b1a92ea06e5d1e637faa5c44de53c7
 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7
 F VERSION 5d234da9b5dae329fab75ff75884cfe0a9cb3fda
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -215,6 +215,7 @@ F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63
 F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4
 F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a
+F ext/misc/scrub.c ea0903701e3ac02b4466ce9cffd0325c7ddbfcf0
 F ext/misc/series.c e11e534ada797d5b816d7e7a93c022306563ca35
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c bf1b922c2750698e9a3d4c50cce6974adb7e93be
@@ -302,7 +303,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 86e6a8d79186ace5ff8b306093da1728d953f796
+F main.mk eab7db27ac2a32b1172a85aa850c0b9c3a00524c
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -364,11 +365,11 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa
 F src/os_unix.c a9443cdab41d7f3cdf0df3a5aab62fd6e1c9b234
 F src/os_win.c 852fc2ff6084296348ed3739c548b2cf32df394e
 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca
-F src/pager.c 76fe5ad6998772a6abda30145c181e99e7743a1d
+F src/pager.c 3910579bfbe323dfabed2b95d201159b61b8ef42
 F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681
 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e
-F src/pcache.c 132a78c71bd510fde0085bfc44608bc64b945d48
-F src/pcache.h 33b40350df1b6c278e019dee37f87e1bac276223
+F src/pcache.c f398c0084399e7481482cbc6a578a3cc4c3675f3
+F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490
 F src/pcache1.c 7f51d2b541aab57596adf62db2c4bb025d34f04d
 F src/pragma.c faf42922bb7ab2f6672cb550356c1967abae3c84
 F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c
@@ -382,7 +383,7 @@ F src/shell.c 14ff7f660530a52b117d110ba3390b7b2eb719b6
 F src/sqlite.h.in 9984129d86243424b765fcb3f147c697bd20bb54
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 98f72cbfe00169c39089115427d06ea05fe4b4a2
-F src/sqliteInt.h 8ec741711b8f42aae2c7a22e3d04f15270e0b6be
+F src/sqliteInt.h 4fe51e4e26d9d576040198cef4f78df3d987387d
 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247
 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba
 F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9
@@ -977,7 +978,7 @@ F test/parser1.test 222b5cbf3e2e659fec1bf7d723488c8b9c94f1d0
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test b6b3e165fdc1b8c82a820033646dbfc6a7a01746
+F test/permutations.test 544a2f317fc5045bc512ae432f89eae303c0d640
 F test/pragma.test 1e94755164a3a3264cd39836de4bebcb7809e5f8
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
@@ -1053,7 +1054,7 @@ F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5
 F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e
 F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506
 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304
-F test/shell1.test 023657b3f8ba108dbd5ff2ebc8fceb3cf6d4ff9f
+F test/shell1.test c90b0415cea6eeefb86b4ab1651b06247922ca52
 F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
 F test/shell3.test da513d522ef6f01cee8475dcf8332bff8982b3dd
 F test/shell4.test 69995ee1cc278eb149aa8746ce1f935f4eaf98b9
@@ -1488,10 +1489,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3bfd2621d13b4f842f3af6d35519653f4eb8cad7
-R 399bfb6002bea3bb0fdfe001a40139f7
-T *branch * pager-dev
-T *sym-pager-dev *
-T -sym-trunk *
+P 5021dfe1f3f723a5938d547a0308f1d63103702d
+R a2f9971160f157164223781dabeb5e95
 U drh
-Z 75fe10a9420951c23350bd3808039177
+Z 2fde01a79f5811045a1dffd48050935f

manifest.uuid

@@ -1 +1 @@
-ad601c7962a60a11699cf31f1faee09b95d5c710
+ab1c5ce50f139070d7a322f43132a93c8af2ed68

src/pager.c

@@ -1876,20 +1876,24 @@ static int pager_error(Pager *pPager, int rc){
 static int pager_truncate(Pager *pPager, Pgno nPage);
 
 /*
-** The write transaction open on the pager passed as the only argument is
-** being committed. This function returns true if all dirty pages should
-** be flushed to disk, or false otherwise. Pages should be flushed to disk
-** unless one of the following is true:
+** The write transaction open on pPager is being committed (bCommit==1)
+** or rolled back (bCommit==0).
 **
-**   * The db is an in-memory database.
+** Return TRUE if and only if all dirty pages should be flushed to disk.
 **
-**   * The db is a temporary database and the db file has not been opened.
+** Rules:
 **
-**   * The db is a temporary database and the cache contains less than
-**     C/4 dirty pages, where C is the configured cache-size.
+**   *  For non-TEMP databases, always sync to disk.  This is necessary
+**      for transactions to be durable.
+**
+**   *  Sync TEMP database only on a COMMIT (not a ROLLBACK) when the backing
+**      file has been created already (via a spill on pagerStress()) and
+**      when the number of dirty pages in memory exceeds 25% of the total
+**      cache size.
 */
-static int pagerFlushOnCommit(Pager *pPager){
+static int pagerFlushOnCommit(Pager *pPager, int bCommit){
   if( pPager->tempFile==0 ) return 1;
+  if( !bCommit ) return 0;
   if( !isOpen(pPager->fd) ) return 0;
   return (sqlite3PCachePercentDirty(pPager->pPCache)>=25);
 }
@@ -2033,7 +2037,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){
   pPager->pInJournal = 0;
   pPager->nRec = 0;
   if( rc==SQLITE_OK ){
-    if( pagerFlushOnCommit(pPager) ){
+    if( pagerFlushOnCommit(pPager, bCommit) ){
       sqlite3PcacheCleanAll(pPager->pPCache);
     }else{
       sqlite3PcacheClearWritable(pPager->pPCache);
@@ -2373,7 +2377,6 @@ static int pager_playback_one_page(
     assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)!=0 );
     pPager->doNotSpill &= ~SPILLFLAG_ROLLBACK;
     if( rc!=SQLITE_OK ) return rc;
-    pPg->flags &= ~PGHDR_NEED_READ;
     sqlite3PcacheMakeDirty(pPg);
   }
   if( pPg ){
@@ -2387,33 +2390,10 @@ static int pager_playback_one_page(
     pData = pPg->pData;
     memcpy(pData, (u8*)aData, pPager->pageSize);
     pPager->xReiniter(pPg);
-    if( isMainJrnl && (!isSavepnt || *pOffset<=pPager->journalHdr) ){
-      /* If the contents of this page were just restored from the main 
-      ** journal file, then its content must be as they were when the 
-      ** transaction was first opened. In this case we can mark the page
-      ** as clean, since there will be no need to write it out to the
-      ** database.
-      **
-      ** There is one exception to this rule. If the page is being rolled
-      ** back as part of a savepoint (or statement) rollback from an 
-      ** unsynced portion of the main journal file, then it is not safe
-      ** to mark the page as clean. This is because marking the page as
-      ** clean will clear the PGHDR_NEED_SYNC flag. Since the page is
-      ** already in the journal file (recorded in Pager.pInJournal) and
-      ** the PGHDR_NEED_SYNC flag is cleared, if the page is written to
-      ** again within this transaction, it will be marked as dirty but
-      ** the PGHDR_NEED_SYNC flag will not be set. It could then potentially
-      ** be written out into the database file before its journal file
-      ** segment is synced. If a crash occurs during or following this,
-      ** database corruption may ensue.
-      **
-      ** Update: Another exception is for temp files that are not 
-      ** in-memory databases. In this case the page may have been dirty
-      ** at the start of the transaction.
-      */
-      assert( !pagerUseWal(pPager) );
-      if( pPager->tempFile==0 ) sqlite3PcacheMakeClean(pPg);
-    }
+    /* It used to be that sqlite3PcacheMakeClean(pPg) was called here.  But
+    ** that call was dangerous and had no detectable benefit since the cache
+    ** is normally cleaned by sqlite3PcacheCleanAll() after rollback and so
+    ** has been removed. */
     pager_set_pagehash(pPg);
 
     /* If this was page 1, then restore the value of Pager.dbFileVers.
@@ -6024,6 +6004,7 @@ void sqlite3PagerDontWrite(PgHdr *pPg){
     IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno))
     pPg->flags |= PGHDR_DONT_WRITE;
     pPg->flags &= ~PGHDR_WRITEABLE;
+    testcase( pPg->flags & PGHDR_NEED_SYNC );
     pager_set_pagehash(pPg);
   }
 }
@@ -6219,7 +6200,7 @@ int sqlite3PagerCommitPhaseOne(
   if( NEVER(pPager->errCode) ) return pPager->errCode;
 
   /* Provide the ability to easily simulate an I/O error during testing */
-  if( (rc = sqlite3FaultSim(400))!=SQLITE_OK ) return rc;
+  if( sqlite3FaultSim(400) ) return SQLITE_IOERR;
 
   PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", 
       pPager->zFilename, zMaster, pPager->dbSize));
@@ -6229,7 +6210,7 @@ int sqlite3PagerCommitPhaseOne(
 
   assert( MEMDB==0 || pPager->tempFile );
   assert( isOpen(pPager->fd) || pPager->tempFile );
-  if( 0==pagerFlushOnCommit(pPager) ){
+  if( 0==pagerFlushOnCommit(pPager, 1) ){
     /* If this is an in-memory db, or no pages have been written to, or this
     ** function has already been called, it is mostly a no-op.  However, any
     ** backup in progress needs to be restarted.  */
@@ -6854,6 +6835,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){
   /* In order to be able to rollback, an in-memory database must journal
   ** the page we are moving from.
   */
+  assert( pPager->tempFile || !MEMDB );
   if( pPager->tempFile ){
     rc = sqlite3PagerWrite(pPg);
     if( rc ) return rc;

src/pcache.c

@@ -53,16 +53,95 @@ struct PCache {
   sqlite3_pcache *pCache;             /* Pluggable cache module */
 };
 
+/********************************** Test and Debug Logic **********************/
 /*
-** Debug tracing macros
+** Debug tracing macros.  Enable by by changing the "0" to "1" and
+** recompiling.
+**
+** When sqlite3PcacheTrace is 1, single line trace messages are issued.
+** When sqlite3PcacheTrace is 2, a dump of the pcache showing all cache entries
+** is displayed for many operations, resulting in a lot of output.
 */
 #if defined(SQLITE_DEBUG) && 0
-  int sqlite3PcacheTrace = 1;
+  int sqlite3PcacheTrace = 2;       /* 0: off  1: simple  2: cache dumps */
+  int sqlite3PcacheMxDump = 9999;   /* Max cache entries for pcacheDump() */
 # define pcacheTrace(X) if(sqlite3PcacheTrace){sqlite3DebugPrintf X;}
-#else
+  void pcacheDump(PCache *pCache){
+    int N;
+    int i, j;
+    sqlite3_pcache_page *pLower;
+    PgHdr *pPg;
+    unsigned char *a;
+  
+    if( sqlite3PcacheTrace<2 ) return;
+    if( pCache->pCache==0 ) return;
+    N = sqlite3PcachePagecount(pCache);
+    if( N>sqlite3PcacheMxDump ) N = sqlite3PcacheMxDump;
+    for(i=1; i<=N; i++){
+       pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0);
+       if( pLower==0 ) continue;
+       pPg = (PgHdr*)pLower->pExtra;
+       printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags);
+       a = (unsigned char *)pLower->pBuf;
+       for(j=0; j<12; j++) printf("%02x", a[j]);
+       printf("\n");
+       if( pPg->pPage==0 ){
+         sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0);
+       }
+    }
+  }
+  #else
 # define pcacheTrace(X)
+# define pcacheDump(X)
 #endif
 
+/*
+** Check invariants on a PgHdr entry.  Return true if everything is OK.
+** Return false if any invariant is violated.
+**
+** This routine is for use inside of assert() statements only.  For
+** example:
+**
+**          assert( sqlite3PcachePageSanity(pPg) );
+*/
+#if SQLITE_DEBUG
+int sqlite3PcachePageSanity(PgHdr *pPg){
+  PCache *pCache;
+  assert( pPg!=0 );
+  assert( pPg->pgno>0 );    /* Page number is 1 or more */
+  pCache = pPg->pCache;
+  assert( pCache!=0 );      /* Every page has an associated PCache */
+  if( pPg->flags & PGHDR_CLEAN ){
+    assert( (pPg->flags & PGHDR_DIRTY)==0 );/* Cannot be both CLEAN and DIRTY */
+    assert( pCache->pDirty!=pPg );          /* CLEAN pages not on dirty list */
+    assert( pCache->pDirtyTail!=pPg );
+  }
+  /* WRITEABLE pages must also be DIRTY */
+  if( pPg->flags & PGHDR_WRITEABLE ){
+    assert( pPg->flags & PGHDR_DIRTY );     /* WRITEABLE implies DIRTY */
+  }
+  /* NEED_SYNC can be set independently of WRITEABLE.  This can happen,
+  ** for example, when using the sqlite3PagerDontWrite() optimization:
+  **    (1)  Page X is journalled, and gets WRITEABLE and NEED_SEEK.
+  **    (2)  Page X moved to freelist, WRITEABLE is cleared
+  **    (3)  Page X reused, WRITEABLE is set again
+  ** If NEED_SYNC had been cleared in step 2, then it would not be reset
+  ** in step 3, and page might be written into the database without first
+  ** syncing the rollback journal, which might cause corruption on a power
+  ** loss.
+  **
+  ** Another example is when the database page size is smaller than the
+  ** disk sector size.  When any page of a sector is journalled, all pages
+  ** in that sector are marked NEED_SYNC even if they are still CLEAN, just
+  ** in case they are later modified, since all pages in the same sector
+  ** must be journalled and synced before any of those pages can be safely
+  ** written.
+  */
+  return 1;
+}
+#endif /* SQLITE_DEBUG */
+
+
 /********************************** Linked List Management ********************/
 
 /* Allowed values for second argument to pcacheManageDirtyList() */
@@ -130,10 +209,19 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){
       }
     }
     p->pDirty = pPage;
-    if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){
+
+    /* If pSynced is NULL and this page has a clear NEED_SYNC flag, set
+    ** pSynced to point to it. Checking the NEED_SYNC flag is an 
+    ** optimization, as if pSynced points to a page with the NEED_SYNC
+    ** flag set sqlite3PcacheFetchStress() searches through all newer 
+    ** entries of the dirty-list for a page with NEED_SYNC clear anyway.  */
+    if( !p->pSynced 
+     && 0==(pPage->flags&PGHDR_NEED_SYNC)   /*OPTIMIZATION-IF-FALSE*/
+    ){
       p->pSynced = pPage;
     }
   }
+  pcacheDump(p);
 }
 
 /*
@@ -144,6 +232,7 @@ static void pcacheUnpin(PgHdr *p){
   if( p->pCache->bPurgeable ){
     pcacheTrace(("%p.UNPIN %d\n", p->pCache, p->pgno));
     sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 0);
+    pcacheDump(p->pCache);
   }
 }
 
@@ -344,6 +433,7 @@ int sqlite3PcacheFetchStress(
 #endif
       pcacheTrace(("%p.SPILL %d\n",pCache,pPg->pgno));
       rc = pCache->xStress(pCache->pStress, pPg);
+      pcacheDump(pCache);
       if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){
         return rc;
       }
@@ -403,6 +493,7 @@ PgHdr *sqlite3PcacheFetchFinish(
   }
   pCache->nRefSum++;
   pPgHdr->nRef++;
+  assert( sqlite3PcachePageSanity(pPgHdr) );
   return pPgHdr;
 }
 
@@ -416,8 +507,11 @@ void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){
   if( (--p->nRef)==0 ){
     if( p->flags&PGHDR_CLEAN ){
       pcacheUnpin(p);
-    }else if( p->pDirtyPrev!=0 ){
-      /* Move the page to the head of the dirty list. */
+    }else if( p->pDirtyPrev!=0 ){ /*OPTIMIZATION-IF-FALSE*/
+      /* Move the page to the head of the dirty list. If p->pDirtyPrev==0,
+      ** then page p is already at the head of the dirty list and the
+      ** following call would be a no-op. Hence the OPTIMIZATION-IF-FALSE
+      ** tag above.  */
       pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT);
     }
   }
@@ -428,6 +522,7 @@ void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){
 */
 void sqlite3PcacheRef(PgHdr *p){
   assert(p->nRef>0);
+  assert( sqlite3PcachePageSanity(p) );
   p->nRef++;
   p->pCache->nRefSum++;
 }
@@ -439,6 +534,7 @@ void sqlite3PcacheRef(PgHdr *p){
 */
 void sqlite3PcacheDrop(PgHdr *p){
   assert( p->nRef==1 );
+  assert( sqlite3PcachePageSanity(p) );
   if( p->flags&PGHDR_DIRTY ){
     pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE);
   }
@@ -452,7 +548,8 @@ void sqlite3PcacheDrop(PgHdr *p){
 */
 void sqlite3PcacheMakeDirty(PgHdr *p){
   assert( p->nRef>0 );
-  if( p->flags & (PGHDR_CLEAN|PGHDR_DONT_WRITE) ){
+  assert( sqlite3PcachePageSanity(p) );
+  if( p->flags & (PGHDR_CLEAN|PGHDR_DONT_WRITE) ){    /*OPTIMIZATION-IF-FALSE*/
     p->flags &= ~PGHDR_DONT_WRITE;
     if( p->flags & PGHDR_CLEAN ){
       p->flags ^= (PGHDR_DIRTY|PGHDR_CLEAN);
@@ -460,6 +557,7 @@ void sqlite3PcacheMakeDirty(PgHdr *p){
       assert( (p->flags & (PGHDR_DIRTY|PGHDR_CLEAN))==PGHDR_DIRTY );
       pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD);
     }
+    assert( sqlite3PcachePageSanity(p) );
   }
 }
 
@@ -468,12 +566,14 @@ void sqlite3PcacheMakeDirty(PgHdr *p){
 ** make it so.
 */
 void sqlite3PcacheMakeClean(PgHdr *p){
-  if( (p->flags & PGHDR_DIRTY) ){
+  assert( sqlite3PcachePageSanity(p) );
+  if( ALWAYS((p->flags & PGHDR_DIRTY)!=0) ){
     assert( (p->flags & PGHDR_CLEAN)==0 );
     pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE);
     p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC|PGHDR_WRITEABLE);
     p->flags |= PGHDR_CLEAN;
     pcacheTrace(("%p.CLEAN %d\n",p->pCache,p->pgno));
+    assert( sqlite3PcachePageSanity(p) );
     if( p->nRef==0 ){
       pcacheUnpin(p);
     }
@@ -485,6 +585,7 @@ void sqlite3PcacheMakeClean(PgHdr *p){
 */
 void sqlite3PcacheCleanAll(PCache *pCache){
   PgHdr *p;
+  pcacheTrace(("%p.CLEAN-ALL\n",pCache));
   while( (p = pCache->pDirty)!=0 ){
     sqlite3PcacheMakeClean(p);
   }
@@ -495,6 +596,7 @@ void sqlite3PcacheCleanAll(PCache *pCache){
 */
 void sqlite3PcacheClearWritable(PCache *pCache){
   PgHdr *p;
+  pcacheTrace(("%p.CLEAR-WRITEABLE\n",pCache));
   for(p=pCache->pDirty; p; p=p->pDirtyNext){
     p->flags &= ~(PGHDR_NEED_SYNC|PGHDR_WRITEABLE);
   }
@@ -519,6 +621,7 @@ void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){
   PCache *pCache = p->pCache;
   assert( p->nRef>0 );
   assert( newPgno>0 );
+  assert( sqlite3PcachePageSanity(p) );
   pcacheTrace(("%p.MOVE %d -> %d\n",pCache,p->pgno,newPgno));
   sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno);
   p->pgno = newPgno;

src/pcache.h

@@ -51,11 +51,10 @@ struct PgHdr {
 #define PGHDR_WRITEABLE       0x004  /* Journaled and ready to modify */
 #define PGHDR_NEED_SYNC       0x008  /* Fsync the rollback journal before
                                      ** writing this page to the database */
-#define PGHDR_NEED_READ       0x010  /* Content is unread */
-#define PGHDR_DONT_WRITE      0x020  /* Do not write content to disk */
-#define PGHDR_MMAP            0x040  /* This is an mmap page object */
+#define PGHDR_DONT_WRITE      0x010  /* Do not write content to disk */
+#define PGHDR_MMAP            0x020  /* This is an mmap page object */
 
-#define PGHDR_WAL_APPEND      0x080  /* Appended to wal file */
+#define PGHDR_WAL_APPEND      0x040  /* Appended to wal file */
 
 /* Initialize and shutdown the page cache subsystem */
 int sqlite3PcacheInitialize(void);
@@ -138,6 +137,11 @@ int sqlite3PcachePagecount(PCache*);
 void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *));
 #endif
 
+#if defined(SQLITE_DEBUG)
+/* Check invariants on a PgHdr object */
+int sqlite3PcachePageSanity(PgHdr*);
+#endif
+
 /* Set and get the suggested cache-size for the specified pager-cache.
 **
 ** If no global maximum is configured, then the system attempts to limit

src/sqliteInt.h

@@ -411,7 +411,7 @@
 ** be true and false so that the unreachable code they specify will
 ** not be counted as untested code.
 */
-#if defined(SQLITE_COVERAGE_TEST)
+#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
 # define ALWAYS(X)      (1)
 # define NEVER(X)       (0)
 #elif !defined(NDEBUG)

test/permutations.test

@@ -746,6 +746,11 @@ test_suite "inmemory_journal" -description {
 
   # WAL mode is different.
   wal* tkt-2d1a5c67d.test backcompat.test e_wal* rowallock.test
+
+  # This test does not work as the "PRAGMA journal_mode = memory"
+  # statement switches the database out of wal mode at inopportune
+  # times.
+  snapshot_fault.test
 }]
 
 ifcapable mem3 {

test/shell1.test

@@ -915,6 +915,9 @@ do_test shell1-5.0 {
   }
 } {}
 
+# These test cases do not work on MinGW
+if 0 {
+
 # The string used here is the word "test" in Chinese.
 # In UTF-8, it is encoded as: \xE6\xB5\x8B\xE8\xAF\x95
 set test \u6D4B\u8BD5
@@ -951,5 +954,6 @@ do_test shell1-6.1 {
   }
   forcedelete test3.db
 } {}
+}
 
 finish_test