paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial check-in of the "scrub.exe" utility program prototype. Not yet

Browse Source 
fully functional.  In particular, no scrubbing is done.

FossilOrigin-Name: bdf2ec77d1542d4e9b68218f558710a3efc15823
This commit is contained in:
drh
2016-05-05 17:15:23 +00:00
parent ad5a9d715f
commit da110bfece
6 changed files with 329 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile.in
+4
View File
@@ -589,6 +589,10 @@ sqlite3$(TEXE): $(TOP)/src/shell.c sqlite3.c
sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS)
scrub$(TEXE): $(TOP)/ext/misc/scrub.c sqlite3.o
$(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \
$(TOP)/ext/misc/scrub.c sqlite3.o $(TLIBS)
srcck1$(BEXE): $(TOP)/tool/srcck1.c
$(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c
Makefile.msc
+3
View File
@@ -1455,6 +1455,9 @@ $(SQLITE3EXE): $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_S
sqldiff.exe: $(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
scrub.exe: $(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H)
$(LTLINK) $(NO_WARN) $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
srcck1.exe: $(TOP)\tool\srcck1.c
$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c
ext/misc/scrub.c
+306
View File
@@ -0,0 +1,306 @@
/*
** 2016-05-05
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file implements a utility function (and a utility program) that
** makes a copy of an SQLite database while simultaneously zeroing out all
** deleted content.
**
** Normally (when PRAGMA secure_delete=OFF, which is the default) when SQLite
** deletes content, it does not overwrite the deleted content but rather marks
** the region of the file that held that content as being reusable. This can
** cause deleted content to recoverable from the database file. This stale
** content is removed by the VACUUM command, but VACUUM can be expensive for
** large databases. When in PRAGMA secure_delete=ON mode, the deleted content
** is zeroed, but secure_delete=ON has overhead as well.
**
** This utility attempts to make a copy of a complete SQLite database where
** all of the deleted content is zeroed out in the copy, and it attempts to
** do so while being faster than running VACUUM.
**
** Usage:
**
** int sqlite3_scrub_backup(
** const char *zSourceFile, // Source database filename
** const char *zDestFile, // Destination database filename
** char **pzErrMsg // Write error message here
** );
**
** Simply call the API above specifying the filename of the source database
** and the name of the backup copy. The source database must already exist
** and can be in active use. (A read lock is held during the backup.) The
** destination file should not previously exist. If the pzErrMsg parameter
** is non-NULL and if an error occurs, then an error message might be written
** into memory obtained from sqlite3_malloc() and *pzErrMsg made to point to
** that error message. But if the error is an OOM, the error might not be
** reported. The routine always returns non-zero if there is an error.
**
** If compiled with -DSCRUB_STANDALONE then a main() procedure is added and
** this file becomes a standalone program that can be run as follows:
**
** ./sqlite3scrub SOURCE DEST
*/
#include "sqlite3.h"
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <string.h>
typedef struct ScrubState ScrubState;
typedef unsigned char u8;
/* State information for a scrub-and-backup operation */
struct ScrubState {
const char *zSrcFile; /* Name of the source file */
const char *zDestFile; /* Name of the destination file */
int rcErr; /* Error code */
char *zErr; /* Error message text */
sqlite3 *dbSrc; /* Source database connection */
sqlite3_file *pSrc; /* Source file handle */
sqlite3 *dbDest; /* Destination database connection */
sqlite3_file *pDest; /* Destination file handle */
unsigned int szPage; /* Page size */
unsigned int nPage; /* Number of pages */
u8 *page1; /* Content of page 1 */
};
/* Store an error message */
static void scrubBackupErr(ScrubState *p, const char *zFormat, ...){
va_list ap;
sqlite3_free(p->zErr);
va_start(ap, zFormat);
p->zErr = sqlite3_vmprintf(zFormat, ap);
va_end(ap);
if( p->rcErr==0 ) p->rcErr = SQLITE_ERROR;
}
/* Allocate memory to hold a single page of content */
static u8 *scrubBackupAllocPage(ScrubState *p){
u8 *pPage;
if( p->rcErr ) return 0;
pPage = sqlite3_malloc( p->szPage );
if( pPage==0 ) p->rcErr = SQLITE_NOMEM;
return pPage;
}
/* Read a page from the source database into memory. Use the memory
** provided by pBuf if not NULL or allocate a new page if pBuf==NULL.
*/
static u8 *scrubBackupRead(ScrubState *p, int pgno, u8 *pBuf){
int rc;
sqlite3_int64 iOff;
u8 *pOut = pBuf;
if( p->rcErr ) return 0;
if( pOut==0 ){
pOut = scrubBackupAllocPage(p);
if( pOut==0 ) return 0;
}
iOff = (pgno-1)*(sqlite3_int64)p->szPage;
rc = p->pSrc->pMethods->xRead(p->pSrc, pOut, p->szPage, iOff);
if( rc!=SQLITE_OK ){
if( pBuf==0 ) sqlite3_free(pOut);
pOut = 0;
scrubBackupErr(p, "read failed for page %d", pgno);
p->rcErr = SQLITE_IOERR;
}
return pOut;
}
/* Write a page to the destination database */
static void scrubBackupWrite(ScrubState *p, int pgno, u8 *pData){
int rc;
sqlite3_int64 iOff;
if( p->rcErr ) return;
iOff = (pgno-1)*(sqlite3_int64)p->szPage;
rc = p->pDest->pMethods->xWrite(p->pDest, pData, p->szPage, iOff);
if( rc!=SQLITE_OK ){
scrubBackupErr(p, "write failed for page %d", pgno);
p->rcErr = SQLITE_IOERR;
}
}
/* Prepare a statement against the "db" database. */
static sqlite3_stmt *scrubBackupPrepare(
ScrubState *p, /* Backup context */
sqlite3 *db, /* Database to prepare against */
const char *zSql /* SQL statement */
){
sqlite3_stmt *pStmt;
if( p->rcErr ) return 0;
p->rcErr = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
if( p->rcErr ){
scrubBackupErr(p, "SQL error \"%s\" on \"%s\"",
sqlite3_errmsg(db), zSql);
sqlite3_finalize(pStmt);
return 0;
}
return pStmt;
}
/* Open the source database file */
static void scrubBackupOpenSrc(ScrubState *p){
sqlite3_stmt *pStmt;
int rc;
/* Open the source database file */
p->rcErr = sqlite3_open_v2(p->zSrcFile, &p->dbSrc,
SQLITE_OPEN_READONLY |
SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0);
if( p->rcErr ){
scrubBackupErr(p, "cannot open source database: %s",
sqlite3_errmsg(p->dbSrc));
return;
}
p->rcErr = sqlite3_exec(p->dbSrc, "BEGIN", 0, 0, 0);
if( p->rcErr ){
scrubBackupErr(p,
"cannot start a read transaction on the source database: %s",
sqlite3_errmsg(p->dbSrc));
return;
}
pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_size");
if( pStmt==0 ) return;
rc = sqlite3_step(pStmt);
if( rc==SQLITE_ROW ){
p->szPage = sqlite3_column_int(pStmt, 0);
}else{
scrubBackupErr(p, "unable to determine the page size");
}
sqlite3_finalize(pStmt);
if( p->rcErr ) return;
pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_count");
if( pStmt==0 ) return;
rc = sqlite3_step(pStmt);
if( rc==SQLITE_ROW ){
p->nPage = sqlite3_column_int(pStmt, 0);
}else{
scrubBackupErr(p, "unable to determine the size of the source database");
}
sqlite3_finalize(pStmt);
sqlite3_file_control(p->dbSrc, "main", SQLITE_FCNTL_FILE_POINTER, &p->pSrc);
if( p->pSrc==0 || p->pSrc->pMethods==0 ){
scrubBackupErr(p, "cannot get the source file handle");
p->rcErr = SQLITE_ERROR;
}
}
/* Create and open the destination file */
static void scrubBackupOpenDest(ScrubState *p){
sqlite3_stmt *pStmt;
int rc;
char *zSql;
if( p->rcErr ) return;
p->rcErr = sqlite3_open_v2(p->zDestFile, &p->dbDest,
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE |
SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0);
if( p->rcErr ){
scrubBackupErr(p, "cannot open destination database: %s",
sqlite3_errmsg(p->dbDest));
return;
}
zSql = sqlite3_mprintf("PRAGMA page_size(%u);", p->szPage);
if( zSql==0 ){
p->rcErr = SQLITE_NOMEM;
return;
}
p->rcErr = sqlite3_exec(p->dbDest, zSql, 0, 0, 0);
sqlite3_free(zSql);
if( p->rcErr ){
scrubBackupErr(p,
"cannot set the page size on the destination database: %s",
sqlite3_errmsg(p->dbDest));
return;
}
sqlite3_exec(p->dbDest, "PRAGMA journal_mode=OFF;", 0, 0, 0);
p->rcErr = sqlite3_exec(p->dbDest, "BEGIN EXCLUSIVE;", 0, 0, 0);
if( p->rcErr ){
scrubBackupErr(p,
"cannot start a write transaction on the destination database: %s",
sqlite3_errmsg(p->dbDest));
return;
}
pStmt = scrubBackupPrepare(p, p->dbDest, "PRAGMA page_count;");
if( pStmt==0 ) return;
rc = sqlite3_step(pStmt);
if( rc!=SQLITE_ROW ){
scrubBackupErr(p, "cannot measure the size of the destination");
}else if( sqlite3_column_int(pStmt, 0)>1 ){
scrubBackupErr(p, "destination database is not empty - holds %d pages",
sqlite3_column_int(pStmt, 0));
}
sqlite3_finalize(pStmt);
sqlite3_file_control(p->dbDest, "main", SQLITE_FCNTL_FILE_POINTER, &p->pDest);
if( p->pDest==0 || p->pDest->pMethods==0 ){
scrubBackupErr(p, "cannot get the destination file handle");
p->rcErr = SQLITE_ERROR;
}
}
int sqlite3_scrub_backup(
const char *zSrcFile, /* Source file */
const char *zDestFile, /* Destination file */
char **pzErr /* Write error here if non-NULL */
){
ScrubState s;
unsigned int i;
u8 *pBuf = 0;
u8 *pData;
memset(&s, 0, sizeof(s));
s.zSrcFile = zSrcFile;
s.zDestFile = zDestFile;
scrubBackupOpenSrc(&s);
scrubBackupOpenDest(&s);
pBuf = scrubBackupAllocPage(&s);
for(i=1; s.rcErr==0 && i<=s.nPage; i++){
pData = scrubBackupRead(&s, i, pBuf);
scrubBackupWrite(&s, i, pData);
}
/* Close the destination database without closing the transaction. If we
** commit, page zero will be overwritten. */
sqlite3_close(s.dbDest);
sqlite3_close(s.dbSrc);
sqlite3_free(s.page1);
if( pzErr ){
*pzErr = s.zErr;
}else{
sqlite3_free(s.zErr);
}
return s.rcErr;
}
#ifdef SCRUB_STANDALONE
/* The main() routine when this utility is run as a stand-alone program */
int main(int argc, char **argv){
char *zErr = 0;
int rc;
if( argc!=3 ){
fprintf(stderr,"Usage: %s SOURCE DESTINATION\n", argv[0]);
exit(1);
}
rc = sqlite3_scrub_backup(argv[1], argv[2], &zErr);
if( rc==SQLITE_NOMEM ){
fprintf(stderr, "%s: out of memory\n", argv[0]);
exit(1);
}
if( zErr ){
fprintf(stderr, "%s: %s\n", argv[0], zErr);
sqlite3_free(zErr);
exit(1);
}
return 0;
}
#endif
main.mk
+3
View File
@@ -488,6 +488,9 @@ sqldiff$(EXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
$(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \
$(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB)
scrub$(EXE): $(TOP)/ext/misc/scrub.c sqlite3.o
$(TCC) -I. -DSCRUB_STANDALONE -o scrub$(EXE) $(TOP)/ext/misc/scrub.c sqlite3.o $(THREADLIB)
srcck1$(EXE): $(TOP)/tool/srcck1.c
$(BCC) -o srcck1$(EXE) $(TOP)/tool/srcck1.c
manifest
+12 -8
View File
@@ -1,8 +1,8 @@
C Renumber\sinternal\sconstants\sin\sthe\sprintf()\simplemention\sfor\sa\ssmall\nperformance\simprovement.
D 2016-05-05T11:53:12.439
F Makefile.in 9eda6e1c90d05c199c3ec8a7069b0682ad307657
C Initial\scheck-in\sof\sthe\s"scrub.exe"\sutility\sprogram\sprototype.\s\sNot\syet\nfully\sfunctional.\s\sIn\sparticular,\sno\sscrubbing\sis\sdone.
D 2016-05-05T17:15:23.301
F Makefile.in f59e0763ff448719fc1bd25513882b0567286317
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc db82b35aef27f412fef14d8534afc022138bcdfd
F Makefile.msc 306d73e854b1a92ea06e5d1e637faa5c44de53c7
F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7
F VERSION 5d234da9b5dae329fab75ff75884cfe0a9cb3fda
F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -215,6 +215,7 @@ F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63
F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4
F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a
F ext/misc/scrub.c 76a9c795078a65bd021d5b5cd9e4655e7d0306c0
F ext/misc/series.c e11e534ada797d5b816d7e7a93c022306563ca35
F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
F ext/misc/spellfix.c bf1b922c2750698e9a3d4c50cce6974adb7e93be
@@ -302,7 +303,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
F main.mk a283660f75c3c4b75d8c9d12a40fa38a066eee9d
F main.mk 0fb2b051c9dbcf7f26a8455cf115ae9457448b93
F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1487,7 +1488,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 71af9ac165ac02272f4886f69bd9ab4770fd7bb6
R 89655379113728f544ce9a18ad294b0b
P 69d11447f4b1a8c536c3b6573d2a3419da870412
R 98e582093972832acc05b11fc975c102
T *branch * scrub-backup
T *sym-scrub-backup *
T -sym-trunk *
U drh
Z f65addcdc03169dedb02067b28293d6c
Z e2196386eaeffd3910bcf89f26dcb9bb
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
69d11447f4b1a8c536c3b6573d2a3419da870412
bdf2ec77d1542d4e9b68218f558710a3efc15823