From 3be8b1a4f7848c1d67324893f4ac9cace8c06eb0 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 21 Oct 2017 12:59:27 +0000 Subject: [PATCH 1/6] Initial implementation of the appendvfs extension. Untested. FossilOrigin-Name: 063a03a3779e8c032dd006712facaaa6d60964425701ea10c753ff981a8f2bd9 --- ext/misc/appendvfs.c | 453 +++++++++++++++++++++++++++++++++++++++++++ manifest | 17 +- manifest.uuid | 2 +- 3 files changed, 464 insertions(+), 8 deletions(-) create mode 100644 ext/misc/appendvfs.c diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c new file mode 100644 index 0000000000..b650b80be3 --- /dev/null +++ b/ext/misc/appendvfs.c @@ -0,0 +1,453 @@ +/* +** 2017-10-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a VFS shim that allows an SQLite database to be +** appended onto the end of some other file, such as an executable. +** +** A special record must appear at the end of the file that identifies the +** file as an appended database and provides an offset to page 1. For +** best performance page 1 should be located at a disk page boundary, though +** that is not required. +** +** An appended database is considered immutable. It is read-only and no +** locks are ever taken. +** +** If the file being opened is not an appended database, then this shim is +** a pass-through into the default underlying VFS. +**/ +#include +SQLITE_EXTENSION_INIT1 +#include +#include + +/* The append mark at the end of the database is: +** +** Start-Of-SQLite3-NNNNNNNN +** 123456789 123456789 12345 +** +** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is +** the offset to page 1. +*/ +#define APND_MARK_PREFIX "Start-Of-SQLite3-" +#define APND_MARK_PREFIX_SZ 17 +#define APND_MARK_SIZE 25 + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs ApndVfs; +typedef struct ApndFile ApndFile; + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) +#define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1)) + +/* An open file */ +struct ApndFile { + sqlite3_file base; /* IO methods */ + sqlite3_int64 iPgOne; /* File offset to page 1 */ +}; + +/* +** Methods for ApndFile +*/ +static int apndClose(sqlite3_file*); +static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int apndTruncate(sqlite3_file*, sqlite3_int64 size); +static int apndSync(sqlite3_file*, int flags); +static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int apndLock(sqlite3_file*, int); +static int apndUnlock(sqlite3_file*, int); +static int apndCheckReservedLock(sqlite3_file*, int *pResOut); +static int apndFileControl(sqlite3_file*, int op, void *pArg); +static int apndSectorSize(sqlite3_file*); +static int apndDeviceCharacteristics(sqlite3_file*); +static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int apndShmLock(sqlite3_file*, int offset, int n, int flags); +static void apndShmBarrier(sqlite3_file*); +static int apndShmUnmap(sqlite3_file*, int deleteFlag); +static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for ApndVfs +*/ +static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *apndDlOpen(sqlite3_vfs*, const char *zFilename); +static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void apndDlClose(sqlite3_vfs*, void*); +static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int apndSleep(sqlite3_vfs*, int microseconds); +static int apndCurrentTime(sqlite3_vfs*, double*); +static int apndGetLastError(sqlite3_vfs*, int, char *); +static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); +static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z); +static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName); + +static sqlite3_vfs apnd_vfs = { + 3, /* iVersion (set when registered) */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "apndvfs", /* zName */ + 0, /* pAppData (set when registered) */ + apndOpen, /* xOpen */ + apndDelete, /* xDelete */ + apndAccess, /* xAccess */ + apndFullPathname, /* xFullPathname */ + apndDlOpen, /* xDlOpen */ + apndDlError, /* xDlError */ + apndDlSym, /* xDlSym */ + apndDlClose, /* xDlClose */ + apndRandomness, /* xRandomness */ + apndSleep, /* xSleep */ + apndCurrentTime, /* xCurrentTime */ + apndGetLastError, /* xGetLastError */ + apndCurrentTimeInt64, /* xCurrentTimeInt64 */ + apndSetSystemCall, /* xSetSystemCall */ + apndGetSystemCall, /* xGetSystemCall */ + apndNextSystemCall /* xNextSystemCall */ +}; + +static const sqlite3_io_methods apnd_io_methods = { + 3, /* iVersion */ + apndClose, /* xClose */ + apndRead, /* xRead */ + apndWrite, /* xWrite */ + apndTruncate, /* xTruncate */ + apndSync, /* xSync */ + apndFileSize, /* xFileSize */ + apndLock, /* xLock */ + apndUnlock, /* xUnlock */ + apndCheckReservedLock, /* xCheckReservedLock */ + apndFileControl, /* xFileControl */ + apndSectorSize, /* xSectorSize */ + apndDeviceCharacteristics, /* xDeviceCharacteristics */ + apndShmMap, /* xShmMap */ + apndShmLock, /* xShmLock */ + apndShmBarrier, /* xShmBarrier */ + apndShmUnmap, /* xShmUnmap */ + apndFetch, /* xFetch */ + apndUnfetch /* xUnfetch */ +}; + + + +/* +** Close an apnd-file. +*/ +static int apndClose(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xClose(pFile); +} + +/* +** Read data from an apnd-file. +*/ +static int apndRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + ApndFile *p = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne); +} + +/* +** Write data to an apnd-file. +*/ +static int apndWrite( + sqlite3_file *pFile, + const void *z, + int iAmt, + sqlite_int64 iOfst +){ + return SQLITE_READONLY; +} + +/* +** Truncate an apnd-file. +*/ +static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){ + return SQLITE_READONLY; +} + +/* +** Sync an apnd-file. +*/ +static int apndSync(sqlite3_file *pFile, int flags){ + return SQLITE_READONLY; +} + +/* +** Return the current file-size of an apnd-file. +*/ +static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + ApndFile *p = (ApndFile*)pFile; + int rc; + pFile = ORIGFILE(p); + rc = pFile->pMethods->xFileSize(pFile, pSize); + if( rc==SQLITE_OK && p->iPgOne ){ + *pSize -= p->iPgOne + APND_MARK_SIZE; + } + return rc; +} + +/* +** Lock an apnd-file. +*/ +static int apndLock(sqlite3_file *pFile, int eLock){ + return SQLITE_READONLY; +} + +/* +** Unlock an apnd-file. +*/ +static int apndUnlock(sqlite3_file *pFile, int eLock){ + return SQLITE_OK; +} + +/* +** Check if another file-handle holds a RESERVED lock on an apnd-file. +*/ +static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + *pResOut = 0; + return SQLITE_OK; +} + +/* +** File control method. For custom operations on an apnd-file. +*/ +static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){ + ApndFile *p = (ApndFile *)pFile; + int rc; + pFile = ORIGFILE(pFile); + rc = pFile->pMethods->xFileControl(pFile, op, pArg); + if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", p->iPgOne, *(char**)pArg); + } + return rc; +} + +/* +** Return the sector-size in bytes for an apnd-file. +*/ +static int apndSectorSize(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSectorSize(pFile); +} + +/* +** Return the device characteristic flags supported by an apnd-file. +*/ +static int apndDeviceCharacteristics(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return SQLITE_IOCAP_IMMUTABLE | + pFile->pMethods->xDeviceCharacteristics(pFile); +} + +/* Create a shared memory file mapping */ +static int apndShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + return SQLITE_READONLY; +} + +/* Perform locking on a shared-memory segment */ +static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + return SQLITE_READONLY; +} + +/* Memory barrier operation on shared memory */ +static void apndShmBarrier(sqlite3_file *pFile){ + return; +} + +/* Unmap a shared memory segment */ +static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){ + return SQLITE_OK; +} + +/* Fetch a page of a memory-mapped file */ +static int apndFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + ApndFile *p = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp); +} + +/* Release a memory-mapped page */ +static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + ApndFile *p = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage); +} + +/* +** Open an apnd file handle. +*/ +static int apndOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + ApndFile *p; + int rc; + sqlite3_int64 sz; + pVfs = ORIGVFS(pVfs); + if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){ + return pVfs->xOpen(pVfs, zName, pFile, flags, pOutFlags); + } + p = (ApndFile*)pFile; + memset(p, 0, sizeof(*p)); + p->base.pMethods = &apnd_io_methods; + pFile = ORIGFILE(pFile); + flags &= ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE); + flags |= SQLITE_OPEN_READONLY; + rc = pVfs->xOpen(pVfs, zName, pFile, flags, pOutFlags); + if( rc ) return rc; + rc = pFile->pMethods->xFileSize(pFile, &sz); + if( rc==SQLITE_OK && sz>512 ){ + unsigned char a[APND_MARK_SIZE]; + rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE); + if( rc==SQLITE_OK + && memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)==0 + ){ + p->iPgOne = + ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ]<<56) + + ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+1]<<48) + + ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+2]<<40) + + ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+3]<<32) + + ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+4]<<24) + + ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+5]<<16) + + ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+6]<<8) + + ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+7]); + } + } + return SQLITE_OK; +} + +/* +** All other VFS methods are pass-thrus. +*/ +static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync); +} +static int apndAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut); +} +static int apndFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut); +} +static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} +static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} +static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} +static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} +static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} +static int apndSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} +static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} +static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} +static int apndSetSystemCall( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_syscall_ptr pCall +){ + return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall); +} +static sqlite3_syscall_ptr apndGetSystemCall( + sqlite3_vfs *pVfs, + const char *zName +){ + return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName); +} +static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ + return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName); +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called when the extension is loaded. +** Register the new VFS. +*/ +int sqlite3_appendvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + sqlite3_vfs *pOrig; + SQLITE_EXTENSION_INIT2(pApi); + pOrig = sqlite3_vfs_find(0); + apnd_vfs.iVersion = pOrig->iVersion; + apnd_vfs.pAppData = pOrig; + apnd_vfs.szOsFile = sizeof(ApndFile); + rc = sqlite3_vfs_register(&apnd_vfs, 1); +#ifdef APPENDVFS_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister); + } +#endif + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} diff --git a/manifest b/manifest index 5c8ae4d13c..e9413c8f80 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Take\sextra\scare\sto\savoid\san\sOOB\sread\scaused\sby\sa\scorrupt\sb-tree\spage. -D 2017-10-19T15:17:38.752 +C Initial\simplementation\sof\sthe\sappendvfs\sextension.\s\sUntested. +D 2017-10-21T12:59:27.233 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -258,6 +258,7 @@ F ext/lsm1/test/lsm1_simple.test ca949efefa102f4644231dcd9291d8cda7699a4ce1006b2 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb +F ext/misc/appendvfs.c 487a5db4427d118413346780c4c90d0256315e8de3fb528ee54daef3bb73276f F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f @@ -1664,8 +1665,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cd0471ca9f75e7c8be74536ff4ec85b5d70f0d7994b680ed5f45b9f12a46cf46 -R 14641e8822b464d39bf472f899e19240 -T +closed 9dd591ef24b302a5fe2af0619d0cda6733348bacc541b3c0a134ac25981d4b2a -U dan -Z 576e55698c218b4128bbad5de39dc1b0 +P 04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b +R b5d8b4b601b02e4902be4e639107c749 +T *branch * appendvfs +T *sym-appendvfs * +T -sym-trunk * +U drh +Z 11816656e34dd3c798cb5d3f2287708a diff --git a/manifest.uuid b/manifest.uuid index 26a2375f60..69ef83190d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b \ No newline at end of file +063a03a3779e8c032dd006712facaaa6d60964425701ea10c753ff981a8f2bd9 \ No newline at end of file From e483d349ab20b49cf704423f3d15e30d722f25e2 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 14 Dec 2017 16:28:27 +0000 Subject: [PATCH 2/6] Add the ability to write to an appended database. This check-in compiles but is otherwise untested. FossilOrigin-Name: e343c63cbd754f37c33c939cd0b6f1ecc6202e60c6e66cd65c23cc8d571a994e --- ext/misc/appendvfs.c | 187 ++++++++++++++++++++++++++++++++++--------- manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 155 insertions(+), 46 deletions(-) diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c index b650b80be3..d0096f1224 100644 --- a/ext/misc/appendvfs.c +++ b/ext/misc/appendvfs.c @@ -18,8 +18,31 @@ ** best performance page 1 should be located at a disk page boundary, though ** that is not required. ** -** An appended database is considered immutable. It is read-only and no -** locks are ever taken. +** When opening a database using this VFS, the connection might treat +** the file as an ordinary SQLite database, or it might treat is as a +** database appended onto some other file. Here are the rules: +** +** (1) When opening a new empty file, that file is treated as an ordinary +** database. +** +** (2) When opening a file that begins with the standard SQLite prefix +** string "SQLite format 3", that file is treated as an ordinary +** database. +** +** (3) When opening a file that ends with the appendvfs trailer string +** "Start-Of-SQLite3-NNNNNNNN" that file is treated as an appended +** database. +** +** (4) If none of the above apply and the SQLITE_OPEN_CREATE flag is +** set, then a new database is appended to the already existing file. +** +** (5) Otherwise, SQLITE_CANTOPEN is returned. +** +** To avoid unnecessary complications with the PENDING_BYTE, the size of +** the file containing the database is limited to 1GB. This VFS will refuse +** to read or write past the 1GB mark. This restriction might be lifted in +** future versions. For now, if you need a large database, then keep the +** database in a separate file. ** ** If the file being opened is not an appended database, then this shim is ** a pass-through into the default underlying VFS. @@ -41,6 +64,12 @@ SQLITE_EXTENSION_INIT1 #define APND_MARK_PREFIX_SZ 17 #define APND_MARK_SIZE 25 +/* +** Maximum size of the combined prefix + database + append-mark. This +** must be less than 0x40000000 to avoid locking issues on Windows. +*/ +#define APND_MAX_SIZE (65536*15259) + /* ** Forward declaration of objects used by this utility */ @@ -57,6 +86,7 @@ typedef struct ApndFile ApndFile; struct ApndFile { sqlite3_file base; /* IO methods */ sqlite3_int64 iPgOne; /* File offset to page 1 */ + sqlite3_int64 iMark; /* Start of the append-mark */ }; /* @@ -172,37 +202,75 @@ static int apndRead( return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne); } +/* +** Add the append-mark onto the end of the file. +*/ +static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){ + int i; + unsigned char a[APND_MARK_SIZE]; + memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ); + for(i=0; i<8; i++){ + a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff; + } + return pFile->pMethods->xWrite(pFile, a, APND_MARK_PREFIX_SZ, p->iMark); +} + /* ** Write data to an apnd-file. */ static int apndWrite( sqlite3_file *pFile, - const void *z, + const void *zBuf, int iAmt, sqlite_int64 iOfst ){ - return SQLITE_READONLY; + int rc; + ApndFile *p = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + if( iOfst+iAmt>=APND_MAX_SIZE ) return SQLITE_FULL; + rc = pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne); + if( rc==SQLITE_OK && iOfst + iAmt + p->iPgOne > p->iMark ){ + sqlite3_int64 sz = 0; + rc = pFile->pMethods->xFileSize(pFile, &sz); + if( rc==SQLITE_OK ){ + p->iMark = sz - APND_MARK_SIZE; + if( iOfst + iAmt + p->iPgOne > p->iMark ){ + p->iMark = p->iPgOne + iOfst + iAmt; + rc = apndWriteMark(p, pFile); + } + } + } + return rc; } /* ** Truncate an apnd-file. */ static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){ - return SQLITE_READONLY; + int rc; + ApndFile *p = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE); + if( rc==SQLITE_OK ){ + p->iMark = p->iPgOne+size; + rc = apndWriteMark(p, pFile); + } + return rc; } /* ** Sync an apnd-file. */ static int apndSync(sqlite3_file *pFile, int flags){ - return SQLITE_READONLY; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSync(pFile, flags); } /* ** Return the current file-size of an apnd-file. */ static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ - ApndFile *p = (ApndFile*)pFile; + ApndFile *p = (ApndFile *)pFile; int rc; pFile = ORIGFILE(p); rc = pFile->pMethods->xFileSize(pFile, pSize); @@ -216,22 +284,24 @@ static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ ** Lock an apnd-file. */ static int apndLock(sqlite3_file *pFile, int eLock){ - return SQLITE_READONLY; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xLock(pFile, eLock); } /* ** Unlock an apnd-file. */ static int apndUnlock(sqlite3_file *pFile, int eLock){ - return SQLITE_OK; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnlock(pFile, eLock); } /* ** Check if another file-handle holds a RESERVED lock on an apnd-file. */ static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){ - *pResOut = 0; - return SQLITE_OK; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xCheckReservedLock(pFile, pResOut); } /* @@ -273,22 +343,26 @@ static int apndShmMap( int bExtend, void volatile **pp ){ - return SQLITE_READONLY; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp); } /* Perform locking on a shared-memory segment */ static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){ - return SQLITE_READONLY; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmLock(pFile,offset,n,flags); } /* Memory barrier operation on shared memory */ static void apndShmBarrier(sqlite3_file *pFile){ - return; + pFile = ORIGFILE(pFile); + pFile->pMethods->xShmBarrier(pFile); } /* Unmap a shared memory segment */ static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){ - return SQLITE_OK; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmUnmap(pFile,deleteFlag); } /* Fetch a page of a memory-mapped file */ @@ -310,6 +384,40 @@ static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage); } +/* +** Check to see if the file is an ordinary SQLite database file. +*/ +static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){ + int rc; + char zHdr[16]; + static const char aSqliteHdr[] = "SQLite format 3"; + if( sz<512 ) return 0; + rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0); + if( rc ) return 0; + return memcmp(zHdr, aSqliteHdr, sizeof(zHdr))==0; +} + +/* +** Try to read the append-mark off the end of a file. Return the +** start of the appended database if the append-mark is present. If +** there is no append-mark, return -1; +*/ +static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){ + int rc, i; + sqlite3_int64 iMark; + unsigned char a[APND_MARK_SIZE]; + + if( sz<=APND_MARK_SIZE ) return -1; + rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE); + if( rc ) return -1; + if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1; + iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ]&0x7f))<<56; + for(i=1; i<8; i++){ + iMark += (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<<(56-8*i); + } + return iMark; +} + /* ** Open an apnd file handle. */ @@ -321,38 +429,39 @@ static int apndOpen( int *pOutFlags ){ ApndFile *p; + sqlite3_file *pSubFile; + sqlite3_vfs *pSubVfs; int rc; sqlite3_int64 sz; - pVfs = ORIGVFS(pVfs); + pSubVfs = ORIGVFS(pVfs); if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){ - return pVfs->xOpen(pVfs, zName, pFile, flags, pOutFlags); + return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); } p = (ApndFile*)pFile; memset(p, 0, sizeof(*p)); p->base.pMethods = &apnd_io_methods; - pFile = ORIGFILE(pFile); - flags &= ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE); - flags |= SQLITE_OPEN_READONLY; - rc = pVfs->xOpen(pVfs, zName, pFile, flags, pOutFlags); + pSubFile = ORIGFILE(pFile); + rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); if( rc ) return rc; - rc = pFile->pMethods->xFileSize(pFile, &sz); - if( rc==SQLITE_OK && sz>512 ){ - unsigned char a[APND_MARK_SIZE]; - rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE); - if( rc==SQLITE_OK - && memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)==0 - ){ - p->iPgOne = - ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ]<<56) + - ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+1]<<48) + - ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+2]<<40) + - ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+3]<<32) + - ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+4]<<24) + - ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+5]<<16) + - ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+6]<<8) + - ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+7]); - } + rc = pSubFile->pMethods->xFileSize(pSubFile, &sz); + if( rc ){ + pSubFile->pMethods->xClose(pSubFile); + return rc; } + if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){ + memmove(pFile, pSubFile, pSubVfs->szOsFile); + return SQLITE_OK; + } + p->iMark = 0; + p->iPgOne = apndReadMark(sz, pFile); + if( p->iPgOne>0 ){ + return SQLITE_OK; + } + if( (flags & SQLITE_OPEN_CREATE)==0 ){ + pSubFile->pMethods->xClose(pSubFile); + return SQLITE_CANTOPEN; + } + p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff; return SQLITE_OK; } @@ -442,7 +551,7 @@ int sqlite3_appendvfs_init( apnd_vfs.iVersion = pOrig->iVersion; apnd_vfs.pAppData = pOrig; apnd_vfs.szOsFile = sizeof(ApndFile); - rc = sqlite3_vfs_register(&apnd_vfs, 1); + rc = sqlite3_vfs_register(&apnd_vfs, 0); #ifdef APPENDVFS_TEST if( rc==SQLITE_OK ){ rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister); diff --git a/manifest b/manifest index 1b36095a3f..4ddbb0d1d5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bring\sin\sthe\slatest\strunk\schanges. -D 2017-12-14T14:50:49.020 +C Add\sthe\sability\sto\swrite\sto\san\sappended\sdatabase.\s\sThis\scheck-in\scompiles\nbut\sis\sotherwise\suntested. +D 2017-12-14T16:28:27.146 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a2492b29176edc3c754aa7a2f7daa20cd3fa20a56e3ee64e376092836177c42a @@ -260,7 +260,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb -F ext/misc/appendvfs.c 487a5db4427d118413346780c4c90d0256315e8de3fb528ee54daef3bb73276f +F ext/misc/appendvfs.c c8b18caae0d22762080a9c48a4e46933f48625bac7fde2af9ef8ec2a23755ac9 F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 @@ -1681,7 +1681,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1a1a73b821eb1a22ca335f582a0aea31c71ca9f5b09d54f26409691c90e38c4a 3765aaf712998af5ffb6bc680a0c1419f2b5deb47ecbc1835ba5879127c4dbe3 -R 29a07f33957881a5b837419ff7f21683 +P 75d8517703f7efa33437079108e2c4ef0de1a118bbe1f4a86afdc34da09d3008 +R 15eddfa7673da0a2579cc83183e243e0 U drh -Z d18bd4436a6dfca865f7b12ca3ce5865 +Z 4e88658420f57b7b81dd23d0961f71ee diff --git a/manifest.uuid b/manifest.uuid index 29cb2dde87..026378badf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75d8517703f7efa33437079108e2c4ef0de1a118bbe1f4a86afdc34da09d3008 \ No newline at end of file +e343c63cbd754f37c33c939cd0b6f1ecc6202e60c6e66cd65c23cc8d571a994e \ No newline at end of file From 233ff96ec5beae1301731a3c0893f8cf358c86f0 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 14 Dec 2017 16:57:11 +0000 Subject: [PATCH 3/6] The "apndvfs" VFS shim now appears to be working for both reads and writes. FossilOrigin-Name: 7f7b72d83633922e2b5dbf0d3455d0fea72cc6b8925ffcb78dfbad7b3c4b26e9 --- ext/misc/appendvfs.c | 7 +++---- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c index d0096f1224..bd1f37509e 100644 --- a/ext/misc/appendvfs.c +++ b/ext/misc/appendvfs.c @@ -212,7 +212,7 @@ static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){ for(i=0; i<8; i++){ a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff; } - return pFile->pMethods->xWrite(pFile, a, APND_MARK_PREFIX_SZ, p->iMark); + return pFile->pMethods->xWrite(pFile, a, APND_MARK_SIZE, p->iMark); } /* @@ -331,8 +331,7 @@ static int apndSectorSize(sqlite3_file *pFile){ */ static int apndDeviceCharacteristics(sqlite3_file *pFile){ pFile = ORIGFILE(pFile); - return SQLITE_IOCAP_IMMUTABLE | - pFile->pMethods->xDeviceCharacteristics(pFile); + return pFile->pMethods->xDeviceCharacteristics(pFile); } /* Create a shared memory file mapping */ @@ -550,7 +549,7 @@ int sqlite3_appendvfs_init( pOrig = sqlite3_vfs_find(0); apnd_vfs.iVersion = pOrig->iVersion; apnd_vfs.pAppData = pOrig; - apnd_vfs.szOsFile = sizeof(ApndFile); + apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile); rc = sqlite3_vfs_register(&apnd_vfs, 0); #ifdef APPENDVFS_TEST if( rc==SQLITE_OK ){ diff --git a/manifest b/manifest index 4ddbb0d1d5..c5da940c02 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sability\sto\swrite\sto\san\sappended\sdatabase.\s\sThis\scheck-in\scompiles\nbut\sis\sotherwise\suntested. -D 2017-12-14T16:28:27.146 +C The\s"apndvfs"\sVFS\sshim\snow\sappears\sto\sbe\sworking\sfor\sboth\sreads\sand\swrites. +D 2017-12-14T16:57:11.644 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a2492b29176edc3c754aa7a2f7daa20cd3fa20a56e3ee64e376092836177c42a @@ -260,7 +260,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb -F ext/misc/appendvfs.c c8b18caae0d22762080a9c48a4e46933f48625bac7fde2af9ef8ec2a23755ac9 +F ext/misc/appendvfs.c ee71fbc918ab4c526a5935a3153c1cc4e0b601c081da8f9a549e0e9d01bdbca5 F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 @@ -1681,7 +1681,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 75d8517703f7efa33437079108e2c4ef0de1a118bbe1f4a86afdc34da09d3008 -R 15eddfa7673da0a2579cc83183e243e0 +P e343c63cbd754f37c33c939cd0b6f1ecc6202e60c6e66cd65c23cc8d571a994e +R 427fca803286f3061d4cdb1346b5bd46 U drh -Z 4e88658420f57b7b81dd23d0961f71ee +Z 919006b84ebd33ef7833311db2cf1160 diff --git a/manifest.uuid b/manifest.uuid index 026378badf..566e5b932a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e343c63cbd754f37c33c939cd0b6f1ecc6202e60c6e66cd65c23cc8d571a994e \ No newline at end of file +7f7b72d83633922e2b5dbf0d3455d0fea72cc6b8925ffcb78dfbad7b3c4b26e9 \ No newline at end of file From dd2a43a225a69a53bbdfe17eec319bbb85e674f9 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 14 Dec 2017 19:24:00 +0000 Subject: [PATCH 4/6] Fixes to the appendvfs.c extension. Add the "sqltclsh" application that uses appendvfs.c to find its scripts. FossilOrigin-Name: ee248b529c2396c5480fb99b0a1dc31032627ec8241eca4a8c0fff257bb4a088 --- Makefile.in | 9 +++++- ext/misc/appendvfs.c | 3 ++ main.mk | 9 +++++- manifest | 18 ++++++------ manifest.uuid | 2 +- tool/sqltclsh.c.in | 42 ++++++++++++++++++++++++++++ tool/sqltclsh.tcl | 66 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 138 insertions(+), 11 deletions(-) create mode 100644 tool/sqltclsh.c.in create mode 100644 tool/sqltclsh.tcl diff --git a/Makefile.in b/Makefile.in index 4883c1ac6b..5d377174f8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -556,7 +556,8 @@ TESTPROGS = \ sqlite3$(TEXE) \ sqlite3_analyzer$(TEXE) \ sqldiff$(TEXE) \ - dbhash$(TEXE) + dbhash$(TEXE) \ + sqltclsh$(TEXE) # Databases containing fuzzer test cases # @@ -1190,6 +1191,12 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $ sqlite3_analyzer$(TEXE): sqlite3_analyzer.c $(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS) +sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in + $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c + +sqltclsh$(TEXE): sqltclsh.c + $(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS) + CHECKER_DEPS =\ $(TOP)/tool/mkccode.tcl \ sqlite3.c \ diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c index bd1f37509e..c6e9e0d58b 100644 --- a/ext/misc/appendvfs.c +++ b/ext/misc/appendvfs.c @@ -47,7 +47,9 @@ ** If the file being opened is not an appended database, then this shim is ** a pass-through into the default underlying VFS. **/ +#if !defined(SQLITEINT_H) #include +#endif SQLITE_EXTENSION_INIT1 #include #include @@ -458,6 +460,7 @@ static int apndOpen( } if( (flags & SQLITE_OPEN_CREATE)==0 ){ pSubFile->pMethods->xClose(pSubFile); + pFile->pMethods = 0; return SQLITE_CANTOPEN; } p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff; diff --git a/main.mk b/main.mk index e978b23a58..bc4e404020 100644 --- a/main.mk +++ b/main.mk @@ -484,7 +484,8 @@ TESTPROGS = \ sqlite3_analyzer$(EXE) \ sqlite3_checker$(EXE) \ sqldiff$(EXE) \ - dbhash$(EXE) + dbhash$(EXE) \ + sqltclsh$(EXE) # Databases containing fuzzer test cases # @@ -809,6 +810,12 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $ sqlite3_analyzer$(EXE): sqlite3_analyzer.c $(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) +sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl + tclsh $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c + +sqltclsh$(EXE): sqltclsh.c + $(TCCX) $(TCL_FLAGS) sqltclsh.c -o $@ $(LIBTCL) $(THREADLIB) + CHECKER_DEPS =\ $(TOP)/tool/mkccode.tcl \ sqlite3.c \ diff --git a/manifest b/manifest index c5da940c02..743670d293 100644 --- a/manifest +++ b/manifest @@ -1,6 +1,6 @@ -C The\s"apndvfs"\sVFS\sshim\snow\sappears\sto\sbe\sworking\sfor\sboth\sreads\sand\swrites. -D 2017-12-14T16:57:11.644 -F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8 +C Fixes\sto\sthe\sappendvfs.c\sextension.\s\sAdd\sthe\s"sqltclsh"\sapplication\sthat\nuses\sappendvfs.c\sto\sfind\sits\sscripts. +D 2017-12-14T19:24:00.444 +F Makefile.in 053284f237e955fba2f386fd9f87020199e6dbcbce9f48bc02cf431458afbe07 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a2492b29176edc3c754aa7a2f7daa20cd3fa20a56e3ee64e376092836177c42a F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681 @@ -260,7 +260,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb -F ext/misc/appendvfs.c ee71fbc918ab4c526a5935a3153c1cc4e0b601c081da8f9a549e0e9d01bdbca5 +F ext/misc/appendvfs.c 8cc3ae6633e7d97ca9c3c9a48c647e9ba23f8d24f3d5f5f596272a81dc50c398 F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 @@ -396,7 +396,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 6123b0b2db806ddb482c24786ad6603a289df720382a3bce8f532d76a94c84b1 +F main.mk d0278b8833203dc60ed81eed12412d90559fcd3470e0fd5d509d41ef74a3d64b F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1650,6 +1650,8 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/split-sqlite3c.tcl 3efcd4240b738f6bb2b5af0aea7e1e0ef9bc1c61654f645076cec883030b710c F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb F tool/sqlite3_analyzer.c.in 7eeaae8b0d7577662acaabbb11107af0659d1b41bc1dfdd4d91422de27127968 +F tool/sqltclsh.c.in e1f48150f755bfbe0194478cba50aa9f2f5183bb1efbdd6456532cce3cd2e18d +F tool/sqltclsh.tcl 71bdb62c8f6b37c05b2d377e870c3dd13ee96cc4fee5b5ae5215ae9a0aca088d F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d @@ -1681,7 +1683,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e343c63cbd754f37c33c939cd0b6f1ecc6202e60c6e66cd65c23cc8d571a994e -R 427fca803286f3061d4cdb1346b5bd46 +P 7f7b72d83633922e2b5dbf0d3455d0fea72cc6b8925ffcb78dfbad7b3c4b26e9 +R 4666f6f48678fbbcc815c221172efa48 U drh -Z 919006b84ebd33ef7833311db2cf1160 +Z 47302a3d5371e2c5e325f342d15c1218 diff --git a/manifest.uuid b/manifest.uuid index 566e5b932a..dcab13a743 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7f7b72d83633922e2b5dbf0d3455d0fea72cc6b8925ffcb78dfbad7b3c4b26e9 \ No newline at end of file +ee248b529c2396c5480fb99b0a1dc31032627ec8241eca4a8c0fff257bb4a088 \ No newline at end of file diff --git a/tool/sqltclsh.c.in b/tool/sqltclsh.c.in new file mode 100644 index 0000000000..bdfac7dd00 --- /dev/null +++ b/tool/sqltclsh.c.in @@ -0,0 +1,42 @@ +/* +** This is the source code to a "tclsh" that has SQLite built-in. +** +** The startup script is located as follows: +** +** (1) Open the executable as an appended SQLite database and try to +** read the startup script out of that database. +** +** (2) If the first argument is a readable file, try to open that file +** as an SQLite database and read the startup script out of that +** database. +** +** (3) If the first argument is a readable file with a ".tcl" extension, +** then try to run that script directly. +** +** If none of the above steps work, then the program runs as an interactive +** tclsh. +*/ +#define TCLSH_INIT_PROC sqlite3_tclapp_init_proc +#define SQLITE_ENABLE_DBSTAT_VTAB 1 +#undef SQLITE_THREADSAFE +#define SQLITE_THREADSAFE 0 +#undef SQLITE_ENABLE_COLUMN_METADATA +#define SQLITE_OMIT_DECLTYPE 1 +#define SQLITE_OMIT_DEPRECATED 1 +#define SQLITE_OMIT_PROGRESS_CALLBACK 1 +#define SQLITE_OMIT_SHARED_CACHE 1 +#define SQLITE_DEFAULT_MEMSTATUS 0 +#define SQLITE_MAX_EXPR_DEPTH 0 +INCLUDE sqlite3.c +INCLUDE $ROOT/ext/misc/appendvfs.c +INCLUDE $ROOT/src/tclsqlite.c + +const char *sqlite3_tclapp_init_proc(Tcl_Interp *interp){ + (void)interp; + sqlite3_appendvfs_init(0,0,0); + return +BEGIN_STRING +INCLUDE $ROOT/tool/sqltclsh.tcl +END_STRING +; +} diff --git a/tool/sqltclsh.tcl b/tool/sqltclsh.tcl new file mode 100644 index 0000000000..999f166ccb --- /dev/null +++ b/tool/sqltclsh.tcl @@ -0,0 +1,66 @@ +# Try to open the executable as a database and read the "scripts.data" +# field where "scripts.name" is 'main.tcl' +# +catch { + sqlite3 db $argv0 -vfs apndvfs -create 0 + set mainscript [db one {SELECT data FROM scripts WHERE name='main.tcl'}] +} +if {[info exists mainscript]} { + eval $mainscript + return +} else { + catch {db close} +} + +# Try to open file named in the first argument as a database and +# read the "scripts.data" field where "scripts.name" is 'main.tcl' +# +if {[llength $argv]>0 && [file readable [lindex $argv 0]]} { + catch { + sqlite3 db [lindex $argv 0] -vfs apndvfs -create 0 + set mainscript [db one {SELECT data FROM scripts WHERE name='main.tcl'}] + set argv0 [lindex $argv 0] + set argv [lrange $argv 1 end] + } + if {[info exists mainscript]} { + eval $mainscript + return + } else { + catch {db close} + } + if {[string match *.tcl [lindex $argv 0]]} { + set fd [open [lindex $argv 0] rb] + set mainscript [read $fd] + close $fd + unset fd + set argv0 [lindex $argv 0] + set argv [lrange $argv 1 end] + } + if {[info exists mainscript]} { + eval $mainscript + return + } +} + +# If all else fails, do an interactive loop +# +set line {} +while {![eof stdin]} { + if {$line!=""} { + puts -nonewline "> " + } else { + puts -nonewline "% " + } + flush stdout + append line [gets stdin] + if {[info complete $line]} { + if {[catch {uplevel #0 $line} result]} { + puts stderr "Error: $result" + } elseif {$result!=""} { + puts $result + } + set line {} + } else { + append line \\n" + } +} From 9dea173154d490489c7794f945a153d1a9de058b Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Jan 2018 04:34:05 +0000 Subject: [PATCH 5/6] Add sqltclsh.exe to the windows makefile. FossilOrigin-Name: a6d5c7c2aabd70b4fb2bd8f6278f70272a5bd166adf2f7225ea582e10a46fafa --- Makefile.msc | 11 ++++++++++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 652bd81fd0..5bea16eab0 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1495,7 +1495,8 @@ TESTPROGS = \ sqlite3_analyzer.exe \ sqlite3_checker.exe \ sqldiff.exe \ - dbhash.exe + dbhash.exe \ + sqltclsh.exe # Databases containing fuzzer test cases # @@ -2222,6 +2223,14 @@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) +sqltclsh.c: sqlite3.c $(TOP)\src\tclsqlite.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in + $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in >sqltclsh.c + +sqltclsh.exe: sqltclsh.c + $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqltclsh.c \ + /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LTLIBS) $(TLIBS) + + sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(LTLINK) $(NO_WARN) $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS) diff --git a/manifest b/manifest index c5aed3eac5..3f55a3838d 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Merge\srecent\senhancements\sfrom\strunk. -D 2017-12-23T18:40:39.094 +C Add\ssqltclsh.exe\sto\sthe\swindows\smakefile. +D 2018-01-06T04:34:05.887 F Makefile.in f2dc8c140e1d728157834da295eaaa8a0cb29620595c2a9f0efc7258797e6f24 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 +F Makefile.msc 80cca5fd5760cb3e26aaedaf6ef0df155cab3761a98efd0aa819973713839ca9 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -1690,7 +1690,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ee248b529c2396c5480fb99b0a1dc31032627ec8241eca4a8c0fff257bb4a088 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977 -R 38880472a25b77bfdeb87344d5313674 +P edceaccd66a65d6b36e53ce33d760a7bd9c2261a592d12189f5f55417b5d5d74 +R 724c729f15e9ebb158a68ffbc1395b1f U drh -Z 5274401ef77d35314ed9b1a10cd6e401 +Z adb127fe5f715fb70b7df6486f1fd9f2 diff --git a/manifest.uuid b/manifest.uuid index 9da9bac5e4..3d96106fb5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -edceaccd66a65d6b36e53ce33d760a7bd9c2261a592d12189f5f55417b5d5d74 \ No newline at end of file +a6d5c7c2aabd70b4fb2bd8f6278f70272a5bd166adf2f7225ea582e10a46fafa \ No newline at end of file From 6b9986e84a3e0b3edce6c7f5a7693c1d9d725616 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Jan 2018 13:33:21 +0000 Subject: [PATCH 6/6] Fix the appendvfs so that it leaves sqlite3_file->pMethods as NULL if it fails to open. FossilOrigin-Name: 46b341e3ad11b807ae50f001b970299de7ea0d523dbb639ee10f1d5aca5d958b --- ext/misc/appendvfs.c | 13 +++++++------ manifest | 14 +++++++------- manifest.uuid | 2 +- tool/sqltclsh.tcl | 3 +++ 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c index c6e9e0d58b..24d5e5f15b 100644 --- a/ext/misc/appendvfs.c +++ b/ext/misc/appendvfs.c @@ -440,14 +440,14 @@ static int apndOpen( } p = (ApndFile*)pFile; memset(p, 0, sizeof(*p)); - p->base.pMethods = &apnd_io_methods; pSubFile = ORIGFILE(pFile); + p->base.pMethods = &apnd_io_methods; rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); - if( rc ) return rc; + if( rc ) goto apnd_open_done; rc = pSubFile->pMethods->xFileSize(pSubFile, &sz); if( rc ){ pSubFile->pMethods->xClose(pSubFile); - return rc; + goto apnd_open_done; } if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){ memmove(pFile, pSubFile, pSubVfs->szOsFile); @@ -460,11 +460,12 @@ static int apndOpen( } if( (flags & SQLITE_OPEN_CREATE)==0 ){ pSubFile->pMethods->xClose(pSubFile); - pFile->pMethods = 0; - return SQLITE_CANTOPEN; + rc = SQLITE_CANTOPEN; } p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff; - return SQLITE_OK; +apnd_open_done: + if( rc ) pFile->pMethods = 0; + return rc; } /* diff --git a/manifest b/manifest index 3f55a3838d..f93f6fa978 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssqltclsh.exe\sto\sthe\swindows\smakefile. -D 2018-01-06T04:34:05.887 +C Fix\sthe\sappendvfs\sso\sthat\sit\sleaves\ssqlite3_file->pMethods\sas\sNULL\sif\sit\nfails\sto\sopen. +D 2018-01-06T13:33:21.118 F Makefile.in f2dc8c140e1d728157834da295eaaa8a0cb29620595c2a9f0efc7258797e6f24 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 80cca5fd5760cb3e26aaedaf6ef0df155cab3761a98efd0aa819973713839ca9 @@ -266,7 +266,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb -F ext/misc/appendvfs.c 8cc3ae6633e7d97ca9c3c9a48c647e9ba23f8d24f3d5f5f596272a81dc50c398 +F ext/misc/appendvfs.c 24b72d0556e81939c19316ca48c628938b4bd0ec81dac076406260c3ccd51def F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 @@ -1658,7 +1658,7 @@ F tool/split-sqlite3c.tcl 3efcd4240b738f6bb2b5af0aea7e1e0ef9bc1c61654f645076cec8 F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb F tool/sqlite3_analyzer.c.in 7eeaae8b0d7577662acaabbb11107af0659d1b41bc1dfdd4d91422de27127968 F tool/sqltclsh.c.in e1f48150f755bfbe0194478cba50aa9f2f5183bb1efbdd6456532cce3cd2e18d -F tool/sqltclsh.tcl 71bdb62c8f6b37c05b2d377e870c3dd13ee96cc4fee5b5ae5215ae9a0aca088d +F tool/sqltclsh.tcl 18adb7d4a24374a3c79a2acc011b5f54a22186beec13056b0fa3ca011711c4f3 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d @@ -1690,7 +1690,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P edceaccd66a65d6b36e53ce33d760a7bd9c2261a592d12189f5f55417b5d5d74 -R 724c729f15e9ebb158a68ffbc1395b1f +P a6d5c7c2aabd70b4fb2bd8f6278f70272a5bd166adf2f7225ea582e10a46fafa +R 11d42f816c24c039c3f505e6266a4bae U drh -Z adb127fe5f715fb70b7df6486f1fd9f2 +Z 83fb91d11f3679f988cee9a994d344b9 diff --git a/manifest.uuid b/manifest.uuid index 3d96106fb5..73d812be11 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a6d5c7c2aabd70b4fb2bd8f6278f70272a5bd166adf2f7225ea582e10a46fafa \ No newline at end of file +46b341e3ad11b807ae50f001b970299de7ea0d523dbb639ee10f1d5aca5d958b \ No newline at end of file diff --git a/tool/sqltclsh.tcl b/tool/sqltclsh.tcl index 999f166ccb..61d00f1df2 100644 --- a/tool/sqltclsh.tcl +++ b/tool/sqltclsh.tcl @@ -2,6 +2,9 @@ # field where "scripts.name" is 'main.tcl' # catch { + if {![file exists $argv0] && [file exists $argv0.exe]} { + append argv0 .exe + } sqlite3 db $argv0 -vfs apndvfs -create 0 set mainscript [db one {SELECT data FROM scripts WHERE name='main.tcl'}] }