paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add more tests for LSM log file recovery. Fix a problem in recovering log

Browse Source 
files that contain range deletes.

FossilOrigin-Name: e34eafd4c5b2bbf2735e136ad69b67bb4288ad4d01a0128d8e107ac46209a182
This commit is contained in:
dan
2017-07-03 09:00:18 +00:00
parent 24f6904e99
commit 05dda7ba8e
9 changed files with 107 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ext/lsm1/lsm-test/lsmtest1.c
+53 -20
View File
@@ -81,12 +81,12 @@ struct Datatest2 {
** Generate a unique name for the test case pTest with database system
** zSystem.
*/
static char *getName(const char *zSystem, Datatest1 *pTest){
static char *getName(const char *zSystem, int bRecover, Datatest1 *pTest){
char *zRet;
char *zData;
zData = testDatasourceName(&pTest->defn);
zRet = testMallocPrintf("data.%s.%s.%d.%d",
zSystem, zData, pTest->nRow, pTest->nVerify
zRet = testMallocPrintf("data.%s.%s.rec=%d.%d.%d",
zSystem, zData, bRecover, pTest->nRow, pTest->nVerify
);
testFree(zData);
return zRet;
@@ -251,8 +251,21 @@ static void printScanCb(
}
#endif
void testReopenRecover(TestDb **ppDb, int *pRc){
if( *pRc==0 ){
const char *zLib = tdb_library_name(*ppDb);
const char *zDflt = tdb_default_db(zLib);
testCopyLsmdb(zDflt, "bak.db");
testClose(ppDb);
testCopyLsmdb("bak.db", zDflt);
*pRc = tdb_open(zLib, 0, 0, ppDb);
}
}
static void doDataTest1(
const char *zSystem, /* Database system to test */
int bRecover,
Datatest1 *p, /* Structure containing test parameters */
int *pRc /* OUT: Error code */
){
@@ -277,8 +290,11 @@ static void doDataTest1(
/* Check that the db content is correct. */
testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
/* Close and reopen the database. */
testReopen(&pDb, &rc);
if( bRecover ){
testReopenRecover(&pDb, &rc);
}else{
testReopen(&pDb, &rc);
}
/* Check that the db content is still correct. */
testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
@@ -299,7 +315,11 @@ static void doDataTest1(
testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc);
/* Close and reopen the database. */
testReopen(&pDb, &rc);
if( bRecover ){
testReopenRecover(&pDb, &rc);
}else{
testReopen(&pDb, &rc);
}
/* Check that the db content is still correct. */
testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc);
@@ -339,13 +359,17 @@ void test_data_1(
};
int i;
int bRecover;
for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
char *zName = getName(zSystem, &aTest[i]);
if( testCaseBegin(pRc, zPattern, "%s", zName) ){
doDataTest1(zSystem, &aTest[i], pRc);
for(bRecover=0; bRecover<2; bRecover++){
if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break;
for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
char *zName = getName(zSystem, bRecover, &aTest[i]);
if( testCaseBegin(pRc, zPattern, "%s", zName) ){
doDataTest1(zSystem, bRecover, &aTest[i], pRc);
}
testFree(zName);
}
testFree(zName);
}
}
@@ -396,6 +420,7 @@ void testCompareDb(
static void doDataTest2(
const char *zSystem, /* Database system to test */
int bRecover,
Datatest2 *p, /* Structure containing test parameters */
int *pRc /* OUT: Error code */
){
@@ -437,7 +462,11 @@ static void doDataTest2(
testFree(pKey1);
testCompareDb(pData, nRange, i, pControl, pDb, &rc);
testReopen(&pDb, &rc);
if( bRecover ){
testReopenRecover(&pDb, &rc);
}else{
testReopen(&pDb, &rc);
}
testCompareDb(pData, nRange, i, pControl, pDb, &rc);
/* Update the progress dots... */
@@ -451,12 +480,12 @@ static void doDataTest2(
*pRc = rc;
}
static char *getName2(const char *zSystem, Datatest2 *pTest){
static char *getName2(const char *zSystem, int bRecover, Datatest2 *pTest){
char *zRet;
char *zData;
zData = testDatasourceName(&pTest->defn);
zRet = testMallocPrintf("data2.%s.%s.%d.%d.%d",
zSystem, zData, pTest->nRange, pTest->nWrite, pTest->nIter
zRet = testMallocPrintf("data2.%s.%s.rec=%d.%d.%d.%d",
zSystem, zData, bRecover, pTest->nRange, pTest->nWrite, pTest->nIter
);
testFree(zData);
return zRet;
@@ -476,13 +505,17 @@ void test_data_2(
};
int i;
int bRecover;
for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
char *zName = getName2(zSystem, &aTest[i]);
if( testCaseBegin(pRc, zPattern, "%s", zName) ){
doDataTest2(zSystem, &aTest[i], pRc);
for(bRecover=0; bRecover<2; bRecover++){
if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break;
for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
char *zName = getName2(zSystem, bRecover, &aTest[i]);
if( testCaseBegin(pRc, zPattern, "%s", zName) ){
doDataTest2(zSystem, bRecover, &aTest[i], pRc);
}
testFree(zName);
}
testFree(zName);
}
}
ext/lsm1/lsm-test/lsmtest_main.c
+1 -1
View File
@@ -26,7 +26,7 @@ void test_failed(){
static void testSetErrorFunc(int rc, int *pRc, const char *zFile, int iLine){
if( rc ){
*pRc = rc;
printf("FAILED (%s:%d) rc=%d ", zFile, iLine, rc);
fprintf(stderr, "FAILED (%s:%d) rc=%d ", zFile, iLine, rc);
test_failed();
}
}
ext/lsm1/lsm-test/lsmtest_tdb.c
+8
View File
@@ -745,6 +745,14 @@ const char *tdb_system_name(int i){
return aLib[i].zName;
}
const char *tdb_default_db(const char *zSys){
int i;
for(i=0; i<ArraySize(aLib); i++){
if( strcmp(aLib[i].zName, zSys)==0 ) return aLib[i].zDefaultDb;
}
return 0;
}
int tdb_open(const char *zLib, const char *zDb, int bClear, TestDb **ppDb){
int i;
int rc = 1;
ext/lsm1/lsm-test/lsmtest_tdb.h
+1
View File
@@ -125,6 +125,7 @@ int tdb_scan(
);
const char *tdb_system_name(int i);
const char *tdb_default_db(const char *zSys);
int tdb_lsm_open(const char *zCfg, const char *zDb, int bClear, TestDb **ppDb);
ext/lsm1/lsmInt.h
+5 -1
View File
@@ -864,7 +864,7 @@ int lsmInfoFreelist(lsm_db *pDb, char **pzOut);
** Functions from file "lsm_log.c".
*/
int lsmLogBegin(lsm_db *pDb);
int lsmLogWrite(lsm_db *, void *, int, void *, int);
int lsmLogWrite(lsm_db *, int, void *, int, void *, int);
int lsmLogCommit(lsm_db *);
void lsmLogEnd(lsm_db *pDb, int bCommit);
void lsmLogTell(lsm_db *, LogMark *);
@@ -874,6 +874,10 @@ void lsmLogClose(lsm_db *);
int lsmLogRecover(lsm_db *);
int lsmInfoLogStructure(lsm_db *pDb, char **pzVal);
/* Valid values for the second argument to lsmLogWrite(). */
#define LSM_WRITE 0x06
#define LSM_DELETE 0x08
#define LSM_DRANGE 0x0A
/**************************************************************************
** Functions from file "lsm_shared.c".
ext/lsm1/lsm_log.c
+24 -6
View File
@@ -199,9 +199,13 @@
#define LSM_LOG_WRITE 0x06
#define LSM_LOG_WRITE_CKSUM 0x07
#define LSM_LOG_DELETE 0x08
#define LSM_LOG_DELETE_CKSUM 0x09
#define LSM_LOG_DRANGE 0x0A
#define LSM_LOG_DRANGE_CKSUM 0x0B
/* Require a checksum every 32KB. */
#define LSM_CKSUM_MAXDATA (32*1024)
@@ -653,6 +657,7 @@ static int logFlush(lsm_db *pDb, int eType){
*/
int lsmLogWrite(
lsm_db *pDb, /* Database handle */
int eType,
void *pKey, int nKey, /* Database key to write to log */
void *pVal, int nVal /* Database value (or nVal<0) to write */
){
@@ -661,13 +666,19 @@ int lsmLogWrite(
int nReq; /* Bytes of space required in log */
int bCksum = 0; /* True to embed a checksum in this record */
assert( eType==LSM_WRITE || eType==LSM_DELETE || eType==LSM_DRANGE );
assert( LSM_LOG_WRITE==LSM_WRITE );
assert( LSM_LOG_DELETE==LSM_DELETE );
assert( LSM_LOG_DRANGE==LSM_DRANGE );
assert( (eType==LSM_LOG_DELETE)==(nVal<0) );
if( pDb->bUseLog==0 ) return LSM_OK;
pLog = pDb->pLogWriter;
/* Determine how many bytes of space are required, assuming that a checksum
** will be embedded in this record (even though it may not be). */
nReq = 1 + lsmVarintLen32(nKey) + 8 + nKey;
if( nVal>=0 ) nReq += lsmVarintLen32(nVal) + nVal;
if( eType!=LSM_LOG_DELETE ) nReq += lsmVarintLen32(nVal) + nVal;
/* Jump over the jump region if required. Set bCksum to true to tell the
** code below to include a checksum in the record if either (a) writing
@@ -687,9 +698,10 @@ int lsmLogWrite(
** DELETE) or 2 (for WRITE) varints. */
assert( LSM_LOG_WRITE_CKSUM == (LSM_LOG_WRITE | 0x0001) );
assert( LSM_LOG_DELETE_CKSUM == (LSM_LOG_DELETE | 0x0001) );
*(a++) = (nVal>=0 ? LSM_LOG_WRITE : LSM_LOG_DELETE) | (u8)bCksum;
assert( LSM_LOG_DRANGE_CKSUM == (LSM_LOG_DRANGE | 0x0001) );
*(a++) = (u8)eType | (u8)bCksum;
a += lsmVarintPut32(a, nKey);
if( nVal>=0 ) a += lsmVarintPut32(a, nVal);
if( eType!=LSM_LOG_DELETE ) a += lsmVarintPut32(a, nVal);
if( bCksum ){
pLog->buf.n = (a - (u8 *)pLog->buf.z);
@@ -699,7 +711,7 @@ int lsmLogWrite(
memcpy(a, pKey, nKey);
a += nKey;
if( nVal>=0 ){
if( eType!=LSM_LOG_DELETE ){
memcpy(a, pVal, nVal);
a += nVal;
}
@@ -1005,6 +1017,8 @@ int lsmLogRecover(lsm_db *pDb){
break;
}
case LSM_LOG_DRANGE:
case LSM_LOG_DRANGE_CKSUM:
case LSM_LOG_WRITE:
case LSM_LOG_WRITE_CKSUM: {
int nKey;
@@ -1013,7 +1027,7 @@ int lsmLogRecover(lsm_db *pDb){
logReaderVarint(&reader, &buf1, &nKey, &rc);
logReaderVarint(&reader, &buf2, &nVal, &rc);
if( eType==LSM_LOG_WRITE_CKSUM ){
if( eType==LSM_LOG_WRITE_CKSUM || eType==LSM_LOG_DRANGE_CKSUM ){
logReaderCksum(&reader, &buf1, &bEof, &rc);
}else{
bEof = logRequireCksum(&reader, nKey+nVal);
@@ -1023,7 +1037,11 @@ int lsmLogRecover(lsm_db *pDb){
logReaderBlob(&reader, &buf1, nKey, 0, &rc);
logReaderBlob(&reader, &buf2, nVal, &aVal, &rc);
if( iPass==1 && rc==LSM_OK ){
rc = lsmTreeInsert(pDb, (u8 *)buf1.z, nKey, aVal, nVal);
if( eType==LSM_LOG_WRITE || eType==LSM_LOG_WRITE_CKSUM ){
rc = lsmTreeInsert(pDb, (u8 *)buf1.z, nKey, aVal, nVal);
}else{
rc = lsmTreeDelete(pDb, (u8 *)buf1.z, nKey, aVal, nVal);
}
}
break;
}
ext/lsm1/lsm_main.c
+2 -5
View File
@@ -669,11 +669,8 @@ static int doWriteOp(
}
if( rc==LSM_OK ){
if( bDeleteRange==0 ){
rc = lsmLogWrite(pDb, (void *)pKey, nKey, (void *)pVal, nVal);
}else{
/* TODO */
}
int eType = (bDeleteRange ? LSM_DRANGE : (nVal>=0?LSM_WRITE:LSM_DELETE));
rc = lsmLogWrite(pDb, eType, (void *)pKey, nKey, (void *)pVal, nVal);
}
lsmSortedSaveTreeCursors(pDb);