Fix an RBU problem with restarting an update after the *-oal file is already larger than 4GiB.

FossilOrigin-Name: 56869f54f31186231a4467f10208bcc08d9edae23c6e21831d6d3dff47da51de

ext/rbu/rbuhack.test

@@ -0,0 +1,165 @@
+# 2014 August 30
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] rbu_common.tcl]
+set ::testprefix rbu1
+
+forcedelete test.db2
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a INTEGER, b, c, PRIMARY KEY(a));
+  CREATE INDEX t1b ON t1(b);         -- root=3
+  CREATE INDEX t1c ON t1(c);         -- root=4
+
+  ATTACH 'test.db2' AS 'rbu';
+  CREATE TABLE rbu.data_t1(a, b, c, rbu_control);
+  INSERT INTO data_t1 VALUES(1, 1, 1, 0);
+  INSERT INTO data_t1 VALUES(2, 2, 2, 0);
+  INSERT INTO data_t1 VALUES(3, 3, 3, 0);
+}
+
+do_test 1.1 {
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 3
+  execsql {
+    CREATE TABLE imp1(b, id, PRIMARY KEY(b, id)) WITHOUT ROWID;
+    INSERT INTO imp1 VALUES(2, 2);
+  }
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 3
+} {}
+
+db close
+sqlite3 db test.db
+
+do_test 1.2 {
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 4
+  execsql {
+    CREATE TABLE imp1(b, id, PRIMARY KEY(b, id)) WITHOUT ROWID;
+    INSERT INTO imp1 VALUES(3, 3);
+  }
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 4
+} {}
+db close
+db_save
+
+sqlite3 db test.db
+do_catchsql_test 1.3 { PRAGMA integrity_check } {0 {{wrong # of entries in index t1c} {wrong # of entries in index t1b}}}
+db close
+
+do_test 1.4 {
+  sqlite3rbu rbu test.db test.db2
+  while 1 {
+    set rc [rbu step]
+    if {$rc!="SQLITE_OK"} break
+  }
+  list [catch { rbu close } msg] $msg
+} {1 {SQLITE_CONSTRAINT - UNIQUE constraint failed: t1.b, t1.a}}
+
+db_restore
+sqlite3 db test.db2
+do_execsql_test 1.5 {
+  PRAGMA writable_schema = 1;
+  CREATE TABLE sqlite_rbu_replace_hack('v');
+}
+db close
+
+do_test 1.6 {
+  sqlite3rbu rbu test.db test.db2
+  while 1 {
+    set rc [rbu step]
+    if {$rc!="SQLITE_OK"} break
+  }
+  list [catch { rbu close } msg] $msg
+} {0 SQLITE_DONE}
+
+sqlite3 db test.db
+do_execsql_test 1.7 {
+  PRAGMA integrity_check;
+  SELECT count(*) FROM t1;
+} {ok 3}
+
+#-------------------------------------------------------------------------
+
+reset_db
+forcedelete test.db2
+do_execsql_test 2.0 {
+  CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b));
+  CREATE INDEX t1c ON t1(c);         -- root=4
+
+  ATTACH 'test.db2' AS 'rbu';
+  CREATE TABLE rbu.data_t1(a, b, c, rbu_control);
+  INSERT INTO data_t1 VALUES(1, 1, 1, 0);
+  INSERT INTO data_t1 VALUES(2, 2, 2, 0);
+  INSERT INTO data_t1 VALUES(3, 3, 3, 0);
+}
+
+do_test 2.1 {
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 3
+  execsql {
+    CREATE TABLE imp1(a, b, id, PRIMARY KEY(b, id)) WITHOUT ROWID;
+    INSERT INTO imp1 VALUES(2, 2, 2);
+  }
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 3
+} {}
+
+db close
+sqlite3 db test.db
+
+do_test 2.2 {
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 4
+  execsql {
+    CREATE TABLE imp1(c, id, PRIMARY KEY(c, id)) WITHOUT ROWID;
+    INSERT INTO imp1 VALUES(3, 3);
+  }
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 4
+} {}
+db close
+db_save
+
+sqlite3 db test.db
+do_catchsql_test 2.3 { PRAGMA integrity_check } {0 {{wrong # of entries in index t1c} {wrong # of entries in index sqlite_autoindex_t1_1}}}
+db close
+
+do_test 2.4 {
+  sqlite3rbu rbu test.db test.db2
+  while 1 {
+    set rc [rbu step]
+    if {$rc!="SQLITE_OK"} break
+  }
+  list [catch { rbu close } msg] $msg
+} {1 {SQLITE_CONSTRAINT - UNIQUE constraint failed: t1.a, t1.b}}
+
+db_restore
+sqlite3 db test.db2
+do_execsql_test 2.5 {
+  PRAGMA writable_schema = 1;
+  CREATE TABLE sqlite_rbu_replace_hack('v');
+}
+db close
+
+do_test 2.6 {
+  sqlite3rbu rbu test.db test.db2
+  while 1 {
+    set rc [rbu step]
+    if {$rc!="SQLITE_OK"} break
+  }
+  list [catch { rbu close } msg] $msg
+} {0 SQLITE_DONE}
+
+sqlite3 db test.db
+do_execsql_test 2.7 {
+  PRAGMA integrity_check;
+  SELECT count(*) FROM t1;
+} {ok 3}
+
+
+
+finish_test
+

ext/rbu/sqlite3rbu.c

@@ -103,6 +103,8 @@
 # define RBU_ENABLE_DELTA_CKSUM 0
 #endif
 
+#define RBU_REPLACE_HACK_TABLE "sqlite_rbu_replace_hack"
+
 /*
 ** Swap two objects of type TYPE.
 */
@@ -376,6 +378,7 @@ struct sqlite3rbu {
   char *zRbu;                     /* Path to rbu db */
   char *zState;                   /* Path to state db (or NULL if zRbu) */
   char zStateDb[5];               /* Db name for state ("stat" or "main") */
+  int bRbuReplaceHack;            /* Do the REPLACE hack */
   int rc;                         /* Value returned by last rbu_step() call */
   char *zErrmsg;                  /* Error message if rc!=SQLITE_OK */
   int nStep;                      /* Rows processed for current object */
@@ -2350,7 +2353,10 @@ static int rbuObjIterPrepareAll(
       if( p->rc==SQLITE_OK ){
         p->rc = prepareFreeAndCollectError(
             p->dbMain, &pIter->pInsert, &p->zErrmsg,
-          sqlite3_mprintf("INSERT INTO \"rbu_imp_%w\" VALUES(%s)", zTbl, zBind)
+          sqlite3_mprintf("%s INTO \"rbu_imp_%w\" VALUES(%s)", 
+            p->bRbuReplaceHack ? "REPLACE" : "INSERT",
+            zTbl, zBind
+          )
         );
       }
 
@@ -2442,7 +2448,8 @@ static int rbuObjIterPrepareAll(
       if( p->rc==SQLITE_OK ){
         p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pInsert, pz,
             sqlite3_mprintf(
-              "INSERT INTO \"%s%w\"(%s%s) VALUES(%s)", 
+              "%s INTO \"%s%w\"(%s%s) VALUES(%s)", 
+              p->bRbuReplaceHack ? "REPLACE" : "INSERT",
               zWrite, zTbl, zCollist, (bRbuRowid ? ", _rowid_" : ""), zBindings
             )
         );
@@ -2725,7 +2732,7 @@ static RbuState *rbuLoadState(sqlite3rbu *p){
         break;
 
       case RBU_STATE_OALSZ:
-        pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1);
+        pRet->iOalSz = sqlite3_column_int64(pStmt, 1);
         break;
 
       case RBU_STATE_PHASEONESTEP:
@@ -2744,6 +2751,24 @@ static RbuState *rbuLoadState(sqlite3rbu *p){
   rc2 = sqlite3_finalize(pStmt);
   if( rc==SQLITE_OK ) rc = rc2;
 
+  /* If this is not a vacuum operation, search for the special table
+  ** "sqlite_rbu_replace_hack". Set the flag if it is found.
+  */
+  if( rc==SQLITE_OK && !rbuIsVacuum(p) ){
+    sqlite3_stmt *pQuery = 0;
+    rc = prepareFreeAndCollectError(p->dbRbu, &pQuery, &p->zErrmsg, 
+        sqlite3_mprintf("SELECT 1 FROM main.sqlite_master WHERE name='%s'",
+          RBU_REPLACE_HACK_TABLE
+        )
+    );
+    if( rc==SQLITE_OK ){
+      if( sqlite3_step(pQuery)==SQLITE_ROW ){
+        p->bRbuReplaceHack = 1;
+      }
+      rc = sqlite3_finalize(pQuery);
+    }
+  }
+
   p->rc = rc;
   return pRet;
 }

manifest

@@ -1,5 +1,5 @@
-C Version\s3.30.0
-D 2019-10-04T15:03:17.190
+C Fix\san\sRBU\sproblem\swith\srestarting\san\supdate\safter\sthe\s*-oal\sfile\sis\salready\slarger\sthan\s4GiB.
+D 2021-08-11T18:56:49.597
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -353,6 +353,7 @@ F ext/rbu/rbufault2.test c81327a3ac2c385b9b954db3644d4e0df93eeebfc3de9f1f29975a1
 F ext/rbu/rbufault3.test b2fcc9db5c982b869f67d1d4688d8cb515d5b92f58011fff95665f2e62cec179
 F ext/rbu/rbufault4.test 03d2849c3df7d7bd14a622e789ff049e5080edd34a79cd432e01204db2a5930a
 F ext/rbu/rbufts.test 0ae8d1da191c75bd776b86e24456db0fb6e97b7c944259fae5407ea55d23c31d
+F ext/rbu/rbuhack.test 95e7b35e58d5b2a8d182dde1a404949edf077c07485e6c2bce8a7f582941e729
 F ext/rbu/rbumisc.test 329986cf5dd51890c4eb906c2f960ebb773a79a64bed90f506b7c417825b37eb
 F ext/rbu/rbumulti.test 5fb139058f37ddc5a113c5b93238de915b769b7792de41b44c983bc7c18cf5b9
 F ext/rbu/rbupartial.test f25df014b8dbe3c5345851fba6e66f79ab237f57dc201b2d5f0dbae658ae5a4c
@@ -365,7 +366,7 @@ F ext/rbu/rbuvacuum.test 55e101e90168c2b31df6c9638fe73dc7f7cc666b6142266d1563697
 F ext/rbu/rbuvacuum2.test b8e5b51dc8b2c0153373d024c0936be3f66f9234acbd6d0baab0869d56b14e6b
 F ext/rbu/rbuvacuum3.test 8addd82e4b83b4c93fa47428eae4fd0dbf410f8512c186f38e348feb49ba03dc
 F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2f1dbccfd10
-F ext/rbu/sqlite3rbu.c f3a3e09f575157052813be667d6ab3b54f47fb02e6e1c9f767ad7bb8f1fb90b3
+F ext/rbu/sqlite3rbu.c d175cbd98baba39cdea85df1de376277ab207ef1e4af8071aa1c208bb4fccc84
 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812
 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
@@ -1846,10 +1847,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7f9a4b6015ac332a04d3e394a6b3210fc95253d8786a261178a5639cb8d9d987
-R f116075cb1c47b7928f41444f3803579
-T +bgcolor * #d0c0ff
-T +sym-release *
-T +sym-version-3.30.0 *
-U drh
-Z edadfb0c72633ac9bcd13396063b6815
+P 4b73e151cdb4b89bdd2f1bab5c0378500234ce6c479cc1d8cd4e006be79d1162
+Q +087b8b41c6ed76b55c11315e7e95679d67590be20ae21108b593d00bb7d1c57a
+R f1bc15780a98eb9fb1abdadc164e3b34
+U dan
+Z 02ec0c40b41079c9dff32d6e22ece993

manifest.uuid

@@ -1 +1 @@
-c20a35336432025445f9f7e289d0cc3e4003fb17f45a4ce74c6269c407c6e09f
+56869f54f31186231a4467f10208bcc08d9edae23c6e21831d6d3dff47da51de