Fix various problems to do with segment promotion. Add test file fts4growth2.test, containing tests to check that the FTS index does not grow indefinitely as the table is updated.

FossilOrigin-Name: 21491a9bc686e63bec32f1a67103622f1160a2f9

test/fts4growth.test

@@ -131,7 +131,7 @@ do_execsql_test 2.5 {
   SELECT end_block FROM x2_segdir WHERE level=3;
   INSERT INTO x2(x2) VALUES('merge=4,4');
   SELECT end_block FROM x2_segdir WHERE level=3;
-} {{3828 3430} {3828 10191} {3828 14109}}
+} {{3828 -3430} {3828 -10191} {3828 -14109}}
 
 do_execsql_test 2.6 {
   SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE 
@@ -235,6 +235,9 @@ do_test 3.3.2 {
 } {1 0 412}
 
 #--------------------------------------------------------------------------
+# Check a theory on a bug in fts4 - that segments with idx==0 were not 
+# being incrementally merged correctly. Theory turned out to be false.
+#
 do_execsql_test 4.1 {
   DROP TABLE IF EXISTS x4;
   DROP TABLE IF EXISTS t1;
@@ -256,14 +259,23 @@ do_execsql_test 4.4 {
   SELECT count(*) FROM x4_segdir;
 } {3}
 
-breakpoint
 do_execsql_test 4.5 {
   INSERT INTO x4(x4) VALUES('merge=10,2');
   SELECT count(*) FROM x4_segdir;
 } {3}
 
-if 0 {
-do_execsql_test 3.1 {
+do_execsql_test 4.6 {
+  INSERT INTO x4(x4) VALUES('merge=1000,2');
+  SELECT count(*) FROM x4_segdir;
+} {1}
+
+
+
+#--------------------------------------------------------------------------
+# Check that segments are not promoted if the "end_block" field does not
+# contain a size.
+#
+do_execsql_test 5.1 {
   DROP TABLE IF EXISTS x2;
   DROP TABLE IF EXISTS t1;
   CREATE TABLE t1(docid, words);
@@ -271,6 +283,41 @@ do_execsql_test 3.1 {
 }
 fts_kjv_genesis 
 
+proc first {L} {lindex $L 0}
+db func first first
+
+do_test 5.2 {
+  foreach r [db eval { SELECT rowid FROM t1 }] {
+    execsql {
+      INSERT INTO x2(docid, content) SELECT docid, words FROM t1 WHERE rowid=$r
+    }
+  }
+  foreach d [db eval { SELECT docid FROM t1 LIMIT -1 OFFSET 20 }] {
+    execsql { DELETE FROM x2 WHERE docid = $d }
+  }
+
+  execsql {
+    INSERT INTO x2(x2) VALUES('optimize');
+    SELECT level, idx, end_block FROM x2_segdir
+  }
+} {2 0 {752 1926}}
+
+do_execsql_test 5.3 {
+  UPDATE x2_segdir SET end_block = CAST( first(end_block) AS INTEGER );
+  SELECT end_block, typeof(end_block) FROM x2_segdir;
+} {752 integer}
+
+do_execsql_test 5.4 {
+  INSERT INTO x2 SELECT words FROM t1 LIMIT 50;
+  SELECT level, idx, end_block FROM x2_segdir
+} {2 0 752 0 0 {758 5174}}
+
+do_execsql_test 5.5 {
+  UPDATE x2_segdir SET end_block = end_block || ' 1926' WHERE level=2;
+  INSERT INTO x2 SELECT words FROM t1 LIMIT 40;
+  SELECT level, idx, end_block FROM x2_segdir
+} {0 0 {752 1926} 0 1 {758 5174} 0 2 {763 4170}}
+
 proc t1_to_x2 {} {
   foreach id [db eval {SELECT docid FROM t1 LIMIT 2}] {
     execsql {
@@ -280,24 +327,60 @@ proc t1_to_x2 {} {
   }
 }
 
+#--------------------------------------------------------------------------
+# Check that segments created by auto-merge are not promoted until they
+# are completed.
+#
+
+do_execsql_test 6.1 {
+  CREATE VIRTUAL TABLE x5 USING fts4;
+  INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 0;
+  INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 25;
+  INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 50;
+  INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 75;
+  SELECT count(*) FROM x5_segdir
+} {4}
+
+do_execsql_test 6.2 {
+  INSERT INTO x5(x5) VALUES('merge=2,4');
+  SELECT level, idx, end_block FROM x5_segdir;
+} {0 0 {10 9216} 0 1 {21 9330} 0 2 {31 8850} 0 3 {40 8689} 1 0 {1320 -3117}}
+
+do_execsql_test 6.3 {
+  INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 100;
+  SELECT level, idx, end_block FROM x5_segdir;
+} {
+  0 0 {10 9216} 0 1 {21 9330} 0 2 {31 8850} 
+  0 3 {40 8689} 1 0 {1320 -3117} 0 4 {1329 8297}
+}
+
+do_execsql_test 6.4 {
+  INSERT INTO x5(x5) VALUES('merge=200,4');
+  SELECT level, idx, end_block FROM x5_segdir;
+} {0 0 {1329 8297} 1 0 {1320 28009}}
+
+do_execsql_test 6.5 {
+  INSERT INTO x5 SELECT words FROM t1;
+  SELECT level, idx, end_block FROM x5_segdir;
+} {
+  0 1 {1329 8297} 0 0 {1320 28009} 0 2 {1449 118006}
+}
+
 #do_test 3.2 {
   #t1_to_x2
   #execsql {SELECT level, count(*) FROM x2_segdir GROUP BY level}
 #} {0 13 1 15 2 5}
 
-proc second {x} { lindex $x 1 }
-db func second second
-for {set i 0} {$i <1000} {incr i} {
-  t1_to_x2
-  db eval {
-    SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level;
-  } {
-    puts "$i.$level: $c"
-  }
-}
-}
-
+#proc second {x} { lindex $x 1 }
+#db func second second
+#for {set i 0} {$i <1000} {incr i} {
+#  t1_to_x2
+#  db eval {
+#    SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level;
+#  } {
+#    puts "$i.$level: $c"
+#  }
+#}
 
 finish_test
 
-