Update zonefile README.md file to mention the frame cache.

FossilOrigin-Name: 84e9351bbd0a9bf9e1d64a441366a90276c231b44548cbc947f7fb2842f82490

Makefile.msc

@@ -1494,6 +1494,7 @@ TESTEXT = \
   $(TOP)\ext\misc\eval.c \
   $(TOP)\ext\misc\fileio.c \
   $(TOP)\ext\misc\fuzzer.c \
+  $(TOP)\ext\zonefile\zonefile.c \
   $(TOP)\ext\fts5\fts5_tcl.c \
   $(TOP)\ext\fts5\fts5_test_mi.c \
   $(TOP)\ext\fts5\fts5_test_tok.c \

ext/zonefile/README.md

@@ -0,0 +1,186 @@
+
+# The Zonefile Extension
+
+## Functionality
+
+### Creating Zonefile Files
+
+To create a new zonefile, first create a database table with the following
+schema:
+
+>     CREATE TABLE data(
+>       k INTEGER PRIMARY KEY,
+>       frame INTEGER DEFAULT -1,   -- frame number.  Automatic if -1
+>       idx INTEGER DEFAULT -1,     -- index of entry within frame.  Auto if -1
+>       v BLOB
+>     );
+
+The table may be created in a persistent or temporary database and may
+take any name, but must contain the columns above. The table must be 
+populated with a row for each key intended to appear in the new zonefile
+file.
+
+Once the table is populated, a zonefile is created using the following
+SQL:
+
+>     SELECT zonefile_write(<file>, <table> [, <parameters>]);
+
+where &lt;file&gt; is the name of the file to create on disk, &lt;table&gt; 
+is the name of the database table to read and optional argument 
+&lt;parameters&gt; is a JSON object containing various attributes that
+influence creation of the zonefile file. 
+
+Currently, the following &lt;parameters&gt; attributes are supported:
+
+<table border=1>
+<tr align=left><th>Attribute<th>Default<th>Interpretation
+<tr valign=top><td>maxAutoFrameSize<td>65536
+<td>The maximum uncompressed frame size in bytes for automatically generated
+zonefile frames.
+
+<tr valign=top><td>compressionTypeContent<td>"none"
+<td>The compression type used to compress each frame in the zonefile. 
+Valid values are "none" (no compression), "zstd", "zstd_global_dict",
+"zlib", "brotli", "lz4" and "lz4hc". Not all compression methods are
+supported by all builds. The compression method supported by a build
+depends on the combination of SQLITE_HAVE_ZSTD, SQLITE_HAVE_ZLIB,
+SQLITE_HAVE_BROTLI and SQLITE_HAVE_LZ4 pre-processor symbols defined
+at build time.
+
+<tr valign=top><td>compressionTypeIndexData<td>"none"
+<td>The compression type used to compress the zonefile index structure.
+All values that are valid for the <i>compressionTypeContent</i> parameter,
+except for "zstd_global_dict", are also valid for this option.
+
+<tr valign=top><td>encryptionType<td>"none"
+<td>The encryption type to use. At present the only valid values are
+"none" (no encryption) and "xor" (an insecure mock encryption method
+useful for testing only). Enhanced implementations may support any or
+all of the following encryption schemes:
+<ul>
+  <li> "AES_128_CTR"
+  <li> "AES_128_CBC"
+  <li> "AES_256_CTR"
+  <li> "AES_256_CBC"
+</ul>
+
+<tr valign=top><td>encryptionKey<td>""
+<td>The encryption key to use. The encryption key must be specified as an
+even number of hexadecimal that will be converted to a binary key before
+use. It is the responsibility of the caller to specify a key of the optimal
+length for each encryption algorithm (e.g. 16 bytes (32 hex digits) for
+a 128-bit encryption, or 32 bytes (64 digits) for a 256-bit method).
+This option is ignored if <i>encryptionType</i> is set to "none".
+</table>
+
+For example, to create a zonefile named "test.zonefile" based on the
+contents of database table "test_input", with a maximum automatic
+frame size of 4096 bytes and using "xor" encryption with a 128-bit key:
+
+>     SELECT zonefile_write('test.zonefile', 'test_input',
+>       '{"maxAutoFrameSize":4096,
+>         "encryptionType":"xor",
+>         "encryptionKey":"e6e600bc063aad12f6387beab650c48a"
+>       }'
+>     );
+
+### Using (Reading) Zonefile Files
+
+To create a new zonefile table, one of the following:
+
+>     CREATE VIRTUAL TABLE z1 USING zonefile;
+>     CREATE VIRTUAL TABLE z1 USING zonefile(cachesize=N);
+
+where <i>N</i> is any non-zero positive integer. If the zonefile is used
+to access any files containing compressed or encrypted data, it maintains
+an LRU cache of uncompressed frame data <i>N</i> frames in size. The
+default value of <i>N</i> is 1.
+
+Creating a "zonefile" virtual table actually creates two virtual tables in the
+database schema. One read-only table named "z1", with a schema equivalent to:
+
+>     CREATE TABLE z1(  -- this whole table is read-only
+>       k INTEGER PRIMARY KEY,     -- key value
+>       v BLOB,                    -- associated blob of data
+>       fileid INTEGER,            -- file id (rowid value for z1_files)
+>       sz INTEGER                 -- size of blob of data in bytes
+>     );
+
+And a read-write table named "z1_files" with a schema like:
+
+>     CREATE TABLE z1_files(
+>       filename TEXT PRIMARY KEY,
+>       ekey BLOB,         -- encryption key
+>       header JSON HIDDEN -- read-only
+>     );
+
+Both tables are initially empty. To add a zonefile to the index, insert a
+row into the "z1_files" table:
+
+>     INSERT INTO z1_files(filename) VALUES(<filename>);
+
+If the file is an encrypted file, then the encryption key (a blob) must
+be inserted into the "ekey" column. Encryption keys are not stored in the
+database, they are held in main-memory only. This means that each new
+connection must configure encryption key using UPDATE statements before
+accessing any encrypted files. For example:
+
+>     -- Add new encrypted file to database:
+>     INSERT INTO z1_files(filename, ekey) VALUES(<filename>, <ekey>);
+>
+>     -- Configure encryption key for existing file after opening database:
+>     UPDATE z1_files SET ekey = <ekey> WHERE filename = <filename>;
+
+Currently, values provided for any columns other than "filename" and
+"ekey" are ignored. Files are removed from the index by deleting rows 
+from the z1_files table:
+
+>     DELETE FROM z1_files WHERE filename = <filename>;
+
+Once zonefile files have been added to the index, their contents are 
+visible in table "z1". To retrieve the value associated with a single
+key from one of the zonefile files in the index:
+
+>     SELECT v FROM z1 WHERE k = <key>;
+
+
+## Notes
+
+  *  Contrary to the spec, the implementation uses 32-bit (not 16-bit) frame
+     numbers. So the KeyOffsetPair structure becomes:
+
+     KeyOffsetPair
+     {
+       uint64  key;
+       uint32  frameNo;
+       uint32  frameByteOffset;
+     };
+
+     Also, the ZonefileHeader.numFrames field is now 32-bit. Which makes
+     the ZonefileHeader structure 26 bytes in size. The implementation
+     pads this out to 32 bytes so that the ZoneFileIndex is 8-byte aligned.
+
+  *  Multi-byte integer values are big-endian.
+
+  *  The offsets in the ZoneFileIndex.byteOffsetZoneFrame[] array are
+     relative to the offset in ZoneFileHeader.byteOffsetFrames. This is
+     necessary as we may not know the offset of the start of the frame data
+     until after the ZoneFileIndex structure is compressed.
+
+  *  The offsets in the ZoneFileIndex.byteOffsetZoneFrame[] array are the
+     offsets for the first byte past the end of the corresponding frame.
+     For example, byteOffsetZoneFrame[] identifies the first byte of the
+     second frame, and byteOffsetZoneFrame[numFrames-1] is one byte past
+     the end of the last frame in the file.
+
+     This is better as if we store the starting offset of each frame, there
+     is no way to determine the size of the last frame in the file without
+     trusting the filesize itself.
+
+  *  Currently there is no support at all for encryption.
+
+  *  Zonefile currently uses json1 to parse the json argument to
+     zonefile\_write(). And so must be used with an SQLITE\_ENABLE\_JSON1
+     or otherwise json1-enabled SQLite.
+
+

ext/zonefile/zonefile1.test

@@ -0,0 +1,644 @@
+# 2018 Feb 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the zonefile extension.
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source [file join $testdir tester.tcl]
+set testprefix zonefile1
+load_static_extension db zonefile
+
+do_execsql_test 1.0 {
+  CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
+  INSERT INTO zz VALUES(1, -1, -1, randomblob(100));
+  INSERT INTO zz VALUES(2, -1, -1, randomblob(100));
+  INSERT INTO zz VALUES(3, -1, -1, randomblob(100));
+}
+
+do_execsql_test 1.1 {
+  SELECT zonefile_write('test.zonefile', 'zz');
+} {{}}
+
+do_execsql_test 1.2 {
+  CREATE VIRTUAL TABLE z1 USING zonefile;
+  SELECT name FROM sqlite_master WHERE name LIKE 'z1%' ORDER BY 1;
+} {z1 z1_files z1_shadow_file z1_shadow_idx}
+
+do_execsql_test 1.3 {
+  INSERT INTO z1_files(filename) VALUES('test.zonefile');
+  SELECT filename, 
+         json_extract(header, '$.magicNumber'),
+         json_extract(header, '$.numFrames'),
+         json_extract(header, '$.numKeys')
+         FROM z1_files;
+} {test.zonefile 1179332920 1 3}
+
+do_execsql_test 1.4 { SELECT count(*) FROM z1_shadow_idx } 3
+
+do_execsql_test 1.5.1 { SELECT k FROM z1 } {1 2 3}
+do_execsql_test 1.5.2 { SELECT fileid FROM z1 } {1 1 1}
+do_execsql_test 1.5.4 { SELECT sz FROM z1 } {100 100 100}
+
+do_execsql_test 1.5.5 {
+  SELECT zz.v==z1.v FROM zz, z1 WHERE zz.k=z1.k
+} {1 1 1}
+
+do_execsql_test 1.5 {
+  DELETE FROM z1_files;
+  SELECT * FROM z1_files;
+} {}
+
+do_execsql_test 1.6 { SELECT count(*) FROM z1_shadow_idx } 0
+
+do_execsql_test 1.7 { DROP TABLE z1 }
+
+do_execsql_test 1.8 {
+  SELECT * FROM sqlite_master WHERE name LIKE 'z1%';
+}
+
+#-------------------------------------------------------------------------
+# Figure out which compression algorithms, if any, are supported by
+# this build. Populate the global list $COMPRESSION_METHODS with the
+# result.
+reset_db
+load_static_extension db zonefile
+do_execsql_test 2.0 {
+  CREATE TABLE bb(
+    k INTEGER PRIMARY KEY, 
+    frame INTEGER DEFAULT -1, 
+    idx INTEGER DEFAULT -1, 
+    v BLOB
+  );
+  INSERT INTO bb(k, v) VALUES(1, randomblob(100));
+}
+set COMPRESSION_METHODS [list]
+foreach cmp {
+  none zlib zstd zstd_global_dict lz4 lz4hc brotli nosuchcmp
+} {
+  set res [catchsql {
+    WITH p(n,v) AS (
+        VALUES('compressionTypeContent', $cmp)
+    )
+    SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p;
+  }]
+
+  if {[lindex $res 0]==0} {
+    lappend COMPRESSION_METHODS $cmp
+  }
+}
+
+# Check that it is not possible to use zstd_global_dict to compress
+# the zonefile index.
+#
+if {[lsearch $COMPRESSION_METHODS zstd_global_dict]>=0} {
+  do_catchsql_test 2.1 {
+    WITH p(n,v) AS (
+        VALUES('compressionTypeIndexData', 'zstd_global_dict')
+    )
+    SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p;
+  } {1 {compressor "zstd_global_dict" may not be used to compress the zonefile index}}
+}
+
+set extra_header 0
+set cachesize 0
+foreach cmp $COMPRESSION_METHODS { foreach cmpidx $COMPRESSION_METHODS {
+  if {$cmpidx == "zstd_global_dict"} continue
+  reset_db
+  load_static_extension db zonefile
+
+  set tn "$cmp/$cmpidx"
+  set extra_header [expr {$extra_header ? 0 : 100}]
+  set cachesize [expr {$cachesize ? 0 : 10}]
+
+  do_execsql_test 2.$tn.0.1 {
+    CREATE TABLE zz(
+      k INTEGER PRIMARY KEY, 
+      frame INTEGER DEFAULT -1, 
+      idx INTEGER DEFAULT -1, 
+      v BLOB
+    );
+    CREATE TABLE rt(k INTEGER PRIMARY KEY, v BLOB);
+  }
+
+  do_execsql_test 2.$tn.0.2 "
+    CREATE VIRTUAL TABLE zone USING zonefile(cachesize = $cachesize)
+  " {}
+  
+  set nMinByte   0
+  set nMaxByte 444
+  foreach {zonefile lKey} {
+    test1.zonefile {195 1238 298 405 297}
+    test2.zonefile {124 1624 82 1929}
+    test3.zonefile {932 683 1751 410 41}
+    test4.zonefile {427 1491}
+    test5.zonefile {1004 473 801 394 1672 816 1577}
+    test6.zonefile {1374 1454 1005}
+    test7.zonefile {450 241 319 133}
+    test8.zonefile {1414 900 1406 1917 127 673}
+    test9.zonefile {1192 226 988 1292 718 1345 1675}
+    test10.zonefile {314}
+    test11.zonefile {1177 1597 60 532 291 1164 812}
+    test12.zonefile {1168 1290 1585 939 1916}
+    test13.zonefile {644 1784 1476 1283 433 506}
+    test14.zonefile {1141 1547 1506 364}
+    test15.zonefile {1756 1885 844 1880 1896 354}
+    test16.zonefile {1383 1928 1371}
+    test17.zonefile {93}
+    test18.zonefile {1067}
+    test19.zonefile {642}
+    test20.zonefile {1380 1857}
+    test21.zonefile {288 293 1968 1207 1739 231 300}
+    test22.zonefile {651 1007 607 830 299 1431}
+    test23.zonefile {81 1651 543 1949 256 119 1088}
+    test24.zonefile {1278 2024 682 1115 194 636 1804}
+    test25.zonefile {514 1155 171 2015 791}
+    test26.zonefile {1615 1228 147 1464}
+    test27.zonefile {55 1130 781 678 78}
+    test28.zonefile {1981 1401 1178}
+    test29.zonefile {1754 864 183 1953 1901}
+    test30.zonefile {1461 817}
+    test31.zonefile {1720 1722 686 1833}
+  } {
+    forcedelete $zonefile
+    execsql { DELETE FROM zz; }
+    foreach k $lKey {
+      execsql { INSERT INTO zz(k, v) VALUES($k, randomblob($k)) }
+    }
+    execsql { INSERT INTO rt SELECT k, v FROM zz }
+    execsql { 
+      WITH p(n,v) AS (
+        VALUES('maxAutoFrameSize', 2000) UNION ALL
+        VALUES('compressionTypeIndexData', $cmpidx) UNION ALL
+        VALUES('compressionTypeContent', $cmp) UNION ALL
+        VALUES('debugExtendedHeaderSize', $extra_header)
+      )
+      SELECT zonefile_write($zonefile, 'zz', json_group_object(n, v)) FROM p;
+      INSERT INTO zone_files(filename) VALUES($zonefile);
+    }
+  }
+  
+  do_execsql_test 2.$tn.1 {
+    SELECT k FROM zone JOIN rt USING (k) WHERE zone.v!=rt.v
+  }
+  do_execsql_test 2.$tn.2 {
+    SELECT count(*) FROM zone JOIN rt USING (k);
+  } {135}
+  do_execsql_test 2.$tn.3 {
+    SELECT filename, 
+           json_extract(header, '$.numKeys'),
+           json_extract(header, '$.numFrames')
+    FROM zone_files 
+    WHERE filename IN ('test19.zonefile', 'test20.zonefile', 'test21.zonefile')
+    ORDER BY 1
+  } {
+    test19.zonefile 1 1
+    test20.zonefile 2 2
+    test21.zonefile 7 4
+  }
+}}
+
+#--------------------------------------------------------------------------
+#
+reset_db
+load_static_extension db zonefile
+do_execsql_test 3.0 {
+  CREATE TABLE dd(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
+  INSERT INTO dd VALUES(1000, 1, -1, randomblob(44));
+  INSERT INTO dd VALUES(1001, 1, -1, randomblob(55));
+  INSERT INTO dd VALUES(1002, 2, -1, randomblob(66));
+  WITH p(n,v) AS (
+      VALUES('maxAutoFrameSize', 2000) UNION ALL
+      VALUES('encryptionType', 'xor') UNION ALL
+      VALUES('debugEncryptionKeyText', 1) UNION ALL
+      VALUES('encryptionKey', '0123456789')
+  )
+  SELECT zonefile_write('test.zonefile', 'dd', json_group_object(n, v)) FROM p;
+} {{}}
+
+do_execsql_test 3.1 {
+  CREATE VIRTUAL TABLE cc USING zonefile;
+  INSERT INTO cc_files(filename,ekey) VALUES('test.zonefile','0123456789');
+  SELECT quote(dd.v)==quote(cc.v) FROM cc JOIN dd USING (k)
+} {1 1 1}
+
+do_execsql_test 3.2.1 {
+  DELETE FROM cc_files;
+  INSERT INTO cc_files(filename,ekey) VALUES('test.zonefile','abcdefghij');
+  SELECT quote(dd.v)==quote(cc.v) FROM cc JOIN dd USING (k)
+} {0 0 0}
+
+do_execsql_test 3.2.2 {
+  SELECT rowid,sz FROM cc;
+} {1000 44 1001 55 1002 66}
+
+do_execsql_test 3.3 {
+  UPDATE cc_files SET ekey = '0123456789';
+  SELECT quote(dd.v)==quote(cc.v) FROM cc JOIN dd USING (k)
+} {1 1 1}
+
+close [open test.zonefile w+]
+do_catchsql_test 3.4 {
+  SELECT header FROM cc_files
+} {1 {failed to read zonefile header from file "test.zonefile"}}
+
+forcedelete test.zonefile
+do_catchsql_test 3.5 {
+  SELECT header FROM cc_files
+} {1 {failed to open file "test.zonefile" for reading}}
+
+do_execsql_test 3.6 {
+  SELECT ekey FROM cc_files
+} {{}}
+
+forcedelete test.zonefile
+do_catchsql_test 3.7 {
+  SELECT * FROM cc;
+} {1 {failed to open file "test.zonefile" for reading}}
+
+#-------------------------------------------------------------------------
+# Check that a file that uses an unknown compression method is handled
+# correctly (an error is returned).
+#
+reset_db
+load_static_extension db zonefile
+do_execsql_test 4.0 {
+  CREATE TABLE dd(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
+  INSERT INTO dd VALUES(1000, 1, -1, randomblob(44));
+  INSERT INTO dd VALUES(1001, 1, -1, randomblob(55));
+  INSERT INTO dd VALUES(1002, 2, -1, randomblob(66));
+  SELECT zonefile_write('test.zonefile', 'dd');
+  CREATE VIRTUAL TABLE x1 USING zonefile;
+} {{}}
+
+do_test 4.1 {
+  hexio_write test.zonefile 5 77
+} {1}
+do_execsql_test 4.2 {
+  INSERT INTO x1_files(filename) VALUES('test.zonefile');
+} {}
+do_catchsql_test 4.3 {
+  SELECT * FROM x1
+} {1 {unsupported compression method: 119}}
+do_test 4.4 {
+  hexio_write test.zonefile 4 77
+} {1}
+do_catchsql_test 4.5 {
+  DELETE FROM x1_files;
+  INSERT INTO x1_files(filename) VALUES('test.zonefile');
+} {1 {unsupported compression method: 119}}
+
+do_test 4.6 {
+  hexio_write test.zonefile 0 00
+} {1}
+do_catchsql_test 4.7 {
+  INSERT INTO x1_files(filename) VALUES('test.zonefile');
+} {1 {failed to read zonefile header from file "test.zonefile"}}
+
+#-------------------------------------------------------------------------
+# Test using various types in the "frame" field of an input table.
+#
+reset_db
+load_static_extension db zonefile
+if {[lsearch $COMPRESSION_METHODS zlib]>=0} {
+  do_execsql_test 5.0 {
+    CREATE TABLE "a b"(k INTEGER PRIMARY KEY,frame INTEGER,idx INTEGER,v BLOB);
+    INSERT INTO "a b" VALUES(1, 0.5, -1, randomblob(44));
+    INSERT INTO "a b" VALUES(2, 0.5, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(3, 1.5, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(4, 1.5, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(5,   2, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(6,   2, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(7, 200, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(8, 200, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(9, 300, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(10, 300, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(11, NULL, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(12, NULL, -1, randomblob(55));
+    INSERT INTO "a b" VALUES(13, 'f1', -1, randomblob(55));
+    INSERT INTO "a b" VALUES(14, 'f1', -1, randomblob(55));
+    INSERT INTO "a b" VALUES(15, 'frame2', -1, randomblob(55));
+    INSERT INTO "a b" VALUES(16, 'frame2', -1, randomblob(55));
+    INSERT INTO "a b" VALUES(17, x'1234', -1, randomblob(55));
+    INSERT INTO "a b" VALUES(18, x'1234', -1, randomblob(55));
+    INSERT INTO "a b" VALUES(19, x'abcd', -1, randomblob(55));
+    INSERT INTO "a b" VALUES(20, x'abcd', -1, randomblob(55));
+  
+    SELECT zonefile_write('test.zonefile', 'a b',
+      '{"compressionTypeContent":"zlib"}'
+    );
+  } {{}}
+
+  do_execsql_test 5.1 {
+    CREATE VIRTUAL TABLE abc USING zonefile; 
+    INSERT INTO abc_files(filename) VALUES('test.zonefile');
+    SELECT group_concat(k) FROM abc_shadow_idx GROUP BY fofst
+  } {
+    11,12   1,2   3,4   5,6   7,8  
+     9,10  13,14 15,16 17,18 19,20
+  }
+}
+
+do_execsql_test 6.0 {
+  CREATE TABLE "ab"(k INTEGER PRIMARY KEY,frame INTEGER,idx INTEGER,v BLOB);
+  INSERT INTO "ab" VALUES(1, 0.5, -1, randomblob(44));
+  INSERT INTO "ab" VALUES(2, 0.5, -1, randomblob(55));
+  INSERT INTO "ab" VALUES(3, 1.5, -1, randomblob(55));
+  INSERT INTO "ab" VALUES(4, 1.5, -1, randomblob(55));
+}
+do_catchsql_test 6.1.1 {
+  SELECT zonefile_write('test.zonefile', 'ab',
+      '{"debugExtendedHeaderSize":-1}'
+  );
+} {1 {debugExtendedHeaderSize value out of range: -1}}
+do_catchsql_test 6.1.2 {
+  SELECT zonefile_write('test.zonefile', 'ab',
+      '{"debugExtendedHeaderSize":256}'
+  );
+} {1 {debugExtendedHeaderSize value out of range: 256}}
+
+do_catchsql_test 6.2 {
+  SELECT zonefile_write('test.zonefile', 'ab',
+      '{"unknown":256}'
+  );
+} {1 {unknown parameter name: "unknown"}}
+
+forcedelete test.dir
+file mkdir test.dir
+do_catchsql_test 6.3 {
+  SELECT zonefile_write('test.dir', 'ab');
+} {1 {failed to open file "test.dir" for writing}}
+
+do_catchsql_test 6.4 {
+  CREATE VIRTUAL TABLE zzz USING zonefile;
+  INSERT INTO zzz_files(filename) VALUES('nosuchfile.zonefile');
+} {1 {failed to open file "nosuchfile.zonefile" for reading}}
+
+if {$tcl_platform(platform)=="windows"} {
+  do_catchsql_test 6.5 {
+    INSERT INTO zzz_files(filename) VALUES('test.dir');
+  } {1 {failed to open file "test.dir" for reading}}
+} else {
+  do_catchsql_test 6.5 {
+    INSERT INTO zzz_files(filename) VALUES('test.dir');
+  } {1 {failed to read zonefile header from file "test.dir"}}
+}
+
+#-------------------------------------------------------------------------
+# Check that errors generated when building a dictionary are handled.
+# The zstd library routines for building a dictionary throw an error
+# if they are provided with too little data.
+#
+# Also test that zstd_global_dict cannot be used to compress the zonefile
+# index (as there is nowhere in the file format to store the dictionary
+# for this compression).
+#
+reset_db
+load_static_extension db zonefile
+if {[lsearch $COMPRESSION_METHODS zstd_global_dict]>=0} {
+  do_execsql_test 7.0 {
+    CREATE TABLE "ab"(k INTEGER PRIMARY KEY,frame INTEGER,idx INTEGER,v BLOB);
+    INSERT INTO "ab" VALUES(1, -1, -1, 'abc');
+  }
+
+  do_catchsql_test 7.1 {
+    SELECT zonefile_write('test.zonefile', 'ab',
+      '{"compressionTypeContent":"zstd_global_dict"}'
+    );
+  } {1 {error generating dictionary}}
+
+  do_catchsql_test 7.2 {
+    SELECT zonefile_write('test.zonefile', 'ab',
+      '{"compressionTypeIndexData":"zstd_global_dict"}'
+    );
+  } {1 {compressor "zstd_global_dict" may not be used to compress the zonefile index}}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+load_static_extension db zonefile
+do_catchsql_test 8.1 {
+  CREATE VIRTUAL TABLE one USING zonefile_files;
+} {1 {do not create zonefile_files tables directly!}}
+do_catchsql_test 8.2 {
+  CREATE VIRTUAL TABLE onetwothree USING zonefile_files;
+} {1 {do not create zonefile_files tables directly!}}
+
+#-------------------------------------------------------------------------
+# A zone file containing zero keys.
+#
+reset_db
+load_static_extension db zonefile
+do_execsql_test 9.0 {
+  CREATE TABLE s(k INTEGER PRIMARY KEY,frame INTEGER,idx INTEGER,v BLOB);
+  SELECT zonefile_write('test.zonefile', 's');
+} {{}}
+do_execsql_test 9.1 {
+  CREATE VIRTUAL TABLE ss USING zonefile;
+  INSERT INTO ss_files(filename) VALUES('test.zonefile');
+  SELECT * FROM ss;
+} {}
+
+#-------------------------------------------------------------------------
+# Test UPDATE and DELETE statements on the %_files virtual table.
+#
+reset_db
+load_static_extension db zonefile
+do_execsql_test 10.0 {
+  CREATE TABLE data(k INTEGER PRIMARY KEY, v BLOB);
+  INSERT INTO data(k, v) VALUES(1, randomblob(100));
+  INSERT INTO data(k, v) VALUES(2, randomblob(100));
+  INSERT INTO data(k, v) VALUES(3, randomblob(100));
+  INSERT INTO data(k, v) VALUES(4, randomblob(100));
+  INSERT INTO data(k, v) VALUES(5, randomblob(100));
+  INSERT INTO data(k, v) VALUES(6, randomblob(100));
+  INSERT INTO data(k, v) VALUES(7, randomblob(100));
+  INSERT INTO data(k, v) VALUES(8, randomblob(100));
+  INSERT INTO data(k, v) VALUES(9, randomblob(100));
+
+  CREATE VIEW v1 AS 
+    SELECT k, -1 AS frame, -1 AS idx, v
+    FROM data WHERE k IN (1,2,3);
+  SELECT zonefile_write('test1.zonefile', 'v1');
+
+  CREATE VIEW v2 AS 
+    SELECT k, -1 AS frame, -1 AS idx, v
+    FROM data WHERE k IN (4,5,6);
+  SELECT zonefile_write('test2.zonefile', 'v2');
+
+  CREATE VIEW v3 AS 
+    SELECT k, -1 AS frame, -1 AS idx, v
+    FROM data WHERE k IN (7,8,9);
+  SELECT zonefile_write('test3.zonefile', 'v3');
+} {{} {} {}}
+
+do_execsql_test 10.1 {
+  CREATE VIRTUAL TABLE "a b" USING zonefile;
+  INSERT INTO "a b_files"(filename) VALUES('test1.zonefile');
+  SELECT k FROM "a b"
+} {1 2 3}
+
+do_execsql_test 10.2 {
+  UPDATE "a b_files" SET filename = 'test2.zonefile';
+  SELECT k FROM "a b"
+} {4 5 6}
+
+do_execsql_test 10.3 {
+  INSERT INTO "a b_files"(filename) VALUES('test1.zonefile');
+  INSERT INTO "a b_files"(filename) VALUES('test3.zonefile');
+  SELECT k FROM "a b"
+} {1 2 3 4 5 6 7 8 9}
+
+do_execsql_test 10.4 {
+  DELETE FROM "a b_files" WHERE filename = 'test2.zonefile';
+  SELECT k FROM "a b"
+} {1 2 3 7 8 9}
+
+do_execsql_test 10.5 {
+  DELETE FROM "a b_files" WHERE filename = 'test1.zonefile';
+  SELECT k FROM "a b"
+} {7 8 9}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+load_static_extension db zonefile
+do_execsql_test 11.0 {
+  CREATE TABLE data(k INTEGER PRIMARY KEY, v BLOB);
+  INSERT INTO data(k, v) VALUES(-1, randomblob(100));
+  INSERT INTO data(k, v) VALUES(1000, randomblob(100));
+  INSERT INTO data(k, v) VALUES(-1000, randomblob(100));
+  INSERT INTO data(k, v) VALUES(9223372036854775807, randomblob(100));
+  INSERT INTO data(k, v) VALUES(-9223372036854775807, randomblob(100));
+  INSERT INTO data(k, v) VALUES(-9223372036854775808, randomblob(100));
+
+  CREATE VIEW v1 AS SELECT k, -1 AS frame, -1 AS idx, v FROM data;
+  SELECT zonefile_write('test1.zonefile', 'v1');
+} {{}}
+
+do_execsql_test 11.1 {
+  CREATE VIRTUAL TABLE one USING zonefile;
+  INSERT INTO "one_files"(filename) VALUES('test1.zonefile');
+  SELECT k FROM one
+} {-9223372036854775808 -9223372036854775807 -1000 -1 1000 9223372036854775807}
+
+do_execsql_test 11.2 {
+  SELECT count(*) FROM data JOIN one USING (k) WHERE one.v==data.v
+} 6
+
+#-------------------------------------------------------------------------
+#
+do_catchsql_test 12.1 {
+  CREATE VIRTUAL TABLE nm USING zonefile(abcd)
+} {1 {parse error in option: abcd}}
+do_catchsql_test 12.2 {
+  CREATE VIRTUAL TABLE nm USING zonefile(a=b)
+} {1 {parse error in option: a=b}}
+do_catchsql_test 12.3 {
+  CREATE VIRTUAL TABLE nm USING zonefile(cachesiza=b)
+} {1 {parse error in option: cachesiza=b}}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+load_static_extension db zonefile
+do_execsql_test 13.0 {
+  CREATE TABLE data(k INTEGER PRIMARY KEY, v BLOB);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<250
+  )
+  INSERT INTO data SELECT i, randomblob(100) FROM s;
+  CREATE VIEW v1 AS SELECT k, -1 AS frame, -1 AS idx, v FROM data;
+  SELECT zonefile_write('test1.zonefile', 'v1');
+
+  CREATE VIRTUAL TABLE nm USING zonefile;
+  INSERT INTO nm_files(filename) VALUES('test1.zonefile');
+} {{}}
+
+foreach {tn cond} {
+  1   "k > 30"
+  2   "k >= 100"
+  3   "k <= 100"
+  4   "k < 55"
+  5   "k LIKE '1%'"
+  6   "k BETWEEN 10 AND 20"
+  7   "k > 100 AND k < 200"
+} {
+  do_execsql_test 13.1.$tn.1 [subst {
+    SELECT count(*) FROM nm WHERE $cond
+  }] [db one "SELECT count(*) FROM data WHERE $cond"]
+
+  do_execsql_test 13.1.$tn.2 [subst {
+    SELECT count(*) FROM nm WHERE $cond AND
+      v!=(SELECT v FROM data WHERE k=nm.k);
+  }] 0
+}
+
+close [open test1.zonefile w+]
+do_catchsql_test 13.2.1 {
+  SELECT * FROM nm WHERE k=24;
+} {1 {SQL logic error}}
+forcedelete test1.zonefile
+do_catchsql_test 13.2.2 {
+  SELECT * FROM nm WHERE k=24;
+} {1 {failed to open file "test1.zonefile" for reading}}
+
+do_catchsql_test 13.3.1 {
+  DELETE FROM nm_shadow_file;
+  SELECT * FROM nm WHERE k=24;
+} {1 {database disk image is malformed}}
+do_catchsql_test 13.3.2 {
+  DROP TABLE nm_shadow_file;
+  SELECT * FROM nm WHERE k=24;
+} {1 {no such table: main.nm_shadow_file}}
+db close
+sqlite3 db test.db
+load_static_extension db zonefile
+do_catchsql_test 13.3.3 {
+  SELECT * FROM nm WHERE k=24;
+} {1 {no such table: main.nm_shadow_file}}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+load_static_extension db zonefile
+do_execsql_test 14.0 {
+  CREATE TABLE data(k INTEGER PRIMARY KEY, frame, idx, v BLOB);
+  INSERT INTO data VALUES(1, 1, -1, randomblob(200));
+  INSERT INTO data VALUES(2, 2, -1, randomblob(200));
+  INSERT INTO data VALUES(3, 3, -1, randomblob(200));
+  SELECT zonefile_write('test.zonefile', 'data',
+  '{"encryptionType":"xor","encryptionKey":"pass","debugEncryptionKeyText":1}'
+  );
+
+  CREATE VIRTUAL TABLE nm USING zonefile(cachesize=2);
+  INSERT INTO nm_files(filename,ekey) VALUES('test.zonefile','pass');
+} {{}}
+
+set i 0
+foreach id {1 2 3 2 3 1} {
+  do_execsql_test 14.1.$i {
+    SELECT data.v=nm.v FROM data,nm WHERE data.k=$id AND nm.k=$id
+  } 1
+  incr i
+}
+
+if {[file exists /dev/null]} {
+  do_catchsql_test 14.2 {
+    INSERT INTO nm_files(filename) VALUES('/dev/null');
+  } {1 {failed to read zonefile header from file "/dev/null"}}
+}
+
+finish_test
+

ext/zonefile/zonefileenc.test

@@ -0,0 +1,285 @@
+# 2018 Feb 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the zonefile extension.
+#
+
+package require Tcl 8.6
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source [file join $testdir tester.tcl]
+set testprefix zonefileenc
+
+
+foreach {tn code} {
+  1 { 
+    set K {
+      braking bramble brambles brambly
+      bran branch branched branches
+      branching branchings brand branded
+    }
+    set textkey 1
+  }
+  2 {
+    set K {
+      5e008542742ce0442e37cbf2512e9492    c91c26e0573ca3464e037568c51126da
+      e90e17489c1aef80ac620c9059271a5a    163338707cbe4c72b18d1058a42c5c78
+      5c6b1e7c7c9e8e4a8d8fdc30dfc11bea    ff1012687828ecaac6c9ca86ea0f895e
+      a203f25eb11d4c6afa841dfcf7cd0be0    b6c71e38ca914c460926ef90db39dba0
+      b38255d031d026c258a0a41a9a75d46a    adccca5e5ffa3a7625144a345713aef0
+      cd423b38b73e42ce5894405e6d0e08c0    b460ad2e370a0386726d6ea46e7b0bac
+      503b81de72cb3ef87d9346a850040000    369c290a464a6b88bfd9d1c4755afd42
+      a8a9343efca528f2bf23a972be49dd66    e366b5226bfe3fd0010fa814aae3b996
+      4cad7e80124c2cd447131bae377e60f6    4a0fd2f054e1b08cad0de2dc6aa93246
+      8a23c85e3337da2c97d498f806870fa8    8d14e1f055fd9bec7d07cf0e8baae042
+      7f6954b0dc373028ab3b030aaf44dd58    d220164c3898435a946de6bcbb478cc4
+      566af7ea88ba4ff87fd868e858cf98ea    a5405832235e8f601516f9c49767bdac
+      1bd5b4dc6b54e5ca92ba67d20bf65740    59da30e203bf73840e38e108b83ddb82
+      e516924c2cdf3114f10f2f0e1bdabbc6    b55dd27222a39764222838007e749984
+      190ae9f81b86a5a024e3b97ee2a7121c    469660843a9a9e507d0fb43e92029296
+      e6e600bc063aad12f6387beef650c48a    3097be5c3a52a2f00747587add01b550
+    }
+    set textkey 0
+  }
+} {
+  reset_db
+  load_static_extension db zonefile
+  set nFile 100
+  eval $code
+
+  do_execsql_test 1.$tn.0 {
+    CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
+    CREATE TABLE rr(k INTEGER PRIMARY KEY, v);
+  }
+  do_test 1.$tn.1 {
+    for {set i 0} {$i < $nFile} {incr i} {
+      set k [lindex $K [expr $i % [llength $K]]]
+      execsql {
+        DELETE FROM zz;
+        INSERT INTO zz VALUES($i*10+1, 1, -1, randomblob(100));
+        INSERT INTO zz VALUES($i*10+2, 2, -1, randomblob(100));
+        INSERT INTO zz VALUES($i*10+3, 1, -1, randomblob(100));
+        INSERT INTO rr SELECT k,v FROM zz;
+  
+        WITH p(n,v) AS (
+            VALUES('encryptionType', 'xor') UNION ALL
+            VALUES('debugEncryptionKeyText', $textkey) UNION ALL
+            VALUES('encryptionKey', $k)
+        )
+        SELECT zonefile_write('test' || $i || '.zonefile', 'zz', 
+            json_group_object(n, v)
+        ) FROM p;
+      }
+    }
+  } {}
+  
+  proc k {i} { 
+    set val [lindex $::K [expr $i % [llength $::K]]]
+    if {$::textkey==0} {
+      return [binary decode hex $val]
+    }
+    return $val
+  }
+  db func k k
+  
+  do_execsql_test 1.$tn.2 {
+    CREATE VIRTUAL TABLE gg USING zonefile;
+  }
+  for {set i 0} {$i < $nFile} {incr i} {
+    do_execsql_test 1.$tn.2.$i { 
+      INSERT INTO gg_files(filename, ekey) 
+        VALUES('test' || $i || '.zonefile', k($i));
+      SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v;
+    } 0
+  }
+  
+  db close
+  sqlite3 db test.db
+  load_static_extension db zonefile
+  db func k k
+  
+  do_catchsql_test 1.$tn.3 {
+    SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v;
+  } {1 {missing encryption key for file "test0.zonefile"}}
+  do_execsql_test 1.$tn.4 {
+    UPDATE gg_files SET ekey = k(0) WHERE filename='test0.zonefile';
+  }
+  do_execsql_test 1.$tn.4.2 {
+    SELECT count(*) FROM rr JOIN gg USING(k) 
+      WHERE rr.v==gg.v AND k IN (1,2,3);
+  } {3}
+  do_catchsql_test 1.5 {
+    SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v;
+  } {1 {missing encryption key for file "test1.zonefile"}}
+  
+  do_execsql_test 1.$tn.6 {
+    UPDATE gg_files SET ekey = k(rowid-1);
+  }
+  do_execsql_test 1.$tn.7 {
+    SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v;
+  } {0}
+  do_execsql_test 1.$tn.8 {
+    SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v==gg.v;
+  } {300}
+  
+  forcedelete test.db2
+  do_execsql_test 1.$tn.9.1 {
+    ATTACH 'test.db2' AS maing;
+    CREATE VIRTUAL TABLE maing.g USING zonefile;
+    INSERT INTO g_files(filename) SELECT filename FROM gg_files;
+  }
+  do_catchsql_test 1.$tn.9.2 {
+    SELECT count(*) FROM rr JOIN g USING(k) WHERE rr.v!=g.v;
+  } {1 {missing encryption key for file "test0.zonefile"}}
+  do_execsql_test 1.$tn.9.3 {
+    UPDATE g_files SET ekey = k(rowid-1);
+    SELECT count(*) FROM rr JOIN g USING(k) WHERE rr.v==g.v;
+  } {300}
+  
+  do_execsql_test 1.$tn.10 {
+    SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v==gg.v;
+  } {300}
+}
+#-------------------------------------------------------------------------
+
+reset_db
+load_static_extension db zonefile
+
+do_execsql_test 2.0 {
+  CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
+}
+foreach {tn alg id} {
+  1 aes_128_ctr 1
+  2 aes_128_cbc 2
+  3 AES_256_CTR 3
+  4 Aes_256_CBC 4
+} {
+  do_catchsql_test 2.1.$tn {
+    WITH p(n,v) AS (
+        VALUES('encryptionType', $alg) UNION ALL
+        VALUES('debugEncryptionKeyText', 1) UNION ALL
+        VALUES('encryptionKey', 'secret')
+    )
+    SELECT zonefile_write('test' || $i || '.zonefile', 'zz', 
+        json_group_object(n, v)
+    ) FROM p;
+  } "1 {unsupported encryption method: $id}"
+}
+
+foreach {tn alg} {
+  1 nosuchmethod! 
+} {
+  do_catchsql_test 2.2.$tn {
+    WITH p(n,v) AS (
+        VALUES('encryptionType', $alg) UNION ALL
+        VALUES('debugEncryptionKeyText', 1) UNION ALL
+        VALUES('encryptionKey', 'secret')
+    )
+    SELECT zonefile_write('test' || $i || '.zonefile', 'zz', 
+        json_group_object(n, v)
+    ) FROM p;
+  } "1 {unknown encryption method: $alg}"
+}
+
+#-------------------------------------------------------------------------
+# Test some hash collisions in the encryption key table.
+#
+
+# This is the same hash function used internally to store keys.
+#
+proc hash {zDb zTab iFile} {
+  binary scan $zDb c*  A
+    binary scan $zTab c* B
+    set h 0
+    foreach i $A { set h [expr ($h + ($h << 3) + $i) & 0xFFFFFFFF] }
+  foreach i $B { set h [expr ($h + ($h << 3) + $i) & 0xFFFFFFFF] }
+  return [expr $h ^ $iFile]
+}
+
+do_test 3.0 {
+  set h1 [expr [hash main zone 1] % 512]
+    for {set i 0} {1} {incr i} {
+      set h2 [expr [hash "aux$i" zone 1] % 512]
+        if {$h1==$h2} break
+    }
+  set i
+} 52
+
+reset_db
+load_static_extension db zonefile
+forcedelete test.db2
+
+do_execsql_test 3.1 {
+  CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
+  INSERT INTO zz VALUES(222, -1, -1, randomblob(60));
+  WITH p(n,v) AS (
+      VALUES('encryptionType', 'xor') UNION ALL
+      VALUES('debugEncryptionKeyText', 1) UNION ALL
+      VALUES('encryptionKey', 'pass')
+  )
+  SELECT zonefile_write('test1.zonefile', 'zz', 
+      json_group_object(n, v)
+  ) FROM p;
+
+  DELETE FROM zz;
+  INSERT INTO zz VALUES(333, -1, -1, randomblob(80));
+  WITH p(n,v) AS (
+      VALUES('encryptionType', 'xor') UNION ALL
+      VALUES('debugEncryptionKeyText', 1) UNION ALL
+      VALUES('encryptionKey', 'pass')
+  )
+  SELECT zonefile_write('test2.zonefile', 'zz', 
+      json_group_object(n, v)
+  ) FROM p;
+} {{} {}}
+
+do_execsql_test 3.2 {
+  ATTACH 'test.db2' AS aux52;
+  CREATE VIRTUAL TABLE main.zone USING zonefile;
+  CREATE VIRTUAL TABLE aux52.zone USING zonefile;
+  INSERT INTO main.zone_files(filename, ekey) VALUES('test1.zonefile', 'pass');
+  INSERT INTO aux52.zone_files(filename, ekey) VALUES('test2.zonefile', 'pass');
+}
+
+do_execsql_test 3.3 {
+  SELECT v IS NULL FROM main.zone;
+  SELECT v IS NULL FROM aux52.zone;
+} {0 0}
+
+do_test 3.4 {
+  set h1 [expr [hash main zone 1] % 512]
+  for {set i 0} {1} {incr i} {
+    set h2 [expr [hash "aux$i" zone 2] % 512]
+    if {$h1==$h2} break
+  }
+  set i
+} 682
+
+forcedelete test.db3
+do_execsql_test 3.5 {
+  ATTACH 'test.db3' AS aux682;
+  CREATE VIRTUAL TABLE aux682.zone USING zonefile;
+  INSERT INTO aux682.zone_files(filename, ekey) VALUES('test1.zonefile','pass');
+  INSERT INTO aux682.zone_files(filename, ekey) VALUES('test2.zonefile','pass');
+  INSERT INTO main.zone_files(filename, ekey) VALUES('test2.zonefile','pass');
+}
+
+do_execsql_test 3.6 {
+  SELECT v IS NULL FROM main.zone;
+  SELECT v IS NULL FROM aux682.zone;
+  SELECT v IS NULL FROM main.zone;
+} {0 0 0 0 0 0}
+
+
+finish_test
+

ext/zonefile/zonefilefault.test

@@ -0,0 +1,251 @@
+# 2018 Feb 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the zonefile extension.
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source [file join $testdir tester.tcl]
+source [file join $testdir malloc_common.tcl]
+set testprefix zonefilefault
+load_static_extension db zonefile
+
+do_execsql_test 1.0 {
+  CREATE TABLE tt(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
+  INSERT INTO tt VALUES(1, -1, -1, randomblob(100));
+  INSERT INTO tt VALUES(2, -1, -1, randomblob(200));
+  INSERT INTO tt VALUES(3,  1, -1, randomblob(300));
+  INSERT INTO tt VALUES(4,  2, -1, randomblob(400));
+}
+
+do_faultsim_test 1.1 -faults oom* -prep {
+  sqlite3 db test.db
+  load_static_extension db zonefile
+} -body {
+  execsql {
+    SELECT zonefile_write('test.zonefile', 'tt',
+        '{"encryptionType":"xor", "encryptionKey":"secret",
+          "debugEncryptionKeyText":1
+        }'
+    );
+  }
+} -test {
+  faultsim_test_result {0 {{}}}
+}
+
+set sql {
+    SELECT zonefile_write('test.zonefile', 'tt',
+        '{"compressionTypeContent":"zstd_global_dict"}'
+    );
+}
+set HAVE_ZSTD 0
+if {[catch { db eval $sql }]==0} {
+  set HAVE_ZSTD 1
+}
+
+if {$HAVE_ZSTD} {
+  do_faultsim_test 1.2 -faults oom* -prep {
+    sqlite3 db test.db
+    load_static_extension db zonefile
+  } -body {
+    execsql {
+      SELECT zonefile_write('test.zonefile', 'tt',
+          '{"compressionTypeContent":"zstd_global_dict"}'
+      );
+    }
+  } -test {
+    faultsim_test_result {0 {{}}}
+  }
+}
+
+do_execsql_test 1.3.0 { UPDATE tt SET frame = NULL; }
+do_faultsim_test 1.3 -faults oom* -prep {
+  sqlite3 db test.db
+  load_static_extension db zonefile
+} -body {
+  execsql {
+    SELECT zonefile_write('test.zonefile', 'tt');
+  }
+} -test {
+  faultsim_test_result {0 {{}}}
+}
+
+do_execsql_test 1.4.0 {
+  INSERT INTO tt VALUES(5, -1, -1, randomblob(100));
+  INSERT INTO tt VALUES(6, -1, -1, randomblob(100));
+  INSERT INTO tt VALUES(7, -1, -1, randomblob(100));
+  INSERT INTO tt VALUES(8, -1, -1, randomblob(100));
+  INSERT INTO tt VALUES(9, -1, -1, randomblob(100));
+  CREATE VIRTUAL TABLE ttz USING zonefile;
+}
+if {$HAVE_ZSTD} {
+  faultsim_save_and_close
+  do_faultsim_test 1.4 -faults oom* -prep {
+    faultsim_restore_and_reopen
+    load_static_extension db zonefile
+  } -body {
+    execsql {
+      SELECT zonefile_write('test.zonefile', 'tt',
+          '{"compressionTypeIndexData":"zstd"}'
+      );
+    }
+  } -test {
+    faultsim_test_result {0 {{}}}
+  }
+
+  faultsim_save_and_close
+  do_faultsim_test 1.5 -faults oom* -prep {
+    faultsim_restore_and_reopen
+    load_static_extension db zonefile
+  } -body {
+    execsql {
+      INSERT INTO ttz_files(filename) VALUES('test.zonefile');
+    }
+  } -test {
+    faultsim_test_result {0 {}}
+  }
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 2.0 {
+  SELECT zonefile_write('test.zonefile', 'tt',
+      '{"encryptionType":"xor", "encryptionKey":"secret",
+          "debugEncryptionKeyText":1
+      }'
+  );
+  CREATE VIRTUAL TABLE zz USING zonefile;
+} {{}}
+
+faultsim_save_and_close
+do_faultsim_test 2.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  load_static_extension db zonefile
+} -body {
+  execsql {
+    INSERT INTO zz_files(filename, ekey) VALUES('test.zonefile', 'secret')
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+faultsim_save_and_close
+do_faultsim_test 2.2 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  load_static_extension db zonefile
+} -body {
+  execsql { SELECT json_extract(header, '$.magicNumber') FROM zz_files }
+} -test {
+  faultsim_test_result {0 1179332920}
+}
+
+do_faultsim_test 2.3 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  load_static_extension db zonefile
+} -body {
+  execsql { DELETE FROM zz_files }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+faultsim_save_and_close
+do_faultsim_test 3.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  load_static_extension db zonefile
+} -body {
+  execsql { CREATE VIRTUAL TABLE t1 USING zonefile(cachesize=5) }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+faultsim_save_and_close
+do_faultsim_test 3.2 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  load_static_extension db zonefile
+} -body {
+  execsql { DROP TABLE t1 }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+load_static_extension db zonefile
+do_execsql_test 4.0 {
+  CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
+  INSERT INTO zz VALUES(1, -1, -1, randomblob(100));
+  INSERT INTO zz VALUES(2, -1, -1, randomblob(100));
+  INSERT INTO zz VALUES(3, -1, -1, randomblob(100));
+  SELECT zonefile_write('test.zonefile', 'zz');
+  CREATE VIRTUAL TABLE zone USING zonefile;
+  INSERT INTO zone_files(filename) VALUES('test.zonefile');
+} {{}}
+
+faultsim_save_and_close
+do_faultsim_test 4.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  load_static_extension db zonefile
+} -body {
+  execsql { SELECT v IS NULL FROM zone WHERE k = 2 }
+} -test {
+  faultsim_test_result {0 0}
+}
+
+
+if {$HAVE_ZSTD} {
+  set params {
+      {"encryptionType":"xor","encryptionKey":"pass",
+       "compressionTypeContent":"zstd_global_dict",
+          "debugEncryptionKeyText":1
+      }
+  }
+} else {
+  set params { 
+      {"encryptionType":"xor","encryptionKey":"pass",
+          "debugEncryptionKeyText":1
+      }
+  }
+}
+do_execsql_test 4.2 {
+  SELECT zonefile_write('test.zonefile', 'zz', $params);
+  CREATE VIRTUAL TABLE zone2 USING zonefile;
+  INSERT INTO zone2_files(filename,ekey) VALUES('test.zonefile','pass');
+} {{}}
+
+faultsim_save_and_close
+do_faultsim_test 4.3 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  load_static_extension db zonefile
+  execsql { UPDATE zone2_files SET ekey='pass' }
+} -body {
+  execsql { SELECT v IS NULL FROM zone2 WHERE k = 2 }
+} -test {
+  faultsim_test_result {0 0}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_faultsim_test 4.4 -faults oom* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  load_static_extension db zonefile
+} -test {
+  faultsim_test_result {0 {}} {1 {initialization of zonefile failed: }}
+}
+
+finish_test
+

main.mk

@@ -372,6 +372,7 @@ TESTSRC += \
   $(TOP)/ext/misc/wholenumber.c \
   $(TOP)/ext/misc/vfslog.c \
   $(TOP)/ext/misc/zipfile.c \
+  $(TOP)/ext/zonefile/zonefile.c \
   $(TOP)/ext/fts5/fts5_tcl.c \
   $(TOP)/ext/fts5/fts5_test_mi.c \
   $(TOP)/ext/fts5/fts5_test_tok.c

manifest

@@ -1,10 +1,10 @@
-C Fix\sharmless\scompiler\swarnings\sin\sthe\szipfile\sextension\sseen\swith\sMSVC.
-D 2018-02-23T13:38:54.742
+C Update\szonefile\sREADME.md\sfile\sto\smention\sthe\sframe\scache.
+D 2018-02-27T20:09:45.338
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc bf19d3a0eb849bd3b114653b0e455aa5b2799a96f413287a5866013db0e47f30
+F Makefile.msc 738b130a37c8855f46c519cdeab968a483042017b8e90a04a8156e26d6a3769d
 F README.md 1d5342ebda97420f114283e604e5fe99b0da939d63b76d492eabbaae23488276
 F VERSION cdf91ac446255ecf3d8f6d8c3ee40d64123235ae5b3cef29d344e61b45ec3759
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -408,10 +408,15 @@ F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
+F ext/zonefile/README.md 49bc392fe0dde788cc63b2ea71e26cfba02141b6ac192e245668d0b7b43b5750
+F ext/zonefile/zonefile.c ce917d60a62c8e384087954f482ab703b75a0660385e7a685bcafc0cff1c3038
+F ext/zonefile/zonefile1.test 3719e1d069064f4e12e915f8d9f6256c769b1582258b3997e0492076a01d5f36
+F ext/zonefile/zonefileenc.test e8624853ac224bfdeab8d2a231796499c3032e2a8bb1d2208f52fddf9507e38f
+F ext/zonefile/zonefilefault.test 6df281fc1ec7859d4ae2a3133d8783184a263edf6d081218958b6ae86b22ee13
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 04969462bfd32c9f08d4a6d40622e8c43c8c5ecfb2ee52ffb5737c5eca4b0c03
+F main.mk e5ac915f9959445602c07fd5aee00c5c09b4616658e67aa12e765399cd686763
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -498,7 +503,7 @@ F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c 11a2618c227fd13ccad73ee02d1199f9880c59db2b3144fd7432db1980a2577d
-F src/test1.c 1ab7cbbb6693e08364c1a9241e2aee17f8c4925e4cc52396be77ae6845a05828
+F src/test1.c 6b0dfd4f0a1996805bc04381b45df71ead9ed84634d2a47a757ab292febf9026
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b
 F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6
@@ -1707,7 +1712,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e9e9f27b3d496531905ad7459cf60366ad3798f507266134dd6388cdec50fdc4
-R 654cd747b1d6661b630741ec0eabab23
-U mistachkin
-Z f6be006b93932e4aec25faabbec16d8c
+P f11beb16a87cc967e896cf67121b1e4045e427ebdc6a424e9f009ffced955d36
+R ce086be2fa4585d5057d94c9c14e09e6
+U dan
+Z fcd96707fe4566207f6172155f7ee098

manifest.uuid

@@ -1 +1 @@
-15c587cf6ffc920dc37f67cacb4f0db6a603fb998a22a639f755387910516414
+84e9351bbd0a9bf9e1d64a441366a90276c231b44548cbc947f7fb2842f82490

src/test1.c

@@ -6992,6 +6992,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd(
 #ifdef SQLITE_HAVE_ZLIB
   extern int sqlite3_zipfile_init(sqlite3*,char**,const sqlite3_api_routines*);
 #endif
+  extern int sqlite3_zonefile_init(sqlite3*,char**,const sqlite3_api_routines*);
   static const struct {
     const char *zExtName;
     int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*);
@@ -7016,6 +7017,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd(
 #ifdef SQLITE_HAVE_ZLIB
     { "zipfile",               sqlite3_zipfile_init              },
 #endif
+    { "zonefile",              sqlite3_zonefile_init             },
   };
   sqlite3 *db;
   const char *zName;