Cause CLI .sha3sum to warn of text fields that do not survive CAST(CAST(t as BLOB) AS TEXT) due to invalid UTF encoding.

FossilOrigin-Name: 123f2a0785790addf9c60a0fd09077dda9cb84d33a2594901a2b22bb555be491

manifest

@@ -1,5 +1,5 @@
-C Rename\swasm.xWrap.resultAdapter()\sX:free\sentries\sto\sX:dealloc\sfor\sconsistency\swith\swasm.dealloc().\sAdd\san\sundocumented\sfeature\sto\sreplace\swasm.alloc/dealloc/realloc()\swith\sthe\sC-standard\sallocators\s(after\san\sallocator\smisuse\sled\sdown\sa\sseveral-hour\srabbit\shole\strying\sto\sdiscover\sa\smis-free()\sviolation).\sRelated\stest\supdates.
-D 2022-12-03T13:10:58.483
+C Cause\sCLI\s.sha3sum\sto\swarn\sof\stext\sfields\sthat\sdo\snot\ssurvive\sCAST(CAST(t\sas\sBLOB)\sAS\sTEXT)\sdue\sto\sinvalid\sUTF\sencoding.
+D 2022-12-03T16:09:32.554
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633
 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
 F src/select.c 6983de0e6b8b68c97f82f9fca27ffb8f17161cff4d0d48fdf9eb40b213da0cc6
-F src/shell.c.in 9fda74d40b206a707aaa69fc5dc38e2c6a9137a3f4a1dcd7af581d59d92c063c
+F src/shell.c.in aa3e4c78ae25c24562b0c3497a88875baa61425714437143170b2c462842035d
 F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f
@@ -2065,8 +2065,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ed1ed21221b048ac5a5275cdfc4d9b2a406acdc7d4b648c3b61bcc822d88d955
-R 04802c5e85209b72659bc288df2e94a1
-U stephan
-Z 23bb6930a31350fa896c81241f2934b6
+P d9807656f8a7c2a893d3f68ee5592f44826b8e999ae66f7d9000674b5c1b0207
+R 9f83c1fb14dddfc272778480afabb0a3
+T *branch * sha3sum_text_validation
+T *sym-sha3sum_text_validation *
+T -sym-trunk *
+U larrybr
+Z 1ba97ab620561a8dccaf913734ec9094
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-d9807656f8a7c2a893d3f68ee5592f44826b8e999ae66f7d9000674b5c1b0207
+123f2a0785790addf9c60a0fd09077dda9cb84d33a2594901a2b22bb555be491

src/shell.c.in

@@ -10077,6 +10077,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     sqlite3_stmt *pStmt;     /* For querying tables names */
     char *zSql;              /* SQL to be run */
     char *zSep;              /* Separator */
+    char *zRevText;          /* Query for reversible to-blob-to-text check */
     ShellText sSql;          /* Complete SQL for the query to run the hash */
     ShellText sQuery;        /* Set of queries used to read all content */
     open_db(p, 0);
@@ -10114,17 +10115,35 @@ static int do_meta_command(char *zLine, ShellState *p){
       }
     }
     if( bSchema ){
-      zSql = "SELECT lower(name) FROM sqlite_schema"
+      zSql = "SELECT lower(name) as tname FROM sqlite_schema"
              " WHERE type='table' AND coalesce(rootpage,0)>1"
              " UNION ALL SELECT 'sqlite_schema'"
              " ORDER BY 1 collate nocase";
     }else{
-      zSql = "SELECT lower(name) FROM sqlite_schema"
+      zSql = "SELECT lower(name) as tname FROM sqlite_schema"
              " WHERE type='table' AND coalesce(rootpage,0)>1"
              " AND name NOT LIKE 'sqlite_%'"
              " ORDER BY 1 collate nocase";
     }
     sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+    zRevText = sqlite3_mprintf(
+        /* lower-case query is first run, producing upper-case query. */
+        "with text_cols as materialized("
+        "select tname, cname "
+        "from ("
+        " select ss.tname as tname, ti.name as cname, lower(type) as lca"
+        " from (%s) ss join pragma_table_info(tname) ti"
+        " where (instr(lca,'int')=0 and "
+        " instr(lca,'char')+instr(lca,'clob')+instr(lca,'text')>0)"
+        "))"
+        "select 'SELECT total(bad_text_count) AS bad_text_count"
+        " FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query"
+        " from (select 'SELECT COUNT(*) AS bad_text_count"
+        " FROM '||tname||' WHERE '"
+        " ||group_concat('CAST(CAST('||cname||' AS BLOB) AS TEXT)<>'||cname,"
+        " ' OR ') as query, tname from text_cols group by tname)",
+        zSql);
+    shell_check_oom(zRevText);
     initText(&sQuery);
     initText(&sSql);
     appendText(&sSql, "WITH [sha3sum$query](a,b) AS(",0);
@@ -10180,7 +10199,35 @@ static int do_meta_command(char *zLine, ShellState *p){
     }else{
       shell_exec(p, zSql, 0);
     }
+    {
+      int lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0);
+      if( bDebug ) utf8_printf(p->out, "%s\n", zRevText);
+      assert(lrc==SQLITE_OK);
+      lrc = SQLITE_ROW==sqlite3_step(pStmt);
+      assert(lrc!=0);
+      if( lrc ){
+        const char *zGenQuery = sqlite3_column_text(pStmt,0);
+        sqlite3_stmt *pCheckStmt;
+        double countIrreversible;
+        lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0);
+        if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery);
+        assert(lrc==SQLITE_OK);
+        lrc = SQLITE_ROW==sqlite3_step(pCheckStmt);
+        assert(lrc!=0);
+        countIrreversible = sqlite3_column_double(pCheckStmt, 0);
+        sqlite3_finalize(pStmt);
+        sqlite3_finalize(pCheckStmt);
+        if( countIrreversible>0 ){
+          int n = (int)(countIrreversible + 0.5);
+          utf8_printf(stderr,
+                      "Digest includes %d invalidly encoded text field%s.\n",
+                      n, (n>1)? "s": "");
+        }
+      }
+
+    }
     sqlite3_free(zSql);
+    sqlite3_free(zRevText);
   }else
 
 #if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE)