Better comments on the bindvtab.c implementation. All the two-argument

version of the .set command.  All bindings from .set and -D are still
string.

FossilOrigin-Name: 7b2a65a65475b80669af4ff5ac7adb6b310cd8275a36e672a3f5c88a259693c0

Makefile.in

@@ -187,7 +187,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
          random.lo resolve.lo rowset.lo rtree.lo \
          sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
          table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
-         update.lo util.lo vacuum.lo \
+         update.lo upsert.lo util.lo vacuum.lo \
          vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
          vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
          utf.lo vtab.lo
@@ -284,6 +284,7 @@ SRC = \
   $(TOP)/src/trigger.c \
   $(TOP)/src/utf.c \
   $(TOP)/src/update.c \
+  $(TOP)/src/upsert.c \
   $(TOP)/src/util.c \
   $(TOP)/src/vacuum.c \
   $(TOP)/src/vdbe.c \
@@ -914,6 +915,9 @@ trigger.lo:	$(TOP)/src/trigger.c $(HDR)
 update.lo:	$(TOP)/src/update.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/update.c
 
+upsert.lo:	$(TOP)/src/upsert.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/upsert.c
+
 utf.lo:	$(TOP)/src/utf.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/utf.c
 
@@ -1008,6 +1012,7 @@ SHELL_SRC = \
 	$(TOP)/ext/misc/shathree.c \
 	$(TOP)/ext/misc/fileio.c \
 	$(TOP)/ext/misc/completion.c \
+        $(TOP)/ext/misc/bindvtab.c \
 	$(TOP)/ext/misc/sqlar.c \
 	$(TOP)/ext/expert/sqlite3expert.c \
 	$(TOP)/ext/expert/sqlite3expert.h \

Makefile.msc

@@ -1193,7 +1193,7 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \
          random.lo resolve.lo rowset.lo rtree.lo \
          sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
          table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
-         update.lo util.lo vacuum.lo \
+         update.lo upsert.lo util.lo vacuum.lo \
          vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
          vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
          utf.lo vtab.lo
@@ -1292,6 +1292,7 @@ SRC01 = \
   $(TOP)\src\trigger.c \
   $(TOP)\src\utf.c \
   $(TOP)\src\update.c \
+  $(TOP)\src\upsert.c \
   $(TOP)\src\util.c \
   $(TOP)\src\vacuum.c \
   $(TOP)\src\vdbe.c \
@@ -1993,6 +1994,9 @@ trigger.lo:	$(TOP)\src\trigger.c $(HDR)
 update.lo:	$(TOP)\src\update.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\update.c
 
+upsert.lo:	$(TOP)\src\upsert.c $(HDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\upsert.c
+
 utf.lo:	$(TOP)\src\utf.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\utf.c
 
@@ -2095,6 +2099,7 @@ SHELL_SRC = \
 	$(TOP)\ext\misc\shathree.c \
 	$(TOP)\ext\misc\fileio.c \
 	$(TOP)\ext\misc\completion.c \
+        $(TOP)\ext\misc\bindvtab.c \
 	$(TOP)\ext\expert\sqlite3expert.c \
 	$(TOP)\ext\expert\sqlite3expert.h \
 	$(TOP)\src\test_windirent.c

VERSION

@@ -1 +1 @@
-3.23.1
+3.24.0

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.23.1.
+# Generated by GNU Autoconf 2.69 for sqlite 3.24.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.23.1'
-PACKAGE_STRING='sqlite 3.23.1'
+PACKAGE_VERSION='3.24.0'
+PACKAGE_STRING='sqlite 3.24.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1465,7 +1465,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures sqlite 3.23.1 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.24.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1530,7 +1530,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.23.1:";;
+     short | recursive ) echo "Configuration of sqlite 3.24.0:";;
    esac
   cat <<\_ACEOF
 
@@ -1655,7 +1655,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-sqlite configure 3.23.1
+sqlite configure 3.24.0
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2074,7 +2074,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by sqlite $as_me 3.23.1, which was
+It was created by sqlite $as_me 3.24.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -12242,7 +12242,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.23.1, which was
+This file was extended by sqlite $as_me 3.24.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -12308,7 +12308,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.23.1
+sqlite config.status 3.24.0
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

doc/lemon.html

@@ -99,6 +99,9 @@ Show only the basis for each parser state in the report file.
 <li><b>-c</b>
 Do not compress the generated action tables.  The parser will be a
 little larger and slower, but it will detect syntax errors sooner.
+<li><b>-d</b><i>directory</i>
+Write all output files into <i>directory</i>.  Normally, output files
+are written into the directory that contains the input grammar file.
 <li><b>-D<i>name</i></b>
 Define C preprocessor macro <i>name</i>.  This macro is usable by
 "<tt><a href='#pifdef'>%ifdef</a></tt>" and
@@ -679,6 +682,30 @@ of type "MyStruct*" and all action routines will have access to
 a variable named "pAbc" that is the value of the 4th parameter
 in the most recent call to Parse().</p>
 
+<p>The <tt>%extra_context</tt> directive works the same except that it
+is passed in on the ParseAlloc() or ParseInit() routines instead of
+on Parse().
+
+<a name='extractx'></a>
+<h4>The <tt>%extra_context</tt> directive</h4>
+
+The <tt>%extra_context</tt> directive instructs Lemon to add a 2th parameter
+to the parameter list of the ParseAlloc() and ParseInif() functions.  Lemon
+doesn't do anything itself with these extra argument, but it does
+store the value make it available to C-code action routines, destructors,
+and so forth.  For example, if the grammar file contains:</p>
+
+<p><pre>
+    %extra_context { MyStruct *pAbc }
+</pre></p>
+
+<p>Then the ParseAlloc() and ParseInit() functions will have an 2th parameter
+of type "MyStruct*" and all action routines will have access to
+a variable named "pAbc" that is the value of that 2th parameter.</p>
+
+<p>The <tt>%extra_argument</tt> directive works the same except that it
+is passed in on the Parse() routine instead of on ParseAlloc()/ParseInit().
+
 <a name='pfallback'></a>
 <h4>The <tt>%fallback</tt> directive</h4>
 

ext/expert/expert1.test

@@ -285,6 +285,7 @@ do_setup_rec_test $tn.14 {
   INSERT INTO t9 VALUES(?, ?, ?);
 } {
   CREATE INDEX t10_idx_00000062 ON t10(b); 
+  0|1|0|-- TRIGGER t9t
   0|0|0|SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
 }
 
@@ -379,4 +380,3 @@ do_execsql_test 3.2 {
 
 
 finish_test
-

ext/misc/bindvtab.c

@@ -0,0 +1,476 @@
+/*
+** 2018-04-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file implements a simple key/value store used to hold bind
+** parameters for SQLite.  The key/value store is a singleton - there
+** is exactly one per process.  The store can be accessed and controlled
+** from SQLite using an eponymous virtual table.
+**
+** This is used to do parameter binding in the command-line shell.
+**
+** The ".set key=value" command and the "-Dkey=value" command-line option
+** invoke shell_bindings_new_text() on the argument ("key=value") in order
+** to create entries in the store.  The CLI then invokes
+** shell_bindings_apply() on each prepared statement just prior to
+** running it.
+**
+** All bindings are accessible through an eponymous-only virtual table
+** named shell_bindings.  Ex:
+**
+**    INSERT INTO shell_bindings(k,v) VALUES('p1',12345);
+**    SELECT $p1, typeof($p1);
+**
+** The above results in an answer of 12345,'integer'.  Bindings generated
+** using the virtual table can have any type other than NULL.  But bindings
+** generated by the .set command and the -D command-line option are always
+** text.
+**
+** The CLI is single-threaded, so there is no attempt to make this code
+** threadsafe.
+**
+** The key/value store is kept in a global list, and uses malloc/free rather
+** than sqlite3_malloc64/sqlite3_free so that it can be completely independent
+** of SQLite, can exist both before sqlite3_initialize() and after
+** sqlite3_shutdown(), and so that it will persist across multiple
+** connections created using ".open".
+**
+** The number of parameters is expected to be small, so they are stored
+** on a simple linked list.  If this proves to be too inefficient, some other
+** algorithm can be substituted in the future without changing the interface.
+*/
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
+SQLITE_EXTENSION_INIT1
+#include <string.h>
+#include <assert.h>
+#include <stdlib.h>
+
+/* Each entry in the key/value store */
+typedef struct BindingEntry  BindingEntry;
+struct BindingEntry {
+  char *zKey;             /* Key */
+  BindingEntry *pNext;    /* Next entry in the list */
+  BindingEntry *pPrev;    /* Previous entry in the list */
+  int eType;              /* SQLITE_INTEGER, _FLOAT, _TEXT, or _BLOB */
+  int len;                /* Length for SQLITE_BLOB values */
+  union {
+    sqlite3_int64 i;         /* Integer value */
+    double r;                /* Real value */
+    char *z;                 /* Text value */
+    unsigned char *b;        /* Blob value */
+  } u;
+};
+
+/* Global list of all entries */
+static BindingEntry *global_pAll = 0;
+
+/* Locate any entry with the given key.  Return NULL if not found.
+*/
+static BindingEntry *shellBindingFind(const char *zKey){
+  BindingEntry *p;
+  for(p=global_pAll; p && strcmp(p->zKey,zKey)!=0; p = p->pNext){}
+  return p;
+}
+
+/* Delete any entry with the given key, if it exists.
+*/
+static void shellBindingDelete(const char *zKey){
+  BindingEntry *p;
+  p = shellBindingFind(zKey);
+  if( p ){
+    if( p->pNext ){
+      p->pNext->pPrev = p->pPrev;
+    }
+    if( p->pPrev ){
+      p->pPrev->pNext = p->pNext;
+    }else{
+      global_pAll = p->pNext;
+    }
+    free(p);
+  }
+}
+
+/* Insert a new shell binding */
+static void shellBindingInsert(BindingEntry *p){
+  p->pNext = global_pAll;
+  if( global_pAll ) global_pAll->pPrev = p;
+  global_pAll = p;
+  p->pPrev = 0;
+}
+
+/*
+** True if c is a valid ID character.
+*/
+static int shellBindIdChar(char c){
+  if( c>='a' && c<='z' ) return 1;
+  if( c>='A' && c<='Z' ) return 1;
+  if( c=='_' ) return 1;
+  if( c>='0' && c<='9' ) return 2;
+  return 0;
+}
+
+/* Create a new binding given a string of the form "KEY=VALUE".  Return
+** values:
+**
+**    0:    success
+**    1:    out of memory
+**    2:    Argument is not a valid KEY=VALUE string
+**
+** The type of VALUE is TEXT.
+*/
+int shell_bindings_new_text(const char *z){
+  int i;
+  int nKey;
+  int nData;
+  BindingEntry *p;
+  for(i=0; shellBindIdChar(z[i]); i++){}
+  if( i==0 ) return 2;
+  if( shellBindIdChar(z[0])==2 ) return 2;
+  nKey = i;
+  if( z[i]!='=' ) return 2;
+  for(nData=0; z[nKey+1+nData]; nData++){}
+  p = malloc( sizeof(*p) + nKey + nData + 2 );
+  if( p==0 ) return 1;
+  memset(p, 0, sizeof(*p));
+  p->zKey = (char*)&p[1];
+  memcpy(p->zKey, z, nKey);
+  p->zKey[nKey] = 0;
+  p->u.z = &p->zKey[nKey+1];
+  p->len = nData;
+  p->eType = SQLITE_TEXT;
+  memcpy(p->u.z, &z[nKey+1], nData+1);
+  shellBindingDelete(p->zKey);
+  shellBindingInsert(p);
+  return 0;
+}
+
+/*
+** Delete all shell bindings
+*/
+void shell_bindings_clear(void){
+  BindingEntry *pNext;
+  while( global_pAll ){
+    pNext = global_pAll->pNext;
+    free(global_pAll);
+    global_pAll = pNext;
+  }
+}
+
+/* Given a prepared statement, apply all bindings for which there are
+** known values in the k-v store
+*/
+void shell_bindings_apply(sqlite3_stmt *pStmt){
+  int n = sqlite3_bind_parameter_count(pStmt);
+  int i;
+  BindingEntry *p;
+  for(i=1; i<=n; i++){
+    const char *zKey = sqlite3_bind_parameter_name(pStmt, i);
+    if( zKey==0 || zKey[0]==0 ) continue;
+    zKey++;
+    p = shellBindingFind(zKey);
+    if( p==0 ) continue;
+    switch( p->eType ){
+      case SQLITE_INTEGER:
+        sqlite3_bind_int64(pStmt, i, p->u.i);
+        break;
+      case SQLITE_FLOAT:
+        sqlite3_bind_double(pStmt, i, p->u.r);
+        break;
+      case SQLITE_TEXT:
+        sqlite3_bind_text(pStmt, i, p->u.z, p->len, SQLITE_TRANSIENT);
+        break;
+      case SQLITE_BLOB:
+        sqlite3_bind_blob(pStmt, i, p->u.b, p->len, SQLITE_TRANSIENT);
+        break;
+    }
+  }
+}
+
+/* bindvtab_vtab is a subclass of sqlite3_vtab which is
+** underlying representation of the virtual table
+*/
+typedef struct bindvtab_vtab bindvtab_vtab;
+struct bindvtab_vtab {
+  sqlite3_vtab base;  /* Base class - must be first */
+};
+
+/* bindvtab_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result
+*/
+typedef struct bindvtab_cursor bindvtab_cursor;
+struct bindvtab_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  BindingEntry *p;           /* Current entry in the scan */
+};
+
+/*
+** The bindvtabConnect() method is invoked to create a new
+** template virtual table.
+**
+** Think of this routine as the constructor for bindvtab_vtab objects.
+**
+** All this routine needs to do is:
+**
+**    (1) Allocate the bindvtab_vtab object and initialize all fields.
+**
+**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+**        result set of queries against the virtual table will look like.
+*/
+static int bindvtabConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  bindvtab_vtab *pNew;
+  int rc;
+
+  rc = sqlite3_declare_vtab(db,
+           "CREATE TABLE shell_bindings(k TEXT PRIMARY KEY,v)"
+           " WITHOUT ROWID"
+       );
+  /* For convenience, define symbolic names for the index to each column. */
+#define BINDVTAB_KEY    0
+#define BINDVTAB_VALUE  1
+  if( rc==SQLITE_OK ){
+    pNew = sqlite3_malloc( sizeof(*pNew) );
+    *ppVtab = (sqlite3_vtab*)pNew;
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for bindvtab_vtab objects.
+*/
+static int bindvtabDisconnect(sqlite3_vtab *pVtab){
+  bindvtab_vtab *p = (bindvtab_vtab*)pVtab;
+  sqlite3_free(p);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new bindvtab_cursor object.
+*/
+static int bindvtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  bindvtab_cursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destructor for a bindvtab_cursor.
+*/
+static int bindvtabClose(sqlite3_vtab_cursor *cur){
+  bindvtab_cursor *pCur = (bindvtab_cursor*)cur;
+  sqlite3_free(pCur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a bindvtab_cursor to its next row of output.
+*/
+static int bindvtabNext(sqlite3_vtab_cursor *cur){
+  bindvtab_cursor *pCur = (bindvtab_cursor*)cur;
+  pCur->p = pCur->p->pNext;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the bindvtab_cursor
+** is currently pointing.
+*/
+static int bindvtabColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  bindvtab_cursor *pCur = (bindvtab_cursor*)cur;
+  BindingEntry *p = pCur->p;
+  if( i==BINDVTAB_KEY ){
+    sqlite3_result_text(ctx, p->zKey, -1, SQLITE_TRANSIENT);
+  }else{
+    assert( i==BINDVTAB_VALUE );
+    switch( p->eType ){
+      case SQLITE_INTEGER:
+        sqlite3_result_int(ctx, p->u.i);
+        break;
+      case SQLITE_FLOAT:
+        sqlite3_result_double(ctx, p->u.r);
+        break;
+      case SQLITE_TEXT:
+        sqlite3_result_text(ctx, p->u.z, p->len, SQLITE_TRANSIENT);
+        break;
+      case SQLITE_BLOB:
+        sqlite3_result_blob(ctx, p->u.b, p->len, SQLITE_TRANSIENT);
+        break;
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.  In this implementation, the
+** rowid is the same as the output value.
+*/
+static int bindvtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int bindvtabEof(sqlite3_vtab_cursor *cur){
+  bindvtab_cursor *pCur = (bindvtab_cursor*)cur;
+  return pCur->p==0;
+}
+
+/*
+** This method is called to "rewind" the bindvtab_cursor object back
+** to the first row of output.  This method is always called at least
+** once prior to any call to bindvtabColumn() or bindvtabRowid() or 
+** bindvtabEof().
+*/
+static int bindvtabFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  bindvtab_cursor *pCur = (bindvtab_cursor *)pVtabCursor;
+  pCur->p = global_pAll;
+  return SQLITE_OK;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+*/
+static int bindvtabBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  pIdxInfo->estimatedCost = (double)10;
+  pIdxInfo->estimatedRows = 10;
+  return SQLITE_OK;
+}
+
+/*
+** Called to make changes to the shell bindings
+*/
+static int bindvtabUpdate(
+  sqlite3_vtab *pVTab,
+  int argc,
+  sqlite3_value **argv,
+  sqlite_int64 *pRowid
+){
+  const char *zKey;
+  BindingEntry *p;
+  int nKey;
+  int len;
+  int eType;
+  if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){
+    zKey = (const char*)sqlite3_value_text(argv[0]);
+    if( zKey ) shellBindingDelete(zKey);
+  }
+  if( argc==1 ) return SQLITE_OK;
+  eType = sqlite3_value_type(argv[3]);
+  if( eType==SQLITE_NULL ) return SQLITE_OK;
+  zKey = (const char*)sqlite3_value_text(argv[2]);
+  if( zKey==0 ) return SQLITE_OK;
+  nKey = sqlite3_value_bytes(argv[2]);
+  shellBindingDelete(zKey);
+  if( eType==SQLITE_BLOB || eType==SQLITE_TEXT ){
+    len = sqlite3_value_bytes(argv[3]);
+  }else{
+    len = 0;
+  }
+  p = malloc( sizeof(*p) + nKey + len + 2 );
+  if( p==0 ) return SQLITE_NOMEM;
+  memset(p, 0, sizeof(*p));
+  p->zKey = (char*)&p[1];
+  memcpy(p->zKey, zKey, nKey+1);
+  p->eType = eType;
+  switch( eType ){
+    case SQLITE_INTEGER: 
+       p->u.i = sqlite3_value_int64(argv[3]);
+       break;
+    case SQLITE_FLOAT: 
+       p->u.r = sqlite3_value_double(argv[3]);
+       break;
+    case SQLITE_TEXT:
+       p->u.z = &p->zKey[nKey+1];
+       memcpy(p->u.z, sqlite3_value_text(argv[3]), len);
+       break;
+    case SQLITE_BLOB:
+       p->u.b = (unsigned char*)&p->zKey[nKey+1];
+       memcpy(p->u.b, sqlite3_value_blob(argv[3]), len);
+       break;
+  }
+  shellBindingInsert(p);
+  return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the 
+** virtual table.
+*/
+static sqlite3_module bindvtabModule = {
+  /* iVersion    */ 0,
+  /* xCreate     */ 0,
+  /* xConnect    */ bindvtabConnect,
+  /* xBestIndex  */ bindvtabBestIndex,
+  /* xDisconnect */ bindvtabDisconnect,
+  /* xDestroy    */ 0,
+  /* xOpen       */ bindvtabOpen,
+  /* xClose      */ bindvtabClose,
+  /* xFilter     */ bindvtabFilter,
+  /* xNext       */ bindvtabNext,
+  /* xEof        */ bindvtabEof,
+  /* xColumn     */ bindvtabColumn,
+  /* xRowid      */ bindvtabRowid,
+  /* xUpdate     */ bindvtabUpdate,
+  /* xBegin      */ 0,
+  /* xSync       */ 0,
+  /* xCommit     */ 0,
+  /* xRollback   */ 0,
+  /* xFindMethod */ 0,
+  /* xRename     */ 0,
+  /* xSavepoint  */ 0,
+  /* xRelease    */ 0,
+  /* xRollbackTo */ 0
+};
+
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_bindvtab_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+  rc = sqlite3_create_module(db, "shell_bindings", &bindvtabModule, 0);
+  return rc;
+}

ext/misc/completion.c

@@ -62,6 +62,7 @@ struct completion_cursor {
   char *zPrefix;             /* The prefix for the word we want to complete */
   char *zLine;               /* The whole that we want to complete */
   const char *zCurrentRow;   /* Current output row */
+  int szRow;                 /* Length of the zCurrentRow string */
   sqlite3_stmt *pStmt;       /* Current statement */
   sqlite3_int64 iRowid;      /* The rowid */
   int ePhase;                /* Current phase */
@@ -174,32 +175,6 @@ static int completionClose(sqlite3_vtab_cursor *cur){
   return SQLITE_OK;
 }
 
-/*
-** All SQL keywords understood by SQLite
-*/
-static const char *completionKwrds[] = {
-  "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
-  "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
-  "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
-  "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
-  "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
-  "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
-  "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
-  "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
-  "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
-  "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
-  "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
-  "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
-  "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
-  "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
-  "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
-  "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
-  "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
-  "WITH", "WITHOUT",
-};
-#define completionKwCount \
-   (int)(sizeof(completionKwrds)/sizeof(completionKwrds[0]))
-
 /*
 ** Advance a completion_cursor to its next row of output.
 **
@@ -222,11 +197,11 @@ static int completionNext(sqlite3_vtab_cursor *cur){
   while( pCur->ePhase!=COMPLETION_EOF ){
     switch( pCur->ePhase ){
       case COMPLETION_KEYWORDS: {
-        if( pCur->j >= completionKwCount ){
+        if( pCur->j >= sqlite3_keyword_count() ){
           pCur->zCurrentRow = 0;
           pCur->ePhase = COMPLETION_DATABASES;
         }else{
-          pCur->zCurrentRow = completionKwrds[pCur->j++];
+          sqlite3_keyword_name(pCur->j++, &pCur->zCurrentRow, &pCur->szRow);
         }
         iCol = -1;
         break;
@@ -298,6 +273,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){
       if( sqlite3_step(pCur->pStmt)==SQLITE_ROW ){
         /* Extract the next row of content */
         pCur->zCurrentRow = (const char*)sqlite3_column_text(pCur->pStmt, iCol);
+        pCur->szRow = sqlite3_column_bytes(pCur->pStmt, iCol);
       }else{
         /* When all rows are finished, advance to the next phase */
         sqlite3_finalize(pCur->pStmt);
@@ -307,7 +283,9 @@ static int completionNext(sqlite3_vtab_cursor *cur){
       }
     }
     if( pCur->nPrefix==0 ) break;
-    if( sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0 ){
+    if( pCur->nPrefix<=pCur->szRow
+     && sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0
+    ){
       break;
     }
   }
@@ -327,7 +305,7 @@ static int completionColumn(
   completion_cursor *pCur = (completion_cursor*)cur;
   switch( i ){
     case COMPLETION_COLUMN_CANDIDATE: {
-      sqlite3_result_text(ctx, pCur->zCurrentRow, -1, SQLITE_TRANSIENT);
+      sqlite3_result_text(ctx, pCur->zCurrentRow, pCur->szRow,SQLITE_TRANSIENT);
       break;
     }
     case COMPLETION_COLUMN_PREFIX: {

ext/misc/csv.c

@@ -132,6 +132,7 @@ static int csv_reader_open(
     }
     p->in = fopen(zFilename, "rb");
     if( p->in==0 ){
+      sqlite3_free(p->zIn);
       csv_reader_reset(p);
       csv_errmsg(p, "cannot open '%s' for reading", zFilename);
       return 1;

ext/misc/dbdump.c

@@ -141,45 +141,12 @@ static void appendText(DText *p, char const *zAppend, char quote){
 ** Return '"' if quoting is required.  Return 0 if no quoting is required.
 */
 static char quoteChar(const char *zName){
-  /* All SQLite keywords, in alphabetical order */
-  static const char *azKeywords[] = {
-    "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
-    "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
-    "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
-    "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
-    "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
-    "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
-    "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
-    "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
-    "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
-    "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
-    "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
-    "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
-    "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
-    "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
-    "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
-    "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
-    "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
-    "WITH", "WITHOUT",
-  };
-  int i, lwr, upr, mid, c;
+  int i;
   if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"';
   for(i=0; zName[i]; i++){
     if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"';
   }
-  lwr = 0;
-  upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1;
-  while( lwr<=upr ){
-    mid = (lwr+upr)/2;
-    c = sqlite3_stricmp(azKeywords[mid], zName);
-    if( c==0 ) return '"';
-    if( c<0 ){
-      lwr = mid+1;
-    }else{
-      upr = mid-1;
-    }
-  }
-  return 0;
+  return sqlite3_keyword_check(zName, i) ? '"' : 0;
 }
 
 

ext/misc/templatevtab.c

@@ -0,0 +1,267 @@
+/*
+** 2018-04-19
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file implements a template virtual-table.
+** Developers can make a copy of this file as a baseline for writing
+** new virtual tables and/or table-valued functions.
+**
+** Steps for writing a new virtual table implementation:
+**
+**     (1)  Make a copy of this file.  Perhaps call it "mynewvtab.c"
+**
+**     (2)  Replace this header comment with something appropriate for
+**          the new virtual table
+**
+**     (3)  Change every occurrence of "templatevtab" to some other string
+**          appropriate for the new virtual table.  Ideally, the new string
+**          should be the basename of the source file: "mynewvtab".  Also
+**          globally change "TEMPLATEVTAB" to "MYNEWVTAB".
+**
+**     (4)  Run a test compilation to make sure the unmodified virtual
+**          table works.
+**
+**     (5)  Begin making incremental changes, testing as you go, to evolve
+**          the new virtual table to do what you want it to do.
+**
+** This template is minimal, in the sense that it uses only the required
+** methods on the sqlite3_module object.  As a result, templatevtab is
+** a read-only and eponymous-only table.  Those limitation can be removed
+** by adding new methods.
+**
+** This template implements an eponymous-only virtual table with a rowid and
+** two columns named "a" and "b".  The table as 10 rows with fixed integer
+** values. Usage example:
+**
+**     SELECT rowid, a, b FROM templatevtab;
+*/
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
+SQLITE_EXTENSION_INIT1
+#include <string.h>
+#include <assert.h>
+
+/* templatevtab_vtab is a subclass of sqlite3_vtab which is
+** underlying representation of the virtual table
+*/
+typedef struct templatevtab_vtab templatevtab_vtab;
+struct templatevtab_vtab {
+  sqlite3_vtab base;  /* Base class - must be first */
+  /* Add new fields here, as necessary */
+};
+
+/* templatevtab_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result
+*/
+typedef struct templatevtab_cursor templatevtab_cursor;
+struct templatevtab_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  /* Insert new fields here.  For this templatevtab we only keep track
+  ** of the rowid */
+  sqlite3_int64 iRowid;      /* The rowid */
+};
+
+/*
+** The templatevtabConnect() method is invoked to create a new
+** template virtual table.
+**
+** Think of this routine as the constructor for templatevtab_vtab objects.
+**
+** All this routine needs to do is:
+**
+**    (1) Allocate the templatevtab_vtab object and initialize all fields.
+**
+**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+**        result set of queries against the virtual table will look like.
+*/
+static int templatevtabConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  templatevtab_vtab *pNew;
+  int rc;
+
+  rc = sqlite3_declare_vtab(db,
+           "CREATE TABLE x(a,b)"
+       );
+  /* For convenience, define symbolic names for the index to each column. */
+#define TEMPLATEVTAB_A  0
+#define TEMPLATEVTAB_B  1
+  if( rc==SQLITE_OK ){
+    pNew = sqlite3_malloc( sizeof(*pNew) );
+    *ppVtab = (sqlite3_vtab*)pNew;
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for templatevtab_vtab objects.
+*/
+static int templatevtabDisconnect(sqlite3_vtab *pVtab){
+  templatevtab_vtab *p = (templatevtab_vtab*)pVtab;
+  sqlite3_free(p);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new templatevtab_cursor object.
+*/
+static int templatevtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  templatevtab_cursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destructor for a templatevtab_cursor.
+*/
+static int templatevtabClose(sqlite3_vtab_cursor *cur){
+  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+  sqlite3_free(pCur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a templatevtab_cursor to its next row of output.
+*/
+static int templatevtabNext(sqlite3_vtab_cursor *cur){
+  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+  pCur->iRowid++;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the templatevtab_cursor
+** is currently pointing.
+*/
+static int templatevtabColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+  switch( i ){
+    case TEMPLATEVTAB_A:
+      sqlite3_result_int(ctx, 1000 + pCur->iRowid);
+      break;
+    default:
+      assert( i==TEMPLATEVTAB_B );
+      sqlite3_result_int(ctx, 2000 + pCur->iRowid);
+      break;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.  In this implementation, the
+** rowid is the same as the output value.
+*/
+static int templatevtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+  *pRowid = pCur->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int templatevtabEof(sqlite3_vtab_cursor *cur){
+  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+  return pCur->iRowid>=10;
+}
+
+/*
+** This method is called to "rewind" the templatevtab_cursor object back
+** to the first row of output.  This method is always called at least
+** once prior to any call to templatevtabColumn() or templatevtabRowid() or 
+** templatevtabEof().
+*/
+static int templatevtabFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  templatevtab_cursor *pCur = (templatevtab_cursor *)pVtabCursor;
+  pCur->iRowid = 1;
+  return SQLITE_OK;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+*/
+static int templatevtabBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  pIdxInfo->estimatedCost = (double)10;
+  pIdxInfo->estimatedRows = 10;
+  return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the 
+** virtual table.
+*/
+static sqlite3_module templatevtabModule = {
+  /* iVersion    */ 0,
+  /* xCreate     */ 0,
+  /* xConnect    */ templatevtabConnect,
+  /* xBestIndex  */ templatevtabBestIndex,
+  /* xDisconnect */ templatevtabDisconnect,
+  /* xDestroy    */ 0,
+  /* xOpen       */ templatevtabOpen,
+  /* xClose      */ templatevtabClose,
+  /* xFilter     */ templatevtabFilter,
+  /* xNext       */ templatevtabNext,
+  /* xEof        */ templatevtabEof,
+  /* xColumn     */ templatevtabColumn,
+  /* xRowid      */ templatevtabRowid,
+  /* xUpdate     */ 0,
+  /* xBegin      */ 0,
+  /* xSync       */ 0,
+  /* xCommit     */ 0,
+  /* xRollback   */ 0,
+  /* xFindMethod */ 0,
+  /* xRename     */ 0,
+  /* xSavepoint  */ 0,
+  /* xRelease    */ 0,
+  /* xRollbackTo */ 0
+};
+
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_templatevtab_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+  rc = sqlite3_create_module(db, "templatevtab", &templatevtabModule, 0);
+  return rc;
+}

ext/rbu/rbu.c

@@ -29,7 +29,9 @@ void usage(const char *zArgv0){
 "Where options are:\n"
 "\n"
 "    -step NSTEP\n"
+"    -statstep NSTATSTEP\n"
 "    -vacuum\n"
+"    -presql SQL\n"
 "\n"
 "  If the -vacuum switch is not present, argument RBU-DB must be an RBU\n"
 "  database containing an update suitable for target database TARGET-DB.\n"
@@ -79,7 +81,9 @@ int main(int argc, char **argv){
   char *zErrmsg;                  /* Error message, if any */
   sqlite3rbu *pRbu;               /* RBU handle */
   int nStep = 0;                  /* Maximum number of step() calls */
+  int nStatStep = 0;              /* Report stats after this many step calls */
   int bVacuum = 0;
+  const char *zPreSql = 0;
   int rc;
   sqlite3_int64 nProgress = 0;
   int nArgc = argc-2;
@@ -90,9 +94,18 @@ int main(int argc, char **argv){
     int nArg = strlen(zArg);
     if( nArg>1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){
       bVacuum = 1;
+    }else if( nArg>1 && nArg<=7 
+           && 0==memcmp(zArg, "-presql", nArg) && i<nArg-1 ){
+      i++;
+      zPreSql = argv[i];
     }else if( nArg>1 && nArg<=5 && 0==memcmp(zArg, "-step", nArg) && i<nArg-1 ){
       i++;
       nStep = atoi(argv[i]);
+    }else if( nArg>1 && nArg<=9 
+           && 0==memcmp(zArg, "-statstep", nArg) && i<nArg-1 
+    ){
+      i++;
+      nStatStep = atoi(argv[i]);
     }else{
       usage(argv[0]);
     }
@@ -112,13 +125,40 @@ int main(int argc, char **argv){
   }
   report_rbu_vfs(pRbu);
 
+  if( zPreSql && pRbu ){
+    sqlite3 *db = sqlite3rbu_db(pRbu, 0);
+    rc = sqlite3_exec(db, zPreSql, 0, 0, 0);
+    if( rc==SQLITE_OK ){
+      sqlite3 *db = sqlite3rbu_db(pRbu, 1);
+      rc = sqlite3_exec(db, zPreSql, 0, 0, 0);
+    }
+  }
+
   /* If nStep is less than or equal to zero, call
   ** sqlite3rbu_step() until either the RBU has been completely applied
   ** or an error occurs. Or, if nStep is greater than zero, call
   ** sqlite3rbu_step() a maximum of nStep times.  */
-  for(i=0; (nStep<=0 || i<nStep) && sqlite3rbu_step(pRbu)==SQLITE_OK; i++);
-  nProgress = sqlite3rbu_progress(pRbu);
-  rc = sqlite3rbu_close(pRbu, &zErrmsg);
+  if( rc==SQLITE_OK ){
+    for(i=0; (nStep<=0 || i<nStep) && sqlite3rbu_step(pRbu)==SQLITE_OK; i++){
+      if( nStatStep>0 && (i % nStatStep)==0 ){
+        sqlite3_int64 nUsed;
+        sqlite3_int64 nHighwater;
+        sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &nUsed, &nHighwater, 0);
+        fprintf(stdout, "memory used=%lld highwater=%lld", nUsed, nHighwater);
+        if( bVacuum==0 ){
+          int one;
+          int two;
+          sqlite3rbu_bp_progress(pRbu, &one, &two);
+          fprintf(stdout, "  progress=%d/%d\n", one, two);
+        }else{
+          fprintf(stdout, "\n");
+        }
+        fflush(stdout);
+      }
+    }
+    nProgress = sqlite3rbu_progress(pRbu);
+    rc = sqlite3rbu_close(pRbu, &zErrmsg);
+  }
 
   /* Let the user know what happened. */
   switch( rc ){

main.mk

@@ -72,7 +72,7 @@ LIBOBJ+= vdbe.o parse.o \
          random.o resolve.o rowset.o rtree.o \
          select.o sqlite3rbu.o status.o stmt.o \
          table.o threads.o tokenize.o treeview.o trigger.o \
-         update.o userauth.o util.o vacuum.o \
+         update.o upsert.o userauth.o util.o vacuum.o \
          vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbesort.o \
 	 vdbetrace.o wal.o walker.o where.o wherecode.o whereexpr.o \
          utf.o vtab.o
@@ -162,6 +162,7 @@ SRC = \
   $(TOP)/src/trigger.c \
   $(TOP)/src/utf.c \
   $(TOP)/src/update.c \
+  $(TOP)/src/upsert.c \
   $(TOP)/src/util.c \
   $(TOP)/src/vacuum.c \
   $(TOP)/src/vdbe.c \
@@ -706,6 +707,7 @@ SHELL_SRC = \
 	$(TOP)/ext/misc/shathree.c \
 	$(TOP)/ext/misc/fileio.c \
 	$(TOP)/ext/misc/completion.c \
+        $(TOP)/ext/misc/bindvtab.c \
 	$(TOP)/ext/misc/sqlar.c \
 	$(TOP)/ext/expert/sqlite3expert.c \
 	$(TOP)/ext/expert/sqlite3expert.h \

manifest

@@ -1,12 +1,12 @@
-C Avoid\sa\sNULL-pointer\sderef\sfollowing\sOOM.
-D 2018-04-18T19:08:44.210
+C Better\scomments\son\sthe\sbindvtab.c\simplementation.\s\sAll\sthe\stwo-argument\nversion\sof\sthe\s.set\scommand.\s\sAll\sbindings\sfrom\s.set\sand\s-D\sare\sstill\nstring.
+D 2018-04-27T20:49:13.739
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
-F Makefile.in 7016fc56c6b9bfe5daac4f34be8be38d8c0b5fab79ccbfb764d3b23bf1c6fff3
+F Makefile.in da02d4d3544992af522b4ece652debef3255a5d45377ebb1b5ab8a9a5cd16859
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 43dd6ae2e5a2bb8df7bfc9ed85935129caffeafb8c1803d24c5d038b1e74c8ca
+F Makefile.msc a885344e4f1277b2dce6e9cfb69bc6eceb9494203fbc08b4b7a106268a1a7808
 F README.md 7764d56778d567913ef11c82da9ab94aefa0826f7c243351e4e2d7adaef6f373
-F VERSION 7169eb6959db9ad1b7004ae3b754ef6e703eb7d8dde3b07d2e63103413eb25fb
+F VERSION b7c9d1d11cb70ef8e90cfcf3c944aa58a9f801cc2ad487eebb0a110c16dfc2df
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
@@ -32,10 +32,10 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63
 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977
 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc
 F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55
-F configure 41d0e05b0d289c1c981aafe5c4070713c8e70b5a7d3472360764a3fce08a82a8 x
+F configure 80e2dcad8ab88aacc58b55eb0e395f79184b45fcfaa3f36fc20d2e71cfa0a7e4 x
 F configure.ac d4529ebb26ae046269334f1dac65f2b1d6927c2efe22b2ec24dce24dfe4f83dd
 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
-F doc/lemon.html 278113807f49d12d04179a93fab92b5b917a08771152ca7949d34e928efa3941
+F doc/lemon.html ac63db056bce24b7368e29319cd1a7eb5f1798cc85922d96a80b6c3a4ff9f51b
 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710
 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a
 F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
@@ -44,7 +44,7 @@ F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4
-F ext/expert/expert1.test fd21496d8e52c817a7741f467f42b0502c0ac7e07dcdd1d6e15a3e8154ed4e41
+F ext/expert/expert1.test 00fb67256755a5194dab3d462f8ed3ad4640b4731fe465e5e28cea42c59ce05e
 F ext/expert/sqlite3expert.c 1dfa561e64dc0f89d56b96e6afda87468c34b43604c2df50c47e3f4362778fb2
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c d56c194b769bdc90cf829a14c9ecbc1edca9c850b837a4d0b13be14095c32a72
@@ -269,13 +269,14 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
 F ext/misc/appendvfs.c 3777f22ec1057dc4e5fd89f2fbddcc7a29fbeef1ad038c736c54411bb1967af7
+F ext/misc/bindvtab.c 70ca40ff9bec3cc6da53e09bf13e0af858dc210e68e41b7915d4ff71c363819c
 F ext/misc/btreeinfo.c 78c8c57d325185ccc04b7679e5b020e34a4d9c87453e6b7ac943d0a26cee3256
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
-F ext/misc/completion.c 0d0bd16378415b982e7119baddef52a0d2cc25860c238a9d2832b0cc6a84a16d
+F ext/misc/completion.c e75b8886a2531f9a7ec02dab5f179bb37e6bd46b5da7665a6cbf2dfbe2daa483
 F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189
-F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
-F ext/misc/dbdump.c 22018e00eb50e9ebf9067c92d4e7162dc5006a3efc4e0c19bc3829825a1043b0
+F ext/misc/csv.c d1e324fac3f87f818d684a3d752d1ef76dbcd4fc0db6746ac4034c19d0bcda21
+F ext/misc/dbdump.c 69ef1be5b210538f77dfcc6fcb55b4b5f5e98b1e0bcfd67d818711e10761db4d
 F ext/misc/eval.c 6ea9b22a5fa0dd973b67ca4e53555be177bc0b7b263aadf1024429457c82c0e3
 F ext/misc/fileio.c 48c7751c78fc4cdd29d8c862fd2f3f98bbfefa2a3cf1ca1496df4bf02eb8cded
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
@@ -297,6 +298,7 @@ F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c 54d650f44f3a69a851814791bd4d304575cdbbf78d96d4f0801b44a8f31a58c5
 F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
+F ext/misc/templatevtab.c 8251b31011dd00fc38e739c78c234c930be42b3b274bbe0493b79cd40db02a9e
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
 F ext/misc/unionvtab.c 0b3173f69b8899da640a13a345dc5ef1400199405f738abe6145b2454195b8ff
 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95
@@ -306,7 +308,7 @@ F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
 F ext/misc/zipfile.c c4de8f0ad446ce4a49aae11ff7b771cd7af60d7136c0bcfb53da1475b9075e79
 F ext/misc/zorder.c b0ff58fa643afa1d846786d51ea8d5c4b6b35aa0254ab5a82617db92f3adda64
-F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
+F ext/rbu/rbu.c 4ae41f01d78dfae2204a63bcf39d5eb3548d039b094c29eec9a8d6683a66a5d3
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
 F ext/rbu/rbu11.test 9bc68c2d3dbeb1720153626e3bd0466dcc017702
@@ -413,7 +415,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 63668484c95454af7fc04a384da27ac556f27368d6d0c345e405e1677c66768f
+F main.mk fb46a40f6573a36b1a97901eefafb287aa31e494e8737e772cc0722580025607
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -427,23 +429,23 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F src/alter.c cf7a8af45cb0ace672f47a1b29ab24092a9e8cd8d945a9974e3b5d925f548594
 F src/analyze.c 71fbbeb7b25417592f54d869fe90c28b48e4cecb9926ef9b06d90fb0aec48941
-F src/attach.c bbdf97bb366d94d2bafff8ef611b3bee7b5f54d695531790d896a7a17e126317
+F src/attach.c 4a3138bd771d5426ae4344d8d5e900440af29fabc5ec2f39f69a45010dfbccd7
 F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
-F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
+F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6
 F src/btree.c 9eb9531c65346bbfccf5325384b7db1849daf4db6601dcfe21ba5c5b20623b64
 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84
 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96
-F src/build.c 91d548027a54044851299e719d60d6a2b6eaf3e3274678098cb73a6ab8c0fa19
+F src/build.c c4227d058d52c24ffce2d33cd3a11234d8e8603901243cdb5165eddf64ee2177
 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 849d4cebe008cfc6e4799b034a172b4eaf8856b100739632a852732ba66eee48
 F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c edabb82611143727511a45ca0859b8cd037851ebe756ae3db289859dd18b6f91
-F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
-F src/expr.c 2448a255ce627c4e772bd68cf5529877c2bdfb6b580803d5fadc8528bdf7c1ef
+F src/delete.c b0f90749e22d5e41a12dbf940f4811138cf97da54b46b737089b93eb64a2896f
+F src/expr.c 4a39bc2a9a3d825571519a1910ecfe3d4b6d596e775c9ed7cef7497ea06140ac
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51
@@ -452,10 +454,10 @@ F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
-F src/insert.c 752740e4619416d4262f6e9e51cdb6af5965eb0c8e943832a5af77d41e2839c7
+F src/insert.c 33a2c72b6182e8ddf697d604cc087c77ff5fc512a32b8b624641d41b390e249e
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c f6e4e416a736369f9e80eba609f0acda97148a8b0453784d670c78d3eed2f302
-F src/main.c 10e3897f5d78cef6bcbd1eedc8ccc3fe9e9783d07e052d9d70e57364ded19274
+F src/main.c f579a7a1e6a602cf752ee910ef541481124bac9dfa7db720e698efdf5521bfd3
 F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -480,23 +482,23 @@ F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 1bb6a57fa0465296a4d6109a1a64610a0e7adde1f3acf3ef539a9d972908ce8f
 F src/pager.h c571b064df842ec8f2e90855dead9acf4cbe0d1b2c05afe0ef0d0145f7fd0388
-F src/parse.y e3c4116efb7d693df412bd42a96c88c6463704d31a29342c2fa671f6e91ecb26
+F src/parse.y 07784439d25f0bc64a656eece4caecc549b147d213f513cdbeb8430345ec2911
 F src/pcache.c 135ef0bc6fb2e3b7178d49ab5c9176254c8a691832c1bceb1156b2fbdd0869bd
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c bea56df3ae0637768c0da4fbbb8f2492f780980d95000034a105ff291bf7ca69
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
-F src/prepare.c b086fea6a1952db88beca31fdd621201ee5e4ce3f02905248cc3035a8174aa89
+F src/prepare.c d16cb8a553c09bc0c46c33f8e0add77c8f755cfb4681ca5101aaff3a04df9625
 F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
-F src/resolve.c 66c73fcb7719b8ff0e841b58338f13604ff3e2b50a723f9b8f383595735262f6
+F src/resolve.c 6415381a0e9d22c0e7cba33ca4a53f81474190862f5d4838190f5eb5b0b47bc9
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 9257a7f26e6c9da27deae819098d1005f4e1c37e19b8f029292bd2b1aab35721
-F src/shell.c.in 6c9e2c1136f3697eb75f5dce010e7af005f62b4e1fda6d1066c3a473eb922889
-F src/sqlite.h.in aa9bd3ae4a077c7002059cb418271abe52214b0227b2a734bc44736b24cbcc40
+F src/select.c daf07d8defce3311f9e69f1280a874d78bc1d16c305f6aa689640f7afa02842f
+F src/shell.c.in d751c9e8aa9c73a700d4033d2bfcfccad90c476b1fcb0d406e3f6b140e0f02c0
+F src/sqlite.h.in 8e70752a57597c08f64f3d49fc1fc46926b862d2e23b038b0d23b9cc748d88ea
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
-F src/sqliteInt.h ad14bfeab6c1ada3aa1181c6cc14a3e6a8f24b35ad96a2daa3d11100a5267236
+F src/sqliteInt.h c1deb023eb117b4437a69d8c5a753deb36581f1933015cd262ad32b45c4d8431
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -553,20 +555,21 @@ F src/test_windirent.c a895e2c068a06644eef91a7f0a32182445a893b9a0f33d0cdb4283dca
 F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a90484215
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
-F src/tokenize.c 5b0c661a85f783d35b9883830736eeb63be4aefc4f6b7d9cd081d48782c041e2
-F src/treeview.c 14d5d1254702ec96876aa52642cb31548612384134970409fae333b25b39d6bb
-F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1
-F src/update.c 97d4c9514229f540f8c441e124d5af7f93c5b030c9574539d01e99462e273998
+F src/tokenize.c bbde32eac9eb1280f5292bcdfef66f5a57e43176cbf9347e0efab9f75e133f97
+F src/treeview.c 6cea286ca9af8b126dae9714799265353387244eba0d451c3cc2cd60946cc4c7
+F src/trigger.c 4ace6d1d5ba9a89822deb287317f33c810440526eafe185c2d8a48c31df1e995
+F src/update.c 5be2f501ddc704fc04183bdb28b25eab930bb8553d973429a089ec94fa85cf2b
+F src/upsert.c ae4a4823b45c4daf87e8aea8c0f582a8844763271f5ed54ee5956c4c612734f4
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157
 F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5
 F src/vdbe.c 066a4e1de2ed83e253adfd2e97a684cf562eaa41d31ee7f3d3e4c8aea4485a55
-F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
+F src/vdbe.h 574ce9a0d57b026fc93ac379a339b8d391977f335ab4176a7e21ba902e9184bd
 F src/vdbeInt.h 95f7adfdc5c8f1353321f55a6c5ec00a90877e3b85af5159e393afb41ff54110
 F src/vdbeapi.c 29d2baf9c1233131ec467d7bed1b7c8a03c27579048d768c4b04acf427838858
-F src/vdbeaux.c 2756ac68ac259c416554100598fc291870063288cd7e1af22847f57b3e130e56
+F src/vdbeaux.c 944bae5207bbce456c466d01dcf2aac3ad49c957325d35c0ba7de882d3e5c25c
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
-F src/vdbemem.c 414e28d3a7e2a8bee2bb247de115dcbc68e3cbac284d5862d077002f7a93bce1
+F src/vdbemem.c 0cbe9b9560e42b72983cf9e1bceba48f297e51142bfb6b57f3747cf60106b92d
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
@@ -574,10 +577,10 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c aa9cffc7a2bad6b826a86c8562dd4978398720ed41cb8ee7aa9d054eb8b456a0
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c d6e5f2056e9a60251e79780fc598a5943e88a3c0fa0019d54922e59f99019287
+F src/where.c 7a1c5555c00bcf49c677472ae83bb49bf24c8d8e9a060d475e86dee39be2fb3a
 F src/whereInt.h 2610cb87dd95509995b63decc674c60f2757697a206cfe0c085ee53d9c43cfff
-F src/wherecode.c 982b7450c53fb272f61a1d20c93e960260ea4dfe8e2e9bacc190e2a041a1f1a4
-F src/whereexpr.c 53532be687e12f3cd314f1e204cd4fbdac7ad250e918a182b048121e16e828ae
+F src/wherecode.c 3cb591fe7323e1bfd49059d75fcab75e383dfcf16a51c0e4f9368ac1b93e291b
+F src/whereexpr.c e90b2e76dcabc81edff56633bf281bc01d93b71e0c81482dc06925ce39f5844a
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
 F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d
@@ -622,7 +625,7 @@ F test/auth.test 3d6cd8f3978ba55b1202574e6ecd79c6e00914ca44b9bfd6c1fe6fb873fcac8
 F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1
 F test/auth3.test db21405b95257c24d29273b6b31d0efc59e1d337e3d5804ba2d1fd4897b1ae49
 F test/autoanalyze1.test b9cc3f32a990fa56669b668d237c6d53e983554ae80c0604992e18869a0b2dec
-F test/autoinc.test 6ae8fb69c9f656962464ae4e6667045d0dfc3b46
+F test/autoinc.test 83aad64411583aac9ff0b629159ab4662029ab4e3f47090fce4efd132b304484
 F test/autoindex1.test 788d0894aa3aee1220036d20696e98733fb7ca02265cb1e801700177120c1aeb
 F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df
 F test/autoindex3.test a3be0d1a53a7d2edff208a5e442312957047e972
@@ -641,7 +644,7 @@ F test/backup_malloc.test 0c9abdf74c51e7bedb66d504cd684f28d4bd4027
 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f
 F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f
 F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c
-F test/bestindex1.test 0cf1bd2d7b97d3a3a8c10736125274f64765c4ee
+F test/bestindex1.test a9b831613b46baae213798cd3a71257833d5d3d2692e0e50c7ff0c3fbaeb0200
 F test/bestindex2.test 4a06b8922ab2fd09434870da8d1cdf525aaf7060
 F test/bestindex3.test 578b6a52dab819e63f28e3640e04b32c85aed320
 F test/bestindex4.test 4cb5ff7dbaebadb87d366f51969271778423b455
@@ -735,7 +738,7 @@ F test/crashM.test d95f59046fa749b0d0822edf18a717788c8f318d
 F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2
 F test/createtab.test b5de160630b209c4b8925bdcbbaf48cc90b67fe8
 F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c
-F test/csv01.test 526fc6aefd052badd5a0283f86b4b395c3df76bfe98d96c801f494f5e2c7836c
+F test/csv01.test b99db7e57ee66fb3aa6e5d25af119003bee90bf679c5885711cbf7374f025b73
 F test/ctime.test 78749e6c9a5f0010d67985be80788f841e3cd2da18114e2ed6010399a7d807f3
 F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856
 F test/cursorhint2.test 0078ae1ded4afcf5eb80d06e3a72b6e1c3f1a646aab26eeb583b0a9ec6f0d56e
@@ -746,7 +749,7 @@ F test/dbpage.test dbf50a4d361f9e45a979432c727506065113124478a7d2db12074fa655e65
 F test/dbstatus.test c15fa97f743dac7ce996814c84b56317e138895ee15ce27f15b608aa6924c90a
 F test/dbstatus2.test f5fe0afed3fa45e57cfa70d1147606c20d2ba23feac78e9a172f2fe8ab5b78ef
 F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d
-F test/delete.test acc38fca8ee4851467705b1c2cfea64cd26667e5
+F test/delete.test 31832b0c45ecb51a54348c68db173be462985901e6ed7f403d6d7a8f70ab4ef0
 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa
 F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab
 F test/delete4.test 21d2113217eeaacac2d99defe14fe6611615ae86
@@ -936,7 +939,7 @@ F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1
 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1
 F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c31
 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b
-F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26
+F test/fuzz_malloc.test cf82eab3d93cd3e38b2d53c13c6739ff4cd5b1c31c2c45edd0bcee78e7060c2b
 F test/fuzzcheck.c 5eb86c6ac96833ee622f45bf47e8045999c1b4b10d05e4eb809894a4b39f2f84
 F test/fuzzdata1.db 7ee3227bad0e7ccdeb08a9e6822916777073c664
 F test/fuzzdata2.db f03a420d3b822cc82e4f894ca957618fbe9c4973
@@ -1004,12 +1007,12 @@ F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd
 F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c
 F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4
 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b
-F test/istrue.test d6e659764da5ccc03adcdba18fe77d7917ba5e4abd04ef14bd4e4cf43e024b5b
+F test/istrue.test 62372ad3ddcc5d0eb8ff9097dcb0aad8961bf1b9cb45ba634f6e284695126f9a
 F test/join.test 2ad9d7fe10e0cc06bc7803c22e5533be11cdadbc592f5f95d789a873b57a5a66
 F test/join2.test f5ea0fd3b0a441c8e439706339dcd17cec63a896a755c04a30bfd442ecce1190
 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0
 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916
-F test/join5.test c6bd62effc37a152bea735f9ef241b19bb967bd4593dc99b20e2fc55ae707e38
+F test/join5.test 000a18e9ccc9fc716e550eb2337729d3aee99031d0ddd9682f6b7d73aa99b924
 F test/join6.test cfe6503791ceb0cbb509966740286ec423cbf10b
 F test/journal1.test c7b768041b7f494471531e17abc2f4f5ebf9e5096984f43ed17c4eb80ba34497
 F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4
@@ -1475,7 +1478,7 @@ F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6
 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76
 F test/trans3.test 91a100e5412b488e22a655fe423a14c26403ab94
 F test/transitive1.test 293300f46916569f08875cdb2fe2134be2c27677
-F test/trigger1.test ea9624cc1dae05645469df6119fa815f9e6f1e8c
+F test/trigger1.test 17e4b43e656c4b354df2357634a6ba887990f510c43629f4feca30e3338d2a61
 F test/trigger2.test 5cd7d69a7ba1143ee045e4ae2963ff32ae4c87a6
 F test/trigger3.test aa640bb2bbb03edd5ff69c055117ea088f121945
 F test/trigger4.test 74700b76ebf3947b2f7a92405141eb2cf2a5d359
@@ -1505,8 +1508,13 @@ F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264
 F test/unique2.test 3674e9f2a3f1fbbfd4772ac74b7a97090d0f77d2
 F test/unixexcl.test d936ba2b06794018e136418addd59a2354eeae97
 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8
-F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32
+F test/update.test 1148de8d913e9817717990603aadeca07aab9ddbb10a30f167cbfd8d3a3ccb60
 F test/update2.test 5e67667e1c54017d964e626db765cf8bedcf87483c184f4c575bdb8c1dd2313e
+F test/upsert1.test d587db593f131e112a98a05685c418e0eacc28df5905403e4ca04cd74c39a3fc
+F test/upsert2.test 9c3cdbb1a890227f6504ce4b0e3de68f4cdfa16bb21d8641208a9239896c5a09
+F test/upsert3.test 88d7d590a1948a9cb6eac1b54b0642f67a9f35a1fc0f19b200e97d5d39e3179c
+F test/upsert4.test 25d2a1da92f149331ae0c51ca6e3eee78189577585eab92de149900d62994fa5
+F test/upsertfault.test f21ca47740841fdb4d61acfa7b17646d773e67724fe8c185b71c018db8a94b35
 F test/uri.test 3481026f00ade6dfe8adb7acb6e1e47b04369568
 F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7
 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9
@@ -1573,14 +1581,14 @@ F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cf
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
 F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2af51747
-F test/where.test f0c325563acde44f2c4ea6ba348e9e29f7121757
+F test/where.test f19ea3fa31c425b04af42c8b192a5b595ee84498df8d27dcd79ec043b25fbbfb
 F test/where2.test 478d2170637b9211f593120648858593bf2445a1
 F test/where3.test 54cdeb02157acc979de41530b804ae7b09552bf1
 F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8
 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2
 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b
 F test/where7.test f520bcec2c3d12dc4615623b06b2aec7c2d67e94
-F test/where8.test 98eedca0d375fb400b8377269c4b4686582dfb45
+F test/where8.test 461ca40265ed996a6305da99bb024b0e41602bb586acf544c08f95922358e49f
 F test/where9.test 729c3ba9b47e8f9f1aab96bae7dad2a524f1d1a2
 F test/whereA.test 6c6a420ca7d313242f9b1bd471dc80e4d0f8323700ba9c78df0bb843d4daa3b4
 F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5
@@ -1613,12 +1621,12 @@ F test/without_rowid3.test 2724c787a51a5dce09d078453a758117b4b728f1
 F test/without_rowid4.test 4e08bcbaee0399f35d58b5581881e7a6243d458a
 F test/without_rowid5.test 89b1c587bd92a0590e440da33e7666bf4891572a
 F test/without_rowid6.test 1f99644e6508447fb050f73697350c7ceca3392e
-F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf8ac
+F test/wordcount.c d721a4b6fae93e6e33449700bce1686bc23257c27425bc3ef1599dc912adec66
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
 F test/zipfile.test a61f6ba6dbaaf4983849df84a31df140c7ddd1362e2fa9ecd3cdf5cd123b7f18
-F test/zipfile2.test fc2f08d5ec19c18c83289fbed32e378dc5116519972166e57a244da7bf2e5805
+F test/zipfile2.test 8a18965530258140ce301b2c2f4245bc505ffead2e17a97fa34c7ace379ea5e0
 F test/zipfilefault.test 44d4d7a7f7cca7521d569d7f71026b241d65a6b1757aa409c1a168827edbbc2c
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
@@ -1637,8 +1645,8 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
-F tool/lemon.c c1a87d15983f96851b253707985dba8783fbe41ba21ba1b76720e06c8a262206
-F tool/lempar.c 468a155e8729cfbccfe1d85bf60d064f1dab76167a51149ec5c7928a2de63953
+F tool/lemon.c 33892e2a243865f73e6c6e7cecce3c6eb4bb95db4a3d9d86d146c8064feb92fd
+F tool/lempar.c bf7db78e7213f1d51516710483eab506fd52bf632c7abfb3e2e9b885c90c03e1
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1646,7 +1654,7 @@ F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439
 F tool/mkautoconfamal.sh 422fc365358a2e92876ffc62971a0ff28ed472fc8bcf9de0df921c736fdeca5e
 F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x
 F tool/mkctimec.tcl dd183b73ae1c28249669741c250525f0407e579a70482371668fd5f130d9feb3
-F tool/mkkeywordhash.c 2e852ac0dfdc5af18886dc1ce7e9676d11714ae3df0a282dc7d90b3a0fe2033c
+F tool/mkkeywordhash.c 20f366ad3794e1db42e333a6f35fa41a024f2e3528579c9d58eb13eaa3ab4913
 F tool/mkmsvcmin.tcl cad0c7b54d7dd92bc87d59f36d4cc4f070eb2e625f14159dc2f5c4204e6a13ea
 F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c
 F tool/mkopcodeh.tcl 4ee2a30ccbd900dc4d5cdb61bdab87cd2166cd2affcc78c9cc0b8d22a65b2eee
@@ -1656,7 +1664,7 @@ F tool/mkshellc.tcl 1f45770aea226ac093a9c72f718efbb88a2a2833409ec2e1c4cecae42026
 F tool/mksourceid.c d458f9004c837bee87a6382228ac20d3eae3c49ea3b0a5aace936f8b60748d3b
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb
-F tool/mksqlite3c.tcl a03cee30de81a2e67b93e5c659f24113a003677c557daeb008205c8e6d4345d6
+F tool/mksqlite3c.tcl d532ccbe81234f766bab6e5c0451c99529debcea138caccc1862a9645b2d54f2
 F tool/mksqlite3h.tcl 080873e3856eceb9d289a08a00c4b30f875ea3feadcbece796bd509b1532792c
 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b
 F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5
@@ -1683,7 +1691,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
 F tool/split-sqlite3c.tcl 3efcd4240b738f6bb2b5af0aea7e1e0ef9bc1c61654f645076cec883030b710c
-F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb
+F tool/sqldiff.c 579d7e4e42c30a963781654c87d2868823120b55d59e16ca77b0edbab0218713
 F tool/sqlite3_analyzer.c.in 7eeaae8b0d7577662acaabbb11107af0659d1b41bc1dfdd4d91422de27127968
 F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898
 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848
@@ -1718,7 +1726,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 902a40897f74ac8a3bc72ef84c2161ab308b5601381cc9eea18147bfefa978ce
-R 4bf2a8a4f284f188f9cc406a1fead38f
+P 1f2944d1d64d36b729450685974c260295cf1e32c204b71e27d20ebc4d65f9e8
+R 8b8e4c122ecc932439216f69c14e3e66
 U drh
-Z 857937520ddb8fedd9d522f195cdf4dc
+Z 42c91d81ee6385ae52b3a143c0c973f3

manifest.uuid

@@ -1 +1 @@
-413015c029d850d4ce7e66be1f59b57f291254240a958856378a62f5ac4a5092
+7b2a65a65475b80669af4ff5ac7adb6b310cd8275a36e672a3f5c88a259693c0

src/attach.c

@@ -158,7 +158,7 @@ static void attachFunc(
     sqlite3_free( zPath );
     db->nDb++;
   }
-  db->skipBtreeMutex = 0;
+  db->noSharedCache = 0;
   if( rc==SQLITE_CONSTRAINT ){
     rc = SQLITE_ERROR;
     zErrDyn = sqlite3MPrintf(db, "database is already attached");
@@ -230,6 +230,7 @@ static void attachFunc(
   if( rc==SQLITE_OK ){
     sqlite3BtreeEnterAll(db);
     db->init.iDb = 0;
+    db->mDbFlags &= ~(DBFLAG_SchemaKnownOk);
     rc = sqlite3Init(db, &zErrDyn);
     sqlite3BtreeLeaveAll(db);
     assert( zErrDyn==0 || rc!=SQLITE_OK );
@@ -604,6 +605,18 @@ int sqlite3FixTriggerStep(
     if( sqlite3FixExprList(pFix, pStep->pExprList) ){
       return 1;
     }
+#ifndef SQLITE_OMIT_UPSERT
+    if( pStep->pUpsert ){
+      Upsert *pUp = pStep->pUpsert;
+      if( sqlite3FixExprList(pFix, pUp->pUpsertTarget)
+       || sqlite3FixExpr(pFix, pUp->pUpsertTargetWhere)
+       || sqlite3FixExprList(pFix, pUp->pUpsertSet)
+       || sqlite3FixExpr(pFix, pUp->pUpsertWhere)
+      ){
+        return 1;
+      }
+    }
+#endif
     pStep = pStep->pNext;
   }
   return 0;

src/btmutex.c

@@ -195,10 +195,10 @@ static void SQLITE_NOINLINE btreeEnterAll(sqlite3 *db){
       skipOk = 0;
     }
   }
-  db->skipBtreeMutex = skipOk;
+  db->noSharedCache = skipOk;
 }
 void sqlite3BtreeEnterAll(sqlite3 *db){
-  if( db->skipBtreeMutex==0 ) btreeEnterAll(db);
+  if( db->noSharedCache==0 ) btreeEnterAll(db);
 }
 static void SQLITE_NOINLINE btreeLeaveAll(sqlite3 *db){
   int i;
@@ -210,7 +210,7 @@ static void SQLITE_NOINLINE btreeLeaveAll(sqlite3 *db){
   }
 }
 void sqlite3BtreeLeaveAll(sqlite3 *db){
-  if( db->skipBtreeMutex==0 ) btreeLeaveAll(db);
+  if( db->noSharedCache==0 ) btreeLeaveAll(db);
 }
 
 #ifndef NDEBUG

src/build.c

@@ -343,24 +343,27 @@ Table *sqlite3LocateTable(
   const char *zDbase     /* Name of the database.  Might be NULL */
 ){
   Table *p;
+  sqlite3 *db = pParse->db;
 
   /* Read the database schema. If an error occurs, leave an error message
   ** and code in pParse and return NULL. */
-  if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){
+  if( (db->mDbFlags & DBFLAG_SchemaKnownOk)==0 
+   && SQLITE_OK!=sqlite3ReadSchema(pParse)
+  ){
     return 0;
   }
 
-  p = sqlite3FindTable(pParse->db, zName, zDbase);
+  p = sqlite3FindTable(db, zName, zDbase);
   if( p==0 ){
     const char *zMsg = flags & LOCATE_VIEW ? "no such view" : "no such table";
 #ifndef SQLITE_OMIT_VIRTUALTABLE
-    if( sqlite3FindDbName(pParse->db, zDbase)<1 ){
+    if( sqlite3FindDbName(db, zDbase)<1 ){
       /* If zName is the not the name of a table in the schema created using
       ** CREATE, then check to see if it is the name of an virtual table that
       ** can be an eponymous virtual table. */
-      Module *pMod = (Module*)sqlite3HashFind(&pParse->db->aModule, zName);
+      Module *pMod = (Module*)sqlite3HashFind(&db->aModule, zName);
       if( pMod==0 && sqlite3_strnicmp(zName, "pragma_", 7)==0 ){
-        pMod = sqlite3PragmaVtabRegister(pParse->db, zName);
+        pMod = sqlite3PragmaVtabRegister(db, zName);
       }
       if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){
         return pMod->pEpoTab;
@@ -525,6 +528,7 @@ void sqlite3ResetOneSchema(sqlite3 *db, int iDb){
     assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
     DbSetProperty(db, iDb, DB_ResetWanted);
     DbSetProperty(db, 1, DB_ResetWanted);
+    db->mDbFlags &= ~DBFLAG_SchemaKnownOk;
   }
 
   if( db->nSchemaLock==0 ){
@@ -550,7 +554,7 @@ void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){
       sqlite3SchemaClear(pDb->pSchema);
     }
   }
-  db->mDbFlags &= ~DBFLAG_SchemaChange;
+  db->mDbFlags &= ~(DBFLAG_SchemaChange|DBFLAG_SchemaKnownOk);
   sqlite3VtabUnlockList(db);
   sqlite3BtreeLeaveAll(db);
   sqlite3CollapseDatabaseArray(db);

src/delete.c

@@ -238,7 +238,7 @@ void sqlite3DeleteFrom(
   AuthContext sContext;  /* Authorization context */
   NameContext sNC;       /* Name context to resolve expressions in */
   int iDb;               /* Database number */
-  int memCnt = -1;       /* Memory cell used for change counting */
+  int memCnt = 0;        /* Memory cell used for change counting */
   int rcauth;            /* Value returned by authorization callback */
   int eOnePass;          /* ONEPASS_OFF or _SINGLE or _MULTI */
   int aiCurOnePass[2];   /* The write cursors opened by WHERE_ONEPASS */
@@ -343,7 +343,7 @@ void sqlite3DeleteFrom(
     goto delete_from_cleanup;
   }
   if( pParse->nested==0 ) sqlite3VdbeCountChanges(v);
-  sqlite3BeginWriteOperation(pParse, 1, iDb);
+  sqlite3BeginWriteOperation(pParse, bComplex, iDb);
 
   /* If we are trying to delete from a view, realize that view into
   ** an ephemeral table.
@@ -371,7 +371,10 @@ void sqlite3DeleteFrom(
   /* Initialize the counter of the number of rows deleted, if
   ** we are counting rows.
   */
-  if( db->flags & SQLITE_CountRows ){
+  if( (db->flags & SQLITE_CountRows)!=0
+   && !pParse->nested
+   && !pParse->pTriggerTab
+  ){
     memCnt = ++pParse->nMem;
     sqlite3VdbeAddOp2(v, OP_Integer, 0, memCnt);
   }
@@ -399,7 +402,7 @@ void sqlite3DeleteFrom(
     assert( !isView );
     sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName);
     if( HasRowid(pTab) ){
-      sqlite3VdbeAddOp4(v, OP_Clear, pTab->tnum, iDb, memCnt,
+      sqlite3VdbeAddOp4(v, OP_Clear, pTab->tnum, iDb, memCnt ? memCnt : -1,
                         pTab->zName, P4_STATIC);
     }
     for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
@@ -444,9 +447,10 @@ void sqlite3DeleteFrom(
     eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
     assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI );
     assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF );
+    if( eOnePass!=ONEPASS_SINGLE ) sqlite3MultiWrite(pParse);
   
     /* Keep track of the number of rows to be deleted */
-    if( db->flags & SQLITE_CountRows ){
+    if( memCnt ){
       sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1);
     }
   
@@ -589,7 +593,7 @@ void sqlite3DeleteFrom(
   ** generating code because of a call to sqlite3NestedParse(), do not
   ** invoke the callback function.
   */
-  if( (db->flags&SQLITE_CountRows) && !pParse->nested && !pParse->pTriggerTab ){
+  if( memCnt ){
     sqlite3VdbeAddOp2(v, OP_ResultRow, memCnt, 1);
     sqlite3VdbeSetNumCols(v, 1);
     sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows deleted", SQLITE_STATIC);

src/expr.c

@@ -1826,6 +1826,8 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){
       }
       /* Fall through */
     case TK_IF_NULL_ROW:
+    case TK_REGISTER:
+      testcase( pExpr->op==TK_REGISTER );
       testcase( pExpr->op==TK_IF_NULL_ROW );
       pWalker->eCode = 0;
       return WRC_Abort;
@@ -1843,8 +1845,8 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){
       }
       /* Fall through */
     default:
-      testcase( pExpr->op==TK_SELECT ); /* sqlite3SelectWalkFail will disallow */
-      testcase( pExpr->op==TK_EXISTS ); /* sqlite3SelectWalkFail will disallow */
+      testcase( pExpr->op==TK_SELECT ); /* sqlite3SelectWalkFail() disallows */
+      testcase( pExpr->op==TK_EXISTS ); /* sqlite3SelectWalkFail() disallows */
       return WRC_Continue;
   }
 }
@@ -2607,17 +2609,6 @@ int sqlite3CodeSubselect(
     jmpIfDynamic = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
   }
 
-#ifndef SQLITE_OMIT_EXPLAIN
-  if( pParse->explain==2 ){
-    char *zMsg = sqlite3MPrintf(pParse->db, "EXECUTE %s%s SUBQUERY %d",
-        jmpIfDynamic>=0?"":"CORRELATED ",
-        pExpr->op==TK_IN?"LIST":"SCALAR",
-        pParse->iNextSelectId
-    );
-    sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC);
-  }
-#endif
-
   switch( pExpr->op ){
     case TK_IN: {
       int addr;                   /* Address of OP_OpenEphemeral instruction */
@@ -2655,6 +2646,17 @@ int sqlite3CodeSubselect(
         Select *pSelect = pExpr->x.pSelect;
         ExprList *pEList = pSelect->pEList;
 
+#ifndef SQLITE_OMIT_EXPLAIN
+        if( pParse->explain==2 ){
+          char *zMsg = sqlite3MPrintf(pParse->db, "EXECUTE %sLIST SUBQUERY %d",
+            jmpIfDynamic>=0?"":"CORRELATED ",
+            pParse->iNextSelectId
+          );
+          sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg,
+                            P4_DYNAMIC);
+        }
+#endif
+
         assert( !isRowid );
         /* If the LHS and RHS of the IN operator do not match, that
         ** error will have been caught long before we reach this point. */
@@ -2696,7 +2698,6 @@ int sqlite3CodeSubselect(
         ExprList *pList = pExpr->x.pList;
         struct ExprList_item *pItem;
         int r1, r2, r3;
-
         affinity = sqlite3ExprAffinity(pLeft);
         if( !affinity ){
           affinity = SQLITE_AFF_BLOB;
@@ -2776,6 +2777,17 @@ int sqlite3CodeSubselect(
       assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT );
       assert( ExprHasProperty(pExpr, EP_xIsSelect) );
 
+#ifndef SQLITE_OMIT_EXPLAIN
+      if( pParse->explain==2 ){
+        char *zMsg = sqlite3MPrintf(pParse->db, "EXECUTE %sSCALAR SUBQUERY %d",
+            jmpIfDynamic>=0?"":"CORRELATED ",
+            pParse->iNextSelectId
+        );
+        sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg,
+                          P4_DYNAMIC);
+      }
+#endif
+
       pSel = pExpr->x.pSelect;
       nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1;
       sqlite3SelectDestInit(&dest, 0, pParse->nMem+1);
@@ -4037,10 +4049,9 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
       assert( p1>=0 && p1<(pTab->nCol*2+2) );
 
       sqlite3VdbeAddOp2(v, OP_Param, p1, target);
-      VdbeComment((v, "%s.%s -> $%d",
+      VdbeComment((v, "r[%d]=%s.%s", target,
         (pExpr->iTable ? "new" : "old"),
-        (pExpr->iColumn<0 ? "rowid" : pExpr->pTab->aCol[pExpr->iColumn].zName),
-        target
+        (pExpr->iColumn<0 ? "rowid" : pExpr->pTab->aCol[pExpr->iColumn].zName)
       ));
 
 #ifndef SQLITE_OMIT_FLOATING_POINT
@@ -4906,8 +4917,10 @@ int sqlite3ExprCompare(Parse *pParse, Expr *pA, Expr *pB, int iTab){
   if( pA->op!=TK_COLUMN && pA->op!=TK_AGG_COLUMN && pA->u.zToken ){
     if( pA->op==TK_FUNCTION ){
       if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2;
+    }else if( pA->op==TK_COLLATE ){
+      if( sqlite3_stricmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2;
     }else if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){
-      return pA->op==TK_COLLATE ? 1 : 2;
+      return 2;
     }
   }
   if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2;
@@ -4916,7 +4929,8 @@ int sqlite3ExprCompare(Parse *pParse, Expr *pA, Expr *pB, int iTab){
     if( sqlite3ExprCompare(pParse, pA->pLeft, pB->pLeft, iTab) ) return 2;
     if( sqlite3ExprCompare(pParse, pA->pRight, pB->pRight, iTab) ) return 2;
     if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList, iTab) ) return 2;
-    if( ALWAYS((combinedFlags & EP_Reduced)==0) && pA->op!=TK_STRING ){
+    assert( (combinedFlags & EP_Reduced)==0 );
+    if( pA->op!=TK_STRING && pA->op!=TK_TRUEFALSE ){
       if( pA->iColumn!=pB->iColumn ) return 2;
       if( pA->iTable!=pB->iTable 
        && (pA->iTable!=iTab || NEVER(pB->iTable>=0)) ) return 2;
@@ -5262,8 +5276,9 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
   NameContext *pNC = pWalker->u.pNC;
   Parse *pParse = pNC->pParse;
   SrcList *pSrcList = pNC->pSrcList;
-  AggInfo *pAggInfo = pNC->pAggInfo;
+  AggInfo *pAggInfo = pNC->uNC.pAggInfo;
 
+  assert( pNC->ncFlags & NC_UAggInfo );
   switch( pExpr->op ){
     case TK_AGG_COLUMN:
     case TK_COLUMN: {

src/insert.c

@@ -488,7 +488,8 @@ void sqlite3Insert(
   SrcList *pTabList,    /* Name of table into which we are inserting */
   Select *pSelect,      /* A SELECT statement to use as the data source */
   IdList *pColumn,      /* Column names corresponding to IDLIST. */
-  int onError           /* How to handle constraint errors */
+  int onError,          /* How to handle constraint errors */
+  Upsert *pUpsert       /* ON CONFLICT clauses for upsert, or NULL */
 ){
   sqlite3 *db;          /* The main database structure */
   Table *pTab;          /* The table to insert into.  aka TABLE */
@@ -783,7 +784,10 @@ void sqlite3Insert(
     
   /* Initialize the count of rows to be inserted
   */
-  if( db->flags & SQLITE_CountRows ){
+  if( (db->flags & SQLITE_CountRows)!=0
+   && !pParse->nested
+   && !pParse->pTriggerTab
+  ){
     regRowCount = ++pParse->nMem;
     sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount);
   }
@@ -803,6 +807,19 @@ void sqlite3Insert(
       pParse->nMem += pIdx->nColumn;
     }
   }
+#ifndef SQLITE_OMIT_UPSERT
+  if( pUpsert ){
+    pTabList->a[0].iCursor = iDataCur;
+    pUpsert->pUpsertSrc = pTabList;
+    pUpsert->regData = regData;
+    pUpsert->iDataCur = iDataCur;
+    pUpsert->iIdxCur = iIdxCur;
+    if( pUpsert->pUpsertTarget ){
+      sqlite3UpsertAnalyzeTarget(pParse, pTabList, pUpsert);
+    }
+  }
+#endif
+
 
   /* This is the top of the main insertion loop */
   if( useTempTable ){
@@ -1005,7 +1022,7 @@ void sqlite3Insert(
       int isReplace;    /* Set to true if constraints may cause a replace */
       int bUseSeek;     /* True to use OPFLAG_SEEKRESULT */
       sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur,
-          regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0
+          regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0, pUpsert
       );
       sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0);
 
@@ -1028,7 +1045,7 @@ void sqlite3Insert(
 
   /* Update the count of rows that are inserted
   */
-  if( (db->flags & SQLITE_CountRows)!=0 ){
+  if( regRowCount ){
     sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1);
   }
 
@@ -1065,7 +1082,7 @@ insert_end:
   ** generating code because of a call to sqlite3NestedParse(), do not
   ** invoke the callback function.
   */
-  if( (db->flags&SQLITE_CountRows) && !pParse->nested && !pParse->pTriggerTab ){
+  if( regRowCount ){
     sqlite3VdbeAddOp2(v, OP_ResultRow, regRowCount, 1);
     sqlite3VdbeSetNumCols(v, 1);
     sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows inserted", SQLITE_STATIC);
@@ -1074,6 +1091,7 @@ insert_end:
 insert_cleanup:
   sqlite3SrcListDelete(db, pTabList);
   sqlite3ExprListDelete(db, pList);
+  sqlite3UpsertDelete(db, pUpsert);
   sqlite3SelectDelete(db, pSelect);
   sqlite3IdListDelete(db, pColumn);
   sqlite3DbFree(db, aRegIdx);
@@ -1145,6 +1163,44 @@ static int checkConstraintUnchanged(Expr *pExpr, int *aiChng, int chngRowid){
   return !w.eCode;
 }
 
+/*
+** An instance of the ConstraintAddr object remembers the byte-code addresses
+** for sections of the constraint checks that deal with uniqueness constraints
+** on the rowid and on the upsert constraint.
+**
+** This information is passed into checkReorderConstraintChecks() to insert
+** some OP_Goto operations so that the rowid and upsert constraints occur
+** in the correct order relative to other constraints.
+*/
+typedef struct ConstraintAddr ConstraintAddr;
+struct ConstraintAddr {
+  int ipkTop;          /* Subroutine for rowid constraint check */
+  int upsertTop;       /* Label for upsert constraint check subroutine */
+  int upsertTop2;      /* Copy of upsertTop not cleared by the call */
+  int upsertBtm;       /* upsert constraint returns to this label */
+  int ipkBtm;          /* Return opcode rowid constraint check */
+};
+
+/*
+** Generate any OP_Goto operations needed to cause constraints to be
+** run that haven't already been run.
+*/
+static void reorderConstraintChecks(Vdbe *v, ConstraintAddr *p){
+  if( p->upsertTop ){
+    testcase( sqlite3VdbeLabelHasBeenResolved(v, p->upsertTop) );
+    sqlite3VdbeGoto(v, p->upsertTop);
+    VdbeComment((v, "call upsert subroutine"));
+    sqlite3VdbeResolveLabel(v, p->upsertBtm);
+    p->upsertTop = 0;
+  }
+  if( p->ipkTop ){
+    sqlite3VdbeGoto(v, p->ipkTop);
+    VdbeComment((v, "call rowid unique-check subroutine"));
+    sqlite3VdbeJumpHere(v, p->ipkBtm);
+    p->ipkTop = 0;
+  }
+}
+
 /*
 ** Generate code to do constraint checks prior to an INSERT or an UPDATE
 ** on table pTab.
@@ -1240,7 +1296,8 @@ void sqlite3GenerateConstraintChecks(
   u8 overrideError,    /* Override onError to this if not OE_Default */
   int ignoreDest,      /* Jump to this label on an OE_Ignore resolution */
   int *pbMayReplace,   /* OUT: Set to true if constraint may cause a replace */
-  int *aiChng          /* column i is unchanged if aiChng[i]<0 */
+  int *aiChng,         /* column i is unchanged if aiChng[i]<0 */
+  Upsert *pUpsert      /* ON CONFLICT clauses, if any.  NULL otherwise */
 ){
   Vdbe *v;             /* VDBE under constrution */
   Index *pIdx;         /* Pointer to one of the indices */
@@ -1253,10 +1310,11 @@ void sqlite3GenerateConstraintChecks(
   int addr1;           /* Address of jump instruction */
   int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */
   int nPkField;        /* Number of fields in PRIMARY KEY. 1 for ROWID tables */
-  int ipkTop = 0;      /* Top of the rowid change constraint check */
-  int ipkBottom = 0;   /* Bottom of the rowid change constraint check */
+  ConstraintAddr sAddr;/* Address information for constraint reordering */
+  Index *pUpIdx = 0;   /* Index to which to apply the upsert */
   u8 isUpdate;         /* True if this is an UPDATE operation */
   u8 bAffinityDone = 0;  /* True if the OP_Affinity operation has been run */
+  int upsertBypass = 0;  /* Address of Goto to bypass upsert subroutine */
 
   isUpdate = regOldData!=0;
   db = pParse->db;
@@ -1264,6 +1322,7 @@ void sqlite3GenerateConstraintChecks(
   assert( v!=0 );
   assert( pTab->pSelect==0 );  /* This table is not a VIEW */
   nCol = pTab->nCol;
+  memset(&sAddr, 0, sizeof(sAddr));
   
   /* pPk is the PRIMARY KEY index for WITHOUT ROWID tables and NULL for
   ** normal rowid tables.  nPkField is the number of key fields in the 
@@ -1363,6 +1422,46 @@ void sqlite3GenerateConstraintChecks(
   }
 #endif /* !defined(SQLITE_OMIT_CHECK) */
 
+  /* UNIQUE and PRIMARY KEY constraints should be handled in the following
+  ** order:
+  **
+  **   (1)  OE_Abort, OE_Fail, OE_Rollback, OE_Ignore
+  **   (2)  OE_Update
+  **   (3)  OE_Replace
+  **
+  ** OE_Fail and OE_Ignore must happen before any changes are made.
+  ** OE_Update guarantees that only a single row will change, so it
+  ** must happen before OE_Replace.  Technically, OE_Abort and OE_Rollback
+  ** could happen in any order, but they are grouped up front for
+  ** convenience.
+  **
+  ** Constraint checking code is generated in this order:
+  **   (A)  The rowid constraint
+  **   (B)  Unique index constraints that do not have OE_Replace as their
+  **        default conflict resolution strategy
+  **   (C)  Unique index that do use OE_Replace by default.
+  **
+  ** The ordering of (2) and (3) is accomplished by making sure the linked
+  ** list of indexes attached to a table puts all OE_Replace indexes last
+  ** in the list.  See sqlite3CreateIndex() for where that happens.
+  */
+
+  if( pUpsert ){
+    if( pUpsert->pUpsertTarget==0 ){
+      /* An ON CONFLICT DO NOTHING clause, without a constraint-target.
+      ** Make all unique constraint resolution be OE_Ignore */
+      assert( pUpsert->pUpsertSet==0 );
+      overrideError = OE_Ignore;
+      pUpsert = 0;
+    }else if( (pUpIdx = pUpsert->pUpsertIdx)!=0 ){
+      /* If the constraint-target is on some column other than
+      ** then ROWID, then we might need to move the UPSERT around
+      ** so that it occurs in the correct order. */
+      sAddr.upsertTop = sAddr.upsertTop2 = sqlite3VdbeMakeLabel(v);
+      sAddr.upsertBtm = sqlite3VdbeMakeLabel(v);
+    }
+  }
+
   /* If rowid is changing, make sure the new rowid does not previously
   ** exist in the table.
   */
@@ -1377,6 +1476,32 @@ void sqlite3GenerateConstraintChecks(
       onError = OE_Abort;
     }
 
+    /* figure out whether or not upsert applies in this case */
+    if( pUpsert && pUpsert->pUpsertIdx==0 ){
+      if( pUpsert->pUpsertSet==0 ){
+        onError = OE_Ignore;  /* DO NOTHING is the same as INSERT OR IGNORE */
+      }else{
+        onError = OE_Update;  /* DO UPDATE */
+      }
+    }
+
+    /* If the response to a rowid conflict is REPLACE but the response
+    ** to some other UNIQUE constraint is FAIL or IGNORE, then we need
+    ** to defer the running of the rowid conflict checking until after
+    ** the UNIQUE constraints have run.
+    */
+    assert( OE_Update>OE_Replace );
+    assert( OE_Ignore<OE_Replace );
+    assert( OE_Fail<OE_Replace );
+    assert( OE_Abort<OE_Replace );
+    assert( OE_Rollback<OE_Replace );
+    if( onError>=OE_Replace
+     && (pUpsert || onError!=overrideError)
+     && pTab->pIndex
+    ){
+      sAddr.ipkTop = sqlite3VdbeAddOp0(v, OP_Goto)+1;
+    }
+
     if( isUpdate ){
       /* pkChng!=0 does not mean that the rowid has changed, only that
       ** it might have changed.  Skip the conflict logic below if the rowid
@@ -1386,26 +1511,12 @@ void sqlite3GenerateConstraintChecks(
       VdbeCoverage(v);
     }
 
-    /* If the response to a rowid conflict is REPLACE but the response
-    ** to some other UNIQUE constraint is FAIL or IGNORE, then we need
-    ** to defer the running of the rowid conflict checking until after
-    ** the UNIQUE constraints have run.
-    */
-    if( onError==OE_Replace && overrideError!=OE_Replace ){
-      for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
-        if( pIdx->onError==OE_Ignore || pIdx->onError==OE_Fail ){
-          ipkTop = sqlite3VdbeAddOp0(v, OP_Goto);
-          break;
-        }
-      }
-    }
-
     /* Check to see if the new rowid already exists in the table.  Skip
     ** the following conflict logic if it does not. */
+    VdbeNoopComment((v, "uniqueness check for ROWID"));
     sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, addrRowidOk, regNewData);
     VdbeCoverage(v);
 
-    /* Generate code that deals with a rowid collision */
     switch( onError ){
       default: {
         onError = OE_Abort;
@@ -1414,6 +1525,9 @@ void sqlite3GenerateConstraintChecks(
       case OE_Rollback:
       case OE_Abort:
       case OE_Fail: {
+        testcase( onError==OE_Rollback );
+        testcase( onError==OE_Abort );
+        testcase( onError==OE_Fail );
         sqlite3RowidConstraint(pParse, onError, pTab);
         break;
       }
@@ -1466,16 +1580,22 @@ void sqlite3GenerateConstraintChecks(
         seenReplace = 1;
         break;
       }
+#ifndef SQLITE_OMIT_UPSERT
+      case OE_Update: {
+        sqlite3UpsertDoUpdate(pParse, pUpsert, pTab, 0, iDataCur);
+        /* Fall through */
+      }
+#endif
       case OE_Ignore: {
-        /*assert( seenReplace==0 );*/
+        testcase( onError==OE_Ignore );
         sqlite3VdbeGoto(v, ignoreDest);
         break;
       }
     }
     sqlite3VdbeResolveLabel(v, addrRowidOk);
-    if( ipkTop ){
-      ipkBottom = sqlite3VdbeAddOp0(v, OP_Goto);
-      sqlite3VdbeJumpHere(v, ipkTop);
+    if( sAddr.ipkTop ){
+      sAddr.ipkBtm = sqlite3VdbeAddOp0(v, OP_Goto);
+      sqlite3VdbeJumpHere(v, sAddr.ipkTop-1);
     }
   }
 
@@ -1493,12 +1613,21 @@ void sqlite3GenerateConstraintChecks(
     int addrUniqueOk;    /* Jump here if the UNIQUE constraint is satisfied */
 
     if( aRegIdx[ix]==0 ) continue;  /* Skip indices that do not change */
+    if( pUpIdx==pIdx ){
+      addrUniqueOk = sAddr.upsertBtm;
+      upsertBypass = sqlite3VdbeGoto(v, 0);
+      VdbeComment((v, "Skip upsert subroutine"));
+      sqlite3VdbeResolveLabel(v, sAddr.upsertTop2);
+    }else{
+      addrUniqueOk = sqlite3VdbeMakeLabel(v);
+    }
+    VdbeNoopComment((v, "uniqueness check for %s", pIdx->zName));
     if( bAffinityDone==0 ){
       sqlite3TableAffinity(v, pTab, regNewData+1);
       bAffinityDone = 1;
     }
     iThisCur = iIdxCur+ix;
-    addrUniqueOk = sqlite3VdbeMakeLabel(v);
+
 
     /* Skip partial indices for which the WHERE clause is not true */
     if( pIdx->pPartIdxWhere ){
@@ -1558,6 +1687,24 @@ void sqlite3GenerateConstraintChecks(
       onError = OE_Abort;
     }
 
+    /* Figure out if the upsert clause applies to this index */
+    if( pUpIdx==pIdx ){
+      if( pUpsert->pUpsertSet==0 ){
+        onError = OE_Ignore;  /* DO NOTHING is the same as INSERT OR IGNORE */
+      }else{
+        onError = OE_Update;  /* DO UPDATE */
+      }
+    }
+
+    /* Invoke subroutines to handle IPK replace and upsert prior to running
+    ** the first REPLACE constraint check. */
+    if( onError==OE_Replace ){
+      testcase( sAddr.ipkTop );
+      testcase( sAddr.upsertTop
+             && sqlite3VdbeLabelHasBeenResolved(v,sAddr.upsertTop) );
+      reorderConstraintChecks(v, &sAddr);
+    }
+
     /* Collision detection may be omitted if all of the following are true:
     **   (1) The conflict resolution algorithm is REPLACE
     **   (2) The table is a WITHOUT ROWID table
@@ -1640,15 +1787,25 @@ void sqlite3GenerateConstraintChecks(
 
     /* Generate code that executes if the new index entry is not unique */
     assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail
-        || onError==OE_Ignore || onError==OE_Replace );
+        || onError==OE_Ignore || onError==OE_Replace || onError==OE_Update );
     switch( onError ){
       case OE_Rollback:
       case OE_Abort:
       case OE_Fail: {
+        testcase( onError==OE_Rollback );
+        testcase( onError==OE_Abort );
+        testcase( onError==OE_Fail );
         sqlite3UniqueConstraint(pParse, onError, pIdx);
         break;
       }
+#ifndef SQLITE_OMIT_UPSERT
+      case OE_Update: {
+        sqlite3UpsertDoUpdate(pParse, pUpsert, pTab, pIdx, iIdxCur+ix);
+        /* Fall through */
+      }
+#endif
       case OE_Ignore: {
+        testcase( onError==OE_Ignore );
         sqlite3VdbeGoto(v, ignoreDest);
         break;
       }
@@ -1666,14 +1823,19 @@ void sqlite3GenerateConstraintChecks(
         break;
       }
     }
-    sqlite3VdbeResolveLabel(v, addrUniqueOk);
+    if( pUpIdx==pIdx ){
+      sqlite3VdbeJumpHere(v, upsertBypass);
+    }else{
+      sqlite3VdbeResolveLabel(v, addrUniqueOk);
+    }
     sqlite3ExprCachePop(pParse);
     if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField);
+
   }
-  if( ipkTop ){
-    sqlite3VdbeGoto(v, ipkTop+1);
-    sqlite3VdbeJumpHere(v, ipkBottom);
-  }
+  testcase( sAddr.ipkTop!=0 );
+  testcase( sAddr.upsertTop
+         && sqlite3VdbeLabelHasBeenResolved(v,sAddr.upsertTop) );
+  reorderConstraintChecks(v, &sAddr);
   
   *pbMayReplace = seenReplace;
   VdbeModuleComment((v, "END: GenCnstCks(%d)", seenReplace));

src/main.c

@@ -3861,24 +3861,6 @@ int sqlite3_test_control(int op, ...){
       break;
     }
 
-#ifdef SQLITE_N_KEYWORD
-    /* sqlite3_test_control(SQLITE_TESTCTRL_ISKEYWORD, const char *zWord)
-    **
-    ** If zWord is a keyword recognized by the parser, then return the
-    ** number of keywords.  Or if zWord is not a keyword, return 0.
-    ** 
-    ** This test feature is only available in the amalgamation since
-    ** the SQLITE_N_KEYWORD macro is not defined in this file if SQLite
-    ** is built using separate source files.
-    */
-    case SQLITE_TESTCTRL_ISKEYWORD: {
-      const char *zWord = va_arg(ap, const char*);
-      int n = sqlite3Strlen30(zWord);
-      rc = (sqlite3KeywordCode((u8*)zWord, n)!=TK_ID) ? SQLITE_N_KEYWORD : 0;
-      break;
-    }
-#endif 
-
     /*   sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff);
     **
     ** If parameter onoff is non-zero, configure the wrappers so that all

src/parse.y

@@ -24,8 +24,9 @@
 %token_type {Token}
 %default_type {Token}
 
-// The generated parser function takes a 4th argument as follows:
-%extra_argument {Parse *pParse}
+// An extra argument to the constructor for the parser, which is available
+// to all actions.
+%extra_context {Parse *pParse}
 
 // This code runs whenever there is a syntax error
 //
@@ -205,7 +206,8 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);}
 //
 %fallback ID
   ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW
-  CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR
+  CONFLICT DATABASE DEFERRED DESC DETACH DO
+  EACH END EXCLUSIVE EXPLAIN FAIL FOR
   IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN
   QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW
   ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT
@@ -239,6 +241,7 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);}
 %left CONCAT.
 %left COLLATE.
 %right BITNOT.
+%nonassoc ON.
 
 // An IDENTIFIER can be a generic identifier, or one of several
 // keywords.  Any non-standard keyword can also be an identifier.
@@ -688,6 +691,21 @@ fullname(A) ::= nm(X).
 fullname(A) ::= nm(X) DOT nm(Y).  
    {A = sqlite3SrcListAppend(pParse->db,0,&X,&Y); /*A-overwrites-X*/}
 
+%type xfullname {SrcList*}
+%destructor xfullname {sqlite3SrcListDelete(pParse->db, $$);}
+xfullname(A) ::= nm(X).  
+   {A = sqlite3SrcListAppend(pParse->db,0,&X,0); /*A-overwrites-X*/}
+xfullname(A) ::= nm(X) DOT nm(Y).  
+   {A = sqlite3SrcListAppend(pParse->db,0,&X,&Y); /*A-overwrites-X*/}
+xfullname(A) ::= nm(X) DOT nm(Y) AS nm(Z).  {
+   A = sqlite3SrcListAppend(pParse->db,0,&X,&Y); /*A-overwrites-X*/
+   if( A ) A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z);
+}
+xfullname(A) ::= nm(X) AS nm(Z). {  
+   A = sqlite3SrcListAppend(pParse->db,0,&X,0); /*A-overwrites-X*/
+   if( A ) A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z);
+}
+
 %type joinop {int}
 joinop(X) ::= COMMA|JOIN.              { X = JT_INNER; }
 joinop(X) ::= JOIN_KW(A) JOIN.
@@ -697,10 +715,27 @@ joinop(X) ::= JOIN_KW(A) nm(B) JOIN.
 joinop(X) ::= JOIN_KW(A) nm(B) nm(C) JOIN.
                   {X = sqlite3JoinType(pParse,&A,&B,&C);/*X-overwrites-A*/}
 
+// There is a parsing abiguity in an upsert statement that uses a
+// SELECT on the RHS of a the INSERT:
+//
+//      INSERT INTO tab SELECT * FROM aaa JOIN bbb ON CONFLICT ...
+//                                        here ----^^
+//
+// When the ON token is encountered, the parser does not know if it is
+// the beginning of an ON CONFLICT clause, or the beginning of an ON
+// clause associated with the JOIN.  The conflict is resolved in favor
+// of the JOIN.  If an ON CONFLICT clause is intended, insert a dummy
+// WHERE clause in between, like this:
+//
+//      INSERT INTO tab SELECT * FROM aaa JOIN bbb WHERE true ON CONFLICT ...
+//
+// The [AND] and [OR] precedence marks in the rules for on_opt cause the
+// ON in this context to always be interpreted as belonging to the JOIN.
+//
 %type on_opt {Expr*}
 %destructor on_opt {sqlite3ExprDelete(pParse->db, $$);}
-on_opt(N) ::= ON expr(E).   {N = E;}
-on_opt(N) ::= .             {N = 0;}
+on_opt(N) ::= ON expr(E).  {N = E;}
+on_opt(N) ::= .     [OR]   {N = 0;}
 
 // Note that this block abuses the Token type just a little. If there is
 // no "INDEXED BY" clause, the returned token is empty (z==0 && n==0). If
@@ -781,14 +816,14 @@ limit_opt(A) ::= LIMIT expr(X) COMMA expr(Y).
 /////////////////////////// The DELETE statement /////////////////////////////
 //
 %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-cmd ::= with DELETE FROM fullname(X) indexed_opt(I) where_opt(W) 
+cmd ::= with DELETE FROM xfullname(X) indexed_opt(I) where_opt(W) 
         orderby_opt(O) limit_opt(L). {
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3DeleteFrom(pParse,X,W,O,L);
 }
 %endif
 %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-cmd ::= with DELETE FROM fullname(X) indexed_opt(I) where_opt(W). {
+cmd ::= with DELETE FROM xfullname(X) indexed_opt(I) where_opt(W). {
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3DeleteFrom(pParse,X,W,0,0);
 }
@@ -803,19 +838,19 @@ where_opt(A) ::= WHERE expr(X).       {A = X;}
 ////////////////////////// The UPDATE command ////////////////////////////////
 //
 %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-cmd ::= with UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
+cmd ::= with UPDATE orconf(R) xfullname(X) indexed_opt(I) SET setlist(Y)
         where_opt(W) orderby_opt(O) limit_opt(L).  {
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3ExprListCheckLength(pParse,Y,"set list"); 
-  sqlite3Update(pParse,X,Y,W,R,O,L);
+  sqlite3Update(pParse,X,Y,W,R,O,L,0);
 }
 %endif
 %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-cmd ::= with UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
+cmd ::= with UPDATE orconf(R) xfullname(X) indexed_opt(I) SET setlist(Y)
         where_opt(W).  {
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3ExprListCheckLength(pParse,Y,"set list"); 
-  sqlite3Update(pParse,X,Y,W,R,0,0);
+  sqlite3Update(pParse,X,Y,W,R,0,0,0);
 }
 %endif
 
@@ -839,14 +874,31 @@ setlist(A) ::= LP idlist(X) RP EQ expr(Y). {
 
 ////////////////////////// The INSERT command /////////////////////////////////
 //
-cmd ::= with insert_cmd(R) INTO fullname(X) idlist_opt(F) select(S). {
-  sqlite3Insert(pParse, X, S, F, R);
+cmd ::= with insert_cmd(R) INTO xfullname(X) idlist_opt(F) select(S)
+        upsert(U). {
+  sqlite3Insert(pParse, X, S, F, R, U);
 }
-cmd ::= with insert_cmd(R) INTO fullname(X) idlist_opt(F) DEFAULT VALUES.
+cmd ::= with insert_cmd(R) INTO xfullname(X) idlist_opt(F) DEFAULT VALUES.
 {
-  sqlite3Insert(pParse, X, 0, F, R);
+  sqlite3Insert(pParse, X, 0, F, R, 0);
 }
 
+%type upsert {Upsert*}
+
+// Because upsert only occurs at the tip end of the INSERT rule for cmd,
+// there is never a case where the value of the upsert pointer will not
+// be destroyed by the cmd action.  So comment-out the destructor to
+// avoid unreachable code.
+//%destructor upsert {sqlite3UpsertDelete(pParse->db,$$);}
+upsert(A) ::= . { A = 0; }
+upsert(A) ::= ON CONFLICT LP sortlist(T) RP where_opt(TW)
+              DO UPDATE SET setlist(Z) where_opt(W).
+              { A = sqlite3UpsertNew(pParse->db,T,TW,Z,W);}
+upsert(A) ::= ON CONFLICT LP sortlist(T) RP where_opt(TW) DO NOTHING.
+              { A = sqlite3UpsertNew(pParse->db,T,TW,0,0); }
+upsert(A) ::= ON CONFLICT DO NOTHING.
+              { A = sqlite3UpsertNew(pParse->db,0,0,0,0); }
+
 %type insert_cmd {int}
 insert_cmd(A) ::= INSERT orconf(R).   {A = R;}
 insert_cmd(A) ::= REPLACE.            {A = OE_Replace;}
@@ -1392,9 +1444,9 @@ trigger_cmd(A) ::=
 
 // INSERT
 trigger_cmd(A) ::= scanpt(B) insert_cmd(R) INTO
-                      trnm(X) idlist_opt(F) select(S) scanpt(Z).
-   {A = sqlite3TriggerInsertStep(pParse->db,&X,F,S,R,B,Z);/*A-overwrites-R*/}
-
+                      trnm(X) idlist_opt(F) select(S) upsert(U) scanpt(Z). {
+   A = sqlite3TriggerInsertStep(pParse->db,&X,F,S,R,U,B,Z);/*A-overwrites-R*/
+}
 // DELETE
 trigger_cmd(A) ::= DELETE(B) FROM trnm(X) tridxby where_opt(Y) scanpt(E).
    {A = sqlite3TriggerDeleteStep(pParse->db, &X, Y, B.z, E);}

src/prepare.c

@@ -145,6 +145,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){
   const char *zMasterName;
   int openedTransaction = 0;
 
+  assert( (db->mDbFlags & DBFLAG_SchemaKnownOk)==0 );
   assert( iDb>=0 && iDb<db->nDb );
   assert( db->aDb[iDb].pSchema );
   assert( sqlite3_mutex_held(db->mutex) );
@@ -374,6 +375,7 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){
   }
   /* All other schemas after the main schema. The "temp" schema must be last */
   for(i=db->nDb-1; i>0; i--){
+    assert( i==1 || sqlite3BtreeHoldsMutex(db->aDb[i].pBt) );
     if( !DbHasProperty(db, i, DB_SchemaLoaded) ){
       rc = sqlite3InitOne(db, i, pzErrMsg);
       if( rc ) return rc;
@@ -395,10 +397,12 @@ int sqlite3ReadSchema(Parse *pParse){
   assert( sqlite3_mutex_held(db->mutex) );
   if( !db->init.busy ){
     rc = sqlite3Init(db, &pParse->zErrMsg);
-  }
-  if( rc!=SQLITE_OK ){
-    pParse->rc = rc;
-    pParse->nErr++;
+    if( rc!=SQLITE_OK ){
+      pParse->rc = rc;
+      pParse->nErr++;
+    }else if( db->noSharedCache ){
+      db->mDbFlags |= DBFLAG_SchemaKnownOk;
+    }
   }
   return rc;
 }

src/resolve.c

@@ -191,7 +191,7 @@ static int lookupName(
   struct SrcList_item *pMatch = 0;  /* The matching pSrcList item */
   NameContext *pTopNC = pNC;        /* First namecontext in the list */
   Schema *pSchema = 0;              /* Schema of the expression */
-  int isTrigger = 0;                /* True if resolved to a trigger column */
+  int eNewExprOp = TK_COLUMN;       /* New value for pExpr->op on success */
   Table *pTab = 0;                  /* Table hold the row */
   Column *pCol;                     /* A column of pTab */
 
@@ -296,22 +296,35 @@ static int lookupName(
       }
     } /* if( pSrcList ) */
 
-#ifndef SQLITE_OMIT_TRIGGER
+#if !defined(SQLITE_OMIT_TRIGGER) || !defined(SQLITE_OMIT_UPSERT)
     /* If we have not already resolved the name, then maybe 
-    ** it is a new.* or old.* trigger argument reference
+    ** it is a new.* or old.* trigger argument reference.  Or
+    ** maybe it is an excluded.* from an upsert.
     */
-    if( zDb==0 && zTab!=0 && cntTab==0 && pParse->pTriggerTab!=0 ){
-      int op = pParse->eTriggerOp;
-      assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT );
-      if( op!=TK_DELETE && sqlite3StrICmp("new",zTab) == 0 ){
-        pExpr->iTable = 1;
-        pTab = pParse->pTriggerTab;
-      }else if( op!=TK_INSERT && sqlite3StrICmp("old",zTab)==0 ){
-        pExpr->iTable = 0;
-        pTab = pParse->pTriggerTab;
-      }else{
-        pTab = 0;
+    if( zDb==0 && zTab!=0 && cntTab==0 ){
+      pTab = 0;
+#ifndef SQLITE_OMIT_TRIGGER
+      if( pParse->pTriggerTab!=0 ){
+        int op = pParse->eTriggerOp;
+        assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT );
+        if( op!=TK_DELETE && sqlite3StrICmp("new",zTab) == 0 ){
+          pExpr->iTable = 1;
+          pTab = pParse->pTriggerTab;
+        }else if( op!=TK_INSERT && sqlite3StrICmp("old",zTab)==0 ){
+          pExpr->iTable = 0;
+          pTab = pParse->pTriggerTab;
+        }
       }
+#endif /* SQLITE_OMIT_TRIGGER */
+#ifndef SQLITE_OMIT_UPSERT
+      if( (pNC->ncFlags & NC_UUpsert)!=0 ){
+        Upsert *pUpsert = pNC->uNC.pUpsert;
+        if( pUpsert && sqlite3StrICmp("excluded",zTab)==0 ){
+          pTab = pUpsert->pUpsertSrc->a[0].pTab;
+          pExpr->iTable = 2;
+        }
+      }
+#endif /* SQLITE_OMIT_UPSERT */
 
       if( pTab ){ 
         int iCol;
@@ -331,24 +344,35 @@ static int lookupName(
         }
         if( iCol<pTab->nCol ){
           cnt++;
-          if( iCol<0 ){
-            pExpr->affinity = SQLITE_AFF_INTEGER;
-          }else if( pExpr->iTable==0 ){
-            testcase( iCol==31 );
-            testcase( iCol==32 );
-            pParse->oldmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<<iCol));
-          }else{
-            testcase( iCol==31 );
-            testcase( iCol==32 );
-            pParse->newmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<<iCol));
+#ifndef SQLITE_OMIT_UPSERT
+          if( pExpr->iTable==2 ){
+            testcase( iCol==(-1) );
+            pExpr->iTable = pNC->uNC.pUpsert->regData + iCol;
+            eNewExprOp = TK_REGISTER;
+          }else
+#endif /* SQLITE_OMIT_UPSERT */
+          {
+#ifndef SQLITE_OMIT_TRIGGER
+            if( iCol<0 ){
+              pExpr->affinity = SQLITE_AFF_INTEGER;
+            }else if( pExpr->iTable==0 ){
+              testcase( iCol==31 );
+              testcase( iCol==32 );
+              pParse->oldmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<<iCol));
+            }else{
+              testcase( iCol==31 );
+              testcase( iCol==32 );
+              pParse->newmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<<iCol));
+            }
+            pExpr->pTab = pTab;
+            pExpr->iColumn = (i16)iCol;
+            eNewExprOp = TK_TRIGGER;
+#endif /* SQLITE_OMIT_TRIGGER */
           }
-          pExpr->iColumn = (i16)iCol;
-          pExpr->pTab = pTab;
-          isTrigger = 1;
         }
       }
     }
-#endif /* !defined(SQLITE_OMIT_TRIGGER) */
+#endif /* !defined(SQLITE_OMIT_TRIGGER) || !defined(SQLITE_OMIT_UPSERT) */
 
     /*
     ** Perhaps the name is a reference to the ROWID
@@ -383,10 +407,12 @@ static int lookupName(
     ** is supported for backwards compatibility only. Hence, we issue a warning
     ** on sqlite3_log() whenever the capability is used.
     */
-    if( (pEList = pNC->pEList)!=0
-     && zTab==0
+    if( (pNC->ncFlags & NC_UEList)!=0
      && cnt==0
+     && zTab==0
     ){
+      pEList = pNC->uNC.pEList;
+      assert( pEList!=0 );
       for(j=0; j<pEList->nExpr; j++){
         char *zAs = pEList->a[j].zName;
         if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){
@@ -483,7 +509,7 @@ static int lookupName(
   pExpr->pLeft = 0;
   sqlite3ExprDelete(db, pExpr->pRight);
   pExpr->pRight = 0;
-  pExpr->op = (isTrigger ? TK_TRIGGER : TK_COLUMN);
+  pExpr->op = eNewExprOp;
   ExprSetProperty(pExpr, EP_Leaf);
 lookupname_end:
   if( cnt==1 ){
@@ -915,8 +941,8 @@ static int resolveOrderByTermToExprList(
   memset(&nc, 0, sizeof(nc));
   nc.pParse = pParse;
   nc.pSrcList = pSelect->pSrc;
-  nc.pEList = pEList;
-  nc.ncFlags = NC_AllowAgg;
+  nc.uNC.pEList = pEList;
+  nc.ncFlags = NC_AllowAgg|NC_UEList;
   nc.nErr = 0;
   db = pParse->db;
   savedSuppErr = db->suppressErr;
@@ -1299,7 +1325,9 @@ static int resolveSelectStep(Walker *pWalker, Select *p){
     ** Minor point: If this is the case, then the expression will be
     ** re-evaluated for each reference to it.
     */
-    sNC.pEList = p->pEList;
+    assert( (sNC.ncFlags & (NC_UAggInfo|NC_UUpsert))==0 );
+    sNC.uNC.pEList = p->pEList;
+    sNC.ncFlags |= NC_UEList;
     if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort;
     if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort;
 
@@ -1532,7 +1560,7 @@ void sqlite3ResolveSelfReference(
   Table *pTab,        /* The table being referenced */
   int type,           /* NC_IsCheck or NC_PartIdx or NC_IdxExpr */
   Expr *pExpr,        /* Expression to resolve.  May be NULL. */
-  ExprList *pList     /* Expression list to resolve.  May be NUL. */
+  ExprList *pList     /* Expression list to resolve.  May be NULL. */
 ){
   SrcList sSrc;                   /* Fake SrcList for pParse->pNewTable */
   NameContext sNC;                /* Name context for pParse->pNewTable */

src/select.c

@@ -21,7 +21,7 @@
 /***/ int sqlite3SelectTrace = 0;
 # define SELECTTRACE(K,P,S,X)  \
   if(sqlite3SelectTrace&(K))   \
-    sqlite3DebugPrintf("%s/%p: ",(S)->zSelName,(S)),\
+    sqlite3DebugPrintf("%s/%d/%p: ",(S)->zSelName,(P)->iSelectId,(S)),\
     sqlite3DebugPrintf X
 #else
 # define SELECTTRACE(K,P,S,X)
@@ -579,7 +579,6 @@ static void pushOntoSorter(
   if( nPrefixReg==0 && nData>0 ){
     sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData);
   }
-  sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord);
   if( nOBSat>0 ){
     int regPrevKey;   /* The first nOBSat columns of the previous row */
     int addrFirst;    /* Address of the OP_IfNot opcode */
@@ -588,6 +587,7 @@ static void pushOntoSorter(
     int nKey;         /* Number of sorting key columns, including OP_Sequence */
     KeyInfo *pKI;     /* Original KeyInfo on the sorter table */
 
+    sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat,regRecord);
     regPrevKey = pParse->nMem+1;
     pParse->nMem += pSort->nOBSat;
     nKey = nExpr - pSort->nOBSat + bSeq;
@@ -621,6 +621,35 @@ static void pushOntoSorter(
     sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat);
     sqlite3VdbeJumpHere(v, addrJmp);
   }
+  if( iLimit ){
+    /* At this point the values for the new sorter entry are stored
+    ** in an array of registers. They need to be composed into a record
+    ** and inserted into the sorter if either (a) there are currently
+    ** less than LIMIT+OFFSET items or (b) the new record is smaller than 
+    ** the largest record currently in the sorter. If (b) is true and there
+    ** are already LIMIT+OFFSET items in the sorter, delete the largest
+    ** entry before inserting the new one. This way there are never more 
+    ** than LIMIT+OFFSET items in the sorter.
+    **
+    ** If the new record does not need to be inserted into the sorter,
+    ** jump to the next iteration of the loop. Or, if the
+    ** pSort->bOrderedInnerLoop flag is set to indicate that the inner
+    ** loop delivers items in sorted order, jump to the next iteration
+    ** of the outer loop.
+    */
+    int iCsr = pSort->iECursor;
+    int iJmp = sqlite3VdbeCurrentAddr(v)+5+(nOBSat<=0)+pSort->bOrderedInnerLoop;
+    assert( pSort->bOrderedInnerLoop==0 || pSort->bOrderedInnerLoop==1 );
+    sqlite3VdbeAddOp2(v, OP_IfNotZero, iLimit, sqlite3VdbeCurrentAddr(v)+4);
+    VdbeCoverage(v);
+    sqlite3VdbeAddOp2(v, OP_Last, iCsr, 0);
+    sqlite3VdbeAddOp4Int(v, OP_IdxLE, iCsr, iJmp, regBase+nOBSat, nExpr-nOBSat);
+    VdbeCoverage(v);
+    sqlite3VdbeAddOp1(v, OP_Delete, iCsr);
+  }
+  if( nOBSat<=0 ){
+    sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat,regRecord);
+  }
   if( pSort->sortFlags & SORTFLAG_UseSorter ){
     op = OP_SorterInsert;
   }else{
@@ -628,34 +657,6 @@ static void pushOntoSorter(
   }
   sqlite3VdbeAddOp4Int(v, op, pSort->iECursor, regRecord,
                        regBase+nOBSat, nBase-nOBSat);
-  if( iLimit ){
-    int addr;
-    int r1 = 0;
-    /* Fill the sorter until it contains LIMIT+OFFSET entries.  (The iLimit
-    ** register is initialized with value of LIMIT+OFFSET.)  After the sorter
-    ** fills up, delete the least entry in the sorter after each insert.
-    ** Thus we never hold more than the LIMIT+OFFSET rows in memory at once */
-    addr = sqlite3VdbeAddOp1(v, OP_IfNotZero, iLimit); VdbeCoverage(v);
-    sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor);
-    if( pSort->bOrderedInnerLoop ){
-      r1 = ++pParse->nMem;
-      sqlite3VdbeAddOp3(v, OP_Column, pSort->iECursor, nExpr, r1);
-      VdbeComment((v, "seq"));
-    }
-    sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor);
-    if( pSort->bOrderedInnerLoop ){
-      /* If the inner loop is driven by an index such that values from
-      ** the same iteration of the inner loop are in sorted order, then
-      ** immediately jump to the next iteration of an inner loop if the
-      ** entry from the current iteration does not fit into the top
-      ** LIMIT+OFFSET entries of the sorter. */
-      int iBrk = sqlite3VdbeCurrentAddr(v) + 2;
-      sqlite3VdbeAddOp3(v, OP_Eq, regBase+nExpr, iBrk, r1);
-      sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
-      VdbeCoverage(v);
-    }
-    sqlite3VdbeJumpHere(v, addr);
-  }
 }
 
 /*
@@ -3819,9 +3820,8 @@ static int flattenSubquery(
       if( pPrior ) pPrior->pNext = pNew;
       pNew->pNext = p;
       p->pPrior = pNew;
-      SELECTTRACE(2,pParse,p,
-         ("compound-subquery flattener creates %s.%p as peer\n",
-         pNew->zSelName, pNew));
+      SELECTTRACE(2,pParse,p,("compound-subquery flattener"
+                              " creates %s.%p as peer\n",pNew->zSelName, pNew));
     }
     if( db->mallocFailed ) return 1;
   }
@@ -5353,7 +5353,10 @@ int sqlite3Select(
   if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1;
   memset(&sAggInfo, 0, sizeof(sAggInfo));
 #if SELECTTRACE_ENABLED
-  SELECTTRACE(1,pParse,p, ("begin processing:\n"));
+#ifndef SQLITE_OMIT_EXPLAIN
+  p->iSelectId = pParse->iSelectId;
+#endif
+  SELECTTRACE(1,pParse,p, ("begin processing:\n", pParse->iSelectId));
   if( sqlite3SelectTrace & 0x100 ){
     sqlite3TreeViewSelect(0, p, 0);
   }
@@ -5384,8 +5387,8 @@ int sqlite3Select(
   assert( p->pEList!=0 );
   isAgg = (p->selFlags & SF_Aggregate)!=0;
 #if SELECTTRACE_ENABLED
-  if( sqlite3SelectTrace & 0x100 ){
-    SELECTTRACE(0x100,pParse,p, ("after name resolution:\n"));
+  if( sqlite3SelectTrace & 0x104 ){
+    SELECTTRACE(0x104,pParse,p, ("after name resolution:\n"));
     sqlite3TreeViewSelect(0, p, 0);
   }
 #endif
@@ -5486,10 +5489,13 @@ int sqlite3Select(
   */
   if( p->pPrior ){
     rc = multiSelect(pParse, p, pDest);
-    explainSetInteger(pParse->iSelectId, iRestoreSelectId);
 #if SELECTTRACE_ENABLED
-    SELECTTRACE(1,pParse,p,("end compound-select processing\n"));
+    SELECTTRACE(0x1,pParse,p,("end compound-select processing\n"));
+    if( pParse->iSelectId==0 && (sqlite3SelectTrace & 0x2000)!=0 ){
+      sqlite3TreeViewSelect(0, p, 0);
+    }
 #endif
+    explainSetInteger(pParse->iSelectId, iRestoreSelectId);
     return rc;
   }
 #endif
@@ -5868,7 +5874,8 @@ int sqlite3Select(
     memset(&sNC, 0, sizeof(sNC));
     sNC.pParse = pParse;
     sNC.pSrcList = pTabList;
-    sNC.pAggInfo = &sAggInfo;
+    sNC.uNC.pAggInfo = &sAggInfo;
+    VVA_ONLY( sNC.ncFlags = NC_UAggInfo; )
     sAggInfo.mnReg = pParse->nMem+1;
     sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0;
     sAggInfo.pGroupBy = pGroupBy;
@@ -6273,12 +6280,15 @@ int sqlite3Select(
   ** successful coding of the SELECT.
   */
 select_end:
-  explainSetInteger(pParse->iSelectId, iRestoreSelectId);
   sqlite3ExprListDelete(db, pMinMaxOrderBy);
   sqlite3DbFree(db, sAggInfo.aCol);
   sqlite3DbFree(db, sAggInfo.aFunc);
 #if SELECTTRACE_ENABLED
-  SELECTTRACE(1,pParse,p,("end processing\n"));
+  SELECTTRACE(0x1,pParse,p,("end processing\n"));
+  if( pParse->iSelectId==0 && (sqlite3SelectTrace & 0x2000)!=0 ){
+    sqlite3TreeViewSelect(0, p, 0);
+  }
 #endif
+  explainSetInteger(pParse->iSelectId, iRestoreSelectId);
   return rc;
 }

src/shell.c.in

@@ -436,6 +436,12 @@ void utf8_printf(FILE *out, const char *zFormat, ...){
 # define raw_printf fprintf
 #endif
 
+/* Indicate out-of-memory and exit. */
+static void shell_out_of_memory(void){
+  raw_printf(stderr,"Error: out of memory\n");
+  exit(1);
+}
+
 /*
 ** Write I/O traces to the following stream.
 */
@@ -768,45 +774,12 @@ static void appendText(ShellText *p, char const *zAppend, char quote){
 ** Return '"' if quoting is required.  Return 0 if no quoting is required.
 */
 static char quoteChar(const char *zName){
-  /* All SQLite keywords, in alphabetical order */
-  static const char *azKeywords[] = {
-    "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
-    "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
-    "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
-    "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
-    "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
-    "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
-    "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
-    "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
-    "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
-    "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
-    "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
-    "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
-    "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
-    "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
-    "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
-    "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
-    "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
-    "WITH", "WITHOUT",
-  };
-  int i, lwr, upr, mid, c;
+  int i;
   if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"';
   for(i=0; zName[i]; i++){
     if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"';
   }
-  lwr = 0;
-  upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1;
-  while( lwr<=upr ){
-    mid = (lwr+upr)/2;
-    c = sqlite3_stricmp(azKeywords[mid], zName);
-    if( c==0 ) return '"';
-    if( c<0 ){
-      lwr = mid+1;
-    }else{
-      upr = mid-1;
-    }
-  }
-  return 0;
+  return sqlite3_keyword_check(zName, i) ? '"' : 0;
 }
 
 /*
@@ -967,6 +940,7 @@ INCLUDE ../ext/misc/shathree.c
 INCLUDE ../ext/misc/fileio.c
 INCLUDE ../ext/misc/completion.c
 INCLUDE ../ext/misc/appendvfs.c
+INCLUDE ../ext/misc/bindvtab.c
 #ifdef SQLITE_HAVE_ZLIB
 INCLUDE ../ext/misc/zipfile.c
 INCLUDE ../ext/misc/sqlar.c
@@ -1005,6 +979,22 @@ struct ExpertInfo {
   int bVerbose;
 };
 
+/* A single line in the EQP output */
+typedef struct EQPGraphRow EQPGraphRow;
+struct EQPGraphRow {
+  int iSelectId;        /* The SelectID for this row */
+  EQPGraphRow *pNext;   /* Next row in sequence */
+  char zText[1];        /* Text to display for this row */
+};
+
+/* All EQP output is collected into an instance of the following */
+typedef struct EQPGraph EQPGraph;
+struct EQPGraph {
+  EQPGraphRow *pRow;    /* Linked list of all rows of the EQP output */
+  EQPGraphRow *pLast;   /* Last element of the pRow list */
+  char zPrefix[100];    /* Graph prefix */
+};
+
 /*
 ** State information about the database connection is contained in an
 ** instance of the following structure.
@@ -1018,6 +1008,8 @@ struct ShellState {
   u8 scanstatsOn;        /* True to display scan stats before each finalize */
   u8 openMode;           /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
   u8 doXdgOpen;          /* Invoke start/open/xdg-open in output_reset() */
+  u8 nEqpLevel;          /* Depth of the EQP output graph */
+  unsigned mEqpLines;    /* Mask of veritical lines in the EQP output graph */
   int outCount;          /* Revert to stdout when reaching zero */
   int cnt;               /* Number of records displayed so far */
   FILE *out;             /* Write results here */
@@ -1051,6 +1043,7 @@ struct ShellState {
   int *aiIndent;         /* Array of indents used in MODE_Explain */
   int nIndent;           /* Size of array aiIndent[] */
   int iIndent;           /* Index of current op in aiIndent[] */
+  EQPGraph sGraph;       /* Information for the graphical EXPLAIN QUERY PLAN */
 #if defined(SQLITE_ENABLE_SESSION)
   int nSession;             /* Number of active sessions */
   OpenSession aSession[4];  /* Array of sessions.  [0] is in focus. */
@@ -1107,6 +1100,7 @@ struct ShellState {
 #define MODE_Explain  9  /* Like MODE_Column, but do not truncate data */
 #define MODE_Ascii   10  /* Use ASCII unit and record separators (0x1F/0x1E) */
 #define MODE_Pretty  11  /* Pretty-print schemas */
+#define MODE_EQP     12  /* Converts EXPLAIN QUERY PLAN output into a graph */
 
 static const char *modeDescr[] = {
   "line",
@@ -1121,6 +1115,7 @@ static const char *modeDescr[] = {
   "explain",
   "ascii",
   "prettyprint",
+  "eqp"
 };
 
 /*
@@ -1669,7 +1664,106 @@ static int wsToEol(const char *z){
   }
   return 1;
 }
-    
+
+/*
+** Add a new entry to the EXPLAIN QUERY PLAN data
+*/
+static void eqp_append(ShellState *p, int iSelectId, const char *zText){
+  EQPGraphRow *pNew;
+  int nText = strlen30(zText);
+  pNew = sqlite3_malloc64( sizeof(*pNew) + nText );
+  if( pNew==0 ) shell_out_of_memory();
+  pNew->iSelectId = iSelectId;
+  memcpy(pNew->zText, zText, nText+1);
+  pNew->pNext = 0;
+  if( p->sGraph.pLast ){
+    p->sGraph.pLast->pNext = pNew;
+  }else{
+    p->sGraph.pRow = pNew;
+  }
+  p->sGraph.pLast = pNew;
+}
+
+/*
+** Free and reset the EXPLAIN QUERY PLAN data that has been collected
+** in p->sGraph.
+*/
+static void eqp_reset(ShellState *p){
+  EQPGraphRow *pRow, *pNext;
+  for(pRow = p->sGraph.pRow; pRow; pRow = pNext){
+    pNext = pRow->pNext;
+    sqlite3_free(pRow);
+  }
+  memset(&p->sGraph, 0, sizeof(p->sGraph));
+}
+
+/* Return the next EXPLAIN QUERY PLAN line with iSelectId that occurs after
+** pOld, or return the first such line if pOld is NULL
+*/
+static EQPGraphRow *eqp_next_row(ShellState *p, int iSelectId, EQPGraphRow *pOld){
+  EQPGraphRow *pRow = pOld ? pOld->pNext : p->sGraph.pRow;
+  while( pRow && pRow->iSelectId!=iSelectId ) pRow = pRow->pNext;
+  return pRow;
+}
+
+/* Render a single level of the graph shell having iSelectId.  Called
+** recursively to render sublevels.
+*/
+static void eqp_render_level(ShellState *p, int iSelectId){
+  EQPGraphRow *pRow, *pNext;
+  int i;
+  int n = strlen30(p->sGraph.zPrefix);
+  char *z;
+  for(pRow = eqp_next_row(p, iSelectId, 0); pRow; pRow = pNext){
+    pNext = eqp_next_row(p, iSelectId, pRow);
+    z = pRow->zText;
+    utf8_printf(p->out, "%s%s%s\n", p->sGraph.zPrefix, pNext ? "|--" : "`--", z);
+    if( n<sizeof(p->sGraph.zPrefix)-7 && (z = strstr(z, " SUBQUER"))!=0 ){
+      memcpy(&p->sGraph.zPrefix[n], pNext ? "|  " : "   ", 4);
+      if( strncmp(z, " SUBQUERY ", 9)==0 && (i = atoi(z+10))>iSelectId ){
+        eqp_render_level(p, i);
+      }else if( strncmp(z, " SUBQUERIES ", 12)==0 ){
+        i = atoi(z+12);
+        if( i>iSelectId ){
+          utf8_printf(p->out, "%s|--SUBQUERY %d\n", p->sGraph.zPrefix, i);
+          memcpy(&p->sGraph.zPrefix[n+3],"|  ",4);
+          eqp_render_level(p, i);
+        }
+        z = strstr(z, " AND ");
+        if( z && (i = atoi(z+5))>iSelectId ){
+          p->sGraph.zPrefix[n+3] = 0;
+          utf8_printf(p->out, "%s`--SUBQUERY %d\n", p->sGraph.zPrefix, i);
+          memcpy(&p->sGraph.zPrefix[n+3],"   ",4);
+          eqp_render_level(p, i);
+        }
+      }
+      p->sGraph.zPrefix[n] = 0;
+    }
+  }
+}
+
+/*
+** Display and reset the EXPLAIN QUERY PLAN data
+*/
+static void eqp_render(ShellState *p){
+  EQPGraphRow *pRow = p->sGraph.pRow;
+  if( pRow ){
+    if( pRow->zText[0]=='-' ){
+      if( pRow->pNext==0 ){
+        eqp_reset(p);
+        return;
+      }
+      utf8_printf(p->out, "%s\n", pRow->zText+3);
+      p->sGraph.pRow = pRow->pNext;
+      sqlite3_free(pRow);
+    }else{
+      utf8_printf(p->out, "QUERY PLAN\n");
+    }
+    p->sGraph.zPrefix[0] = 0;
+    eqp_render_level(p, 0);
+    eqp_reset(p);
+  }
+}
 
 /*
 ** This is the callback routine that the shell
@@ -2020,6 +2114,10 @@ static int shell_callback(
       utf8_printf(p->out, "%s", p->rowSeparator);
       break;
     }
+    case MODE_EQP: {
+      eqp_append(p, atoi(azArg[0]), azArg[3]);
+      break;
+    }
   }
   return 0;
 }
@@ -2118,10 +2216,7 @@ static void set_table_name(ShellState *p, const char *zName){
   n = strlen30(zName);
   if( cQuote ) n += n+2;
   z = p->zDestTable = malloc( n+1 );
-  if( z==0 ){
-    raw_printf(stderr,"Error: out of memory\n");
-    exit(1);
-  }
+  if( z==0 ) shell_out_of_memory();
   n = 0;
   if( cQuote ) z[n++] = cQuote;
   for(i=0; zName[i]; i++){
@@ -2861,11 +2956,12 @@ static int shell_exec(
         rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
         if( rc==SQLITE_OK ){
           while( sqlite3_step(pExplain)==SQLITE_ROW ){
-            raw_printf(pArg->out,"--EQP-- %d,",sqlite3_column_int(pExplain, 0));
-            raw_printf(pArg->out,"%d,", sqlite3_column_int(pExplain, 1));
-            raw_printf(pArg->out,"%d,", sqlite3_column_int(pExplain, 2));
-            utf8_printf(pArg->out,"%s\n", sqlite3_column_text(pExplain, 3));
+            const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3);
+            int iSelectId = sqlite3_column_int(pExplain, 0);
+            if( zEQPLine[0]=='-' ) eqp_render(pArg);
+            eqp_append(pArg, iSelectId, zEQPLine);
           }
+          eqp_render(pArg);
         }
         sqlite3_finalize(pExplain);
         sqlite3_free(zEQP);
@@ -2893,11 +2989,16 @@ static int shell_exec(
 
       if( pArg ){
         pArg->cMode = pArg->mode;
-        if( pArg->autoExplain
-         && sqlite3_column_count(pStmt)==8
-         && sqlite3_strlike("EXPLAIN%", zStmtSql,0)==0
-        ){
-          pArg->cMode = MODE_Explain;
+        if( pArg->autoExplain ){
+          if( sqlite3_column_count(pStmt)==8
+           && sqlite3_strlike("EXPLAIN%", zStmtSql,0)==0
+          ){
+            pArg->cMode = MODE_Explain;
+          }
+          if( sqlite3_column_count(pStmt)==4
+           && sqlite3_strlike("EXPLAIN QUERY PLAN%", zStmtSql,0)==0 ){
+            pArg->cMode = MODE_EQP;
+          }
         }
 
         /* If the shell is currently in ".explain" mode, gather the extra
@@ -2907,8 +3008,10 @@ static int shell_exec(
         }
       }
 
+      shell_bindings_apply(pStmt);
       exec_prepared_stmt(pArg, pStmt);
       explain_data_delete(pArg);
+      eqp_render(pArg);
 
       /* print usage stats if stats on */
       if( pArg && pArg->statsOn ){
@@ -2986,10 +3089,7 @@ static char **tableColumnList(ShellState *p, const char *zTab){
     if( nCol>=nAlloc-2 ){
       nAlloc = nAlloc*2 + nCol + 10;
       azCol = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0]));
-      if( azCol==0 ){
-        raw_printf(stderr, "Error: out of memory\n");
-        exit(1);
-      }
+      if( azCol==0 ) shell_out_of_memory();
     }
     azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1));
     if( sqlite3_column_int(pStmt, 5) ){
@@ -3308,6 +3408,7 @@ static char zHelp[] =
 #if defined(SQLITE_ENABLE_SESSION)
   ".session CMD ...       Create or control sessions\n"
 #endif
+  ".set KEY=VALUE         Set bind parameter KEY to be string VALUE\n"
   ".sha3sum ?OPTIONS...?  Compute a SHA3 hash of database content\n"
 #ifndef SQLITE_NOHAVE_SYSTEM
   ".shell CMD ARGS...     Run CMD ARGS... in a system shell\n"
@@ -3484,7 +3585,6 @@ static int deduceDatabaseType(const char *zName, int dfltZip){
 */
 static void open_db(ShellState *p, int keepAlive){
   if( p->db==0 ){
-    sqlite3_initialize();
     if( p->openMode==SHELL_OPEN_UNSPEC && access(p->zDbFilename,0)==0 ){
       p->openMode = (u8)deduceDatabaseType(p->zDbFilename, 0);
     }
@@ -3521,6 +3621,7 @@ static void open_db(ShellState *p, int keepAlive){
     sqlite3_fileio_init(p->db, 0, 0);
     sqlite3_shathree_init(p->db, 0, 0);
     sqlite3_completion_init(p->db, 0, 0);
+    sqlite3_bindvtab_init(p->db, 0, 0);
 #ifdef SQLITE_HAVE_ZLIB
     sqlite3_zipfile_init(p->db, 0, 0);
     sqlite3_sqlar_init(p->db, 0, 0);
@@ -3787,10 +3888,7 @@ static void import_append_char(ImportCtx *p, int c){
   if( p->n+1>=p->nAlloc ){
     p->nAlloc += p->nAlloc + 100;
     p->z = sqlite3_realloc64(p->z, p->nAlloc);
-    if( p->z==0 ){
-      raw_printf(stderr, "out of memory\n");
-      exit(1);
-    }
+    if( p->z==0 ) shell_out_of_memory();
   }
   p->z[p->n++] = (char)c;
 }
@@ -3951,10 +4049,7 @@ static void tryToCloneData(
   }
   n = sqlite3_column_count(pQuery);
   zInsert = sqlite3_malloc64(200 + nTable + n*3);
-  if( zInsert==0 ){
-    raw_printf(stderr, "out of memory\n");
-    goto end_data_xfer;
-  }
+  if( zInsert==0 ) shell_out_of_memory();
   sqlite3_snprintf(200+nTable,zInsert,
                    "INSERT OR IGNORE INTO \"%s\" VALUES(?", zTable);
   i = strlen30(zInsert);
@@ -4295,14 +4390,6 @@ static int shellDatabaseError(sqlite3 *db){
   return 1;
 }
 
-/*
-** Print an out-of-memory message to stderr and return 1.
-*/
-static int shellNomemError(void){
-  raw_printf(stderr, "Error: out of memory\n");
-  return 1;
-}
-
 /*
 ** Compare the pattern in zGlob[] against the text in z[].  Return TRUE
 ** if they match and FALSE (0) if they do not match.
@@ -5996,9 +6083,8 @@ static int do_meta_command(char *zLine, ShellState *p){
     sCtx.cRowSep = p->rowSeparator[0];
     zSql = sqlite3_mprintf("SELECT * FROM %s", zTable);
     if( zSql==0 ){
-      raw_printf(stderr, "Error: out of memory\n");
       xCloser(sCtx.in);
-      return 1;
+      shell_out_of_memory();
     }
     nByte = strlen30(zSql);
     rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
@@ -6043,9 +6129,8 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( nCol==0 ) return 0; /* no columns, no error */
     zSql = sqlite3_malloc64( nByte*2 + 20 + nCol*2 );
     if( zSql==0 ){
-      raw_printf(stderr, "Error: out of memory\n");
       xCloser(sCtx.in);
-      return 1;
+      shell_out_of_memory();
     }
     sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\" VALUES(?", zTable);
     j = strlen30(zSql);
@@ -6121,12 +6206,17 @@ static int do_meta_command(char *zLine, ShellState *p){
     sqlite3_stmt *pStmt;
     int tnum = 0;
     int i;
-    if( nArg!=3 ){
-      utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n");
+    if( !(nArg==3 || (nArg==2 && sqlite3_stricmp(azArg[1],"off")==0)) ){
+      utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n"
+                          "       .imposter off\n");
       rc = 1;
       goto meta_command_exit;
     }
     open_db(p, 0);
+    if( nArg==2 ){
+      sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 1);
+      goto meta_command_exit;
+    }
     zSql = sqlite3_mprintf("SELECT rootpage FROM sqlite_master"
                            " WHERE name='%q' AND type='index'", azArg[1]);
     sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
@@ -7062,6 +7152,30 @@ static int do_meta_command(char *zLine, ShellState *p){
     }
   }else
 
+  if( c=='s' && n==3 && strncmp(azArg[0],"set",3)==0 ){
+    int x;
+    char *zKey = 0;
+    char *zToFree = 0;
+    if( nArg==2 ){
+      zKey = azArg[1];
+    }else if( nArg==3 ){
+      zKey = zToFree = sqlite3_mprintf("%s=%s",azArg[1],azArg[2]);
+    }else{
+      raw_printf(stderr,
+         "Usage: .set KEY VALUE\n   Or: .set KEY=VALUE\n"
+         "Use SQL on the \"shell_bindings\" table to query or delete keys.\n"
+      );
+      rc = 1;
+      goto meta_command_exit;
+    }
+    x = shell_bindings_new_text(zKey);
+    if( x ){
+      utf8_printf(stderr, "Error: bad setting: %s\n", zKey);
+    }
+    sqlite3_free(zToFree);
+  }else
+
+
   if( c=='s' && n>=4 && strncmp(azArg[0],"sha3sum",n)==0 ){
     const char *zLike = 0;   /* Which table to checksum. 0 means everything */
     int i;                   /* Loop counter */
@@ -7311,18 +7425,12 @@ static int do_meta_command(char *zLine, ShellState *p){
         char **azNew;
         int n2 = nAlloc*2 + 10;
         azNew = sqlite3_realloc64(azResult, sizeof(azResult[0])*n2);
-        if( azNew==0 ){
-          rc = shellNomemError();
-          break;
-        }
+        if( azNew==0 ) shell_out_of_memory();
         nAlloc = n2;
         azResult = azNew;
       }
       azResult[nRow] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));
-      if( 0==azResult[nRow] ){
-        rc = shellNomemError();
-        break;
-      }
+      if( 0==azResult[nRow] ) shell_out_of_memory();
       nRow++;
     }
     if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
@@ -7383,9 +7491,6 @@ static int do_meta_command(char *zLine, ShellState *p){
       { "byteorder",          SQLITE_TESTCTRL_BYTEORDER,     ""                   },
     /*{ "fault_install",      SQLITE_TESTCTRL_FAULT_INSTALL, ""                }, */
       { "imposter",           SQLITE_TESTCTRL_IMPOSTER,   "SCHEMA ON/OFF ROOTPAGE"},
-#ifdef SQLITE_N_KEYWORD
-      { "iskeyword",          SQLITE_TESTCTRL_ISKEYWORD,     "IDENTIFIER"         },
-#endif
       { "localtime_fault",    SQLITE_TESTCTRL_LOCALTIME_FAULT,"BOOLEAN"           },
       { "never_corrupt",      SQLITE_TESTCTRL_NEVER_CORRUPT, "BOOLEAN"            },
       { "optimizations",      SQLITE_TESTCTRL_OPTIMIZATIONS, "DISABLE-MASK"       },
@@ -7497,17 +7602,6 @@ static int do_meta_command(char *zLine, ShellState *p){
           }
           break;
 
-        /* sqlite3_test_control(int, char *) */
-#ifdef SQLITE_N_KEYWORD
-        case SQLITE_TESTCTRL_ISKEYWORD:
-          if( nArg==3 ){
-            const char *opt = azArg[2];
-            rc2 = sqlite3_test_control(testctrl, opt);
-            isOk = 1;
-          }
-          break;
-#endif
-
         case SQLITE_TESTCTRL_IMPOSTER:
           if( nArg==5 ){
             rc2 = sqlite3_test_control(testctrl, p->db,
@@ -7893,10 +7987,7 @@ static int process_input(ShellState *p, FILE *in){
     if( nSql+nLine+2>=nAlloc ){
       nAlloc = nSql+nLine+100;
       zSql = realloc(zSql, nAlloc);
-      if( zSql==0 ){
-        raw_printf(stderr, "Error: out of memory\n");
-        exit(1);
-      }
+      if( zSql==0 ) shell_out_of_memory();
     }
     nSqlPrior = nSql;
     if( nSql==0 ){
@@ -8025,7 +8116,6 @@ static void process_sqliterc(
                       " cannot read ~/.sqliterc\n");
       return;
     }
-    sqlite3_initialize();
     zBuf = sqlite3_mprintf("%s/.sqliterc",home_dir);
     sqliterc = zBuf;
   }
@@ -8054,6 +8144,7 @@ static const char zOptions[] =
   "   -column              set output mode to 'column'\n"
   "   -cmd COMMAND         run \"COMMAND\" before reading stdin\n"
   "   -csv                 set output mode to 'csv'\n"
+  "   -Dkey=value          set shell binding variable \"key\" to \"value\"\n"
   "   -echo                print commands before execution\n"
   "   -init FILENAME       read/process named file\n"
   "   -[no]header          turn headers on or off\n"
@@ -8102,6 +8193,17 @@ static void usage(int showDetail){
   exit(1);
 }
 
+/*
+** Internal check:  Verify that the SQLite is uninitialized.  Print a
+** error message if it is initialized.
+*/
+static void verify_uninitialized(void){
+  if( sqlite3_config(-1)==SQLITE_MISUSE ){
+    utf8_printf(stdout, "WARNING: attempt to configuration SQLite after"
+                        " initialization.\n");
+  }
+}
+
 /*
 ** Initialize the state information in data
 */
@@ -8113,6 +8215,7 @@ static void main_init(ShellState *data) {
   memcpy(data->rowSeparator,SEP_Row, 2);
   data->showHeader = 0;
   data->shellFlgs = SHFLG_Lookaside;
+  verify_uninitialized();
   sqlite3_config(SQLITE_CONFIG_URI, 1);
   sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data);
   sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
@@ -8176,6 +8279,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   int readStdin = 1;
   int nCmd = 0;
   char **azCmd = 0;
+  const char *zVfs = 0;           /* Value of -vfs command-line option */
 
   setBinaryMode(stdin, 0);
   setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */
@@ -8200,23 +8304,14 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #if !SQLITE_SHELL_IS_UTF8
   sqlite3_initialize();
   argv = malloc(sizeof(argv[0])*argc);
-  if( argv==0 ){
-    raw_printf(stderr, "out of memory\n");
-    exit(1);
-  }
+  if( argv==0 ) shell_out_of_memory();
   for(i=0; i<argc; i++){
     char *z = sqlite3_win32_unicode_to_utf8(wargv[i]);
     int n;
-    if( z==0 ){
-      raw_printf(stderr, "out of memory\n");
-      exit(1);
-    }
+    if( z==0 ) shell_out_of_memory();
     n = (int)strlen(z);
     argv[i] = malloc( n+1 );
-    if( argv[i]==0 ){
-      raw_printf(stderr, "out of memory\n");
-      exit(1);
-    }
+    if( argv[i]==0 ) shell_out_of_memory();
     memcpy(argv[i], z, n+1);
     sqlite3_free(z);
   }
@@ -8252,6 +8347,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   ** the size of the alternative malloc heap,
   ** and the first command to execute.
   */
+  verify_uninitialized();
   for(i=1; i<argc; i++){
     char *z;
     z = argv[i];
@@ -8264,10 +8360,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
         readStdin = 0;
         nCmd++;
         azCmd = realloc(azCmd, sizeof(azCmd[0])*nCmd);
-        if( azCmd==0 ){
-          raw_printf(stderr, "out of memory\n");
-          exit(1);
-        }
+        if( azCmd==0 ) shell_out_of_memory();
         azCmd[nCmd-1] = z;
       }
     }
@@ -8340,13 +8433,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
       sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, (int)sz);
 #endif
     }else if( strcmp(z,"-vfs")==0 ){
-      sqlite3_vfs *pVfs = sqlite3_vfs_find(cmdline_option_value(argc,argv,++i));
-      if( pVfs ){
-        sqlite3_vfs_register(pVfs, 1);
-      }else{
-        utf8_printf(stderr, "no such VFS: \"%s\"\n", argv[i]);
-        exit(1);
-      }
+      zVfs = cmdline_option_value(argc, argv, ++i);
 #ifdef SQLITE_HAVE_ZLIB
     }else if( strcmp(z,"-zip")==0 ){
       data.openMode = SHELL_OPEN_ZIPFILE;
@@ -8361,8 +8448,42 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
       ** command, so ignore them */
       break;
 #endif
+    }else if( strncmp(z, "-D",2)==0 ){
+      int x;
+      x = shell_bindings_new_text(z+2);
+      if( x ){
+        utf8_printf(stderr, "Error: bad binding: %s\n", z);
+      }
     }
   }
+  verify_uninitialized();
+
+
+#ifdef SQLITE_SHELL_INIT_PROC
+  {
+    /* If the SQLITE_SHELL_INIT_PROC macro is defined, then it is the name
+    ** of a C-function that will perform initialization actions on SQLite that
+    ** occur just before or after sqlite3_initialize(). Use this compile-time
+    ** option to embed this shell program in larger applications. */
+    extern void SQLITE_SHELL_INIT_PROC(void);
+    SQLITE_SHELL_INIT_PROC();
+  }
+#else
+  /* All the sqlite3_config() calls have now been made. So it is safe
+  ** to call sqlite3_initialize() and process any command line -vfs option. */
+  sqlite3_initialize();
+#endif
+
+  if( zVfs ){
+    sqlite3_vfs *pVfs = sqlite3_vfs_find(zVfs);
+    if( pVfs ){
+      sqlite3_vfs_register(pVfs, 1);
+    }else{
+      utf8_printf(stderr, "no such VFS: \"%s\"\n", argv[i]);
+      exit(1);
+    }
+  }
+
   if( data.zDbFilename==0 ){
 #ifndef SQLITE_OMIT_MEMORYDB
     data.zDbFilename = ":memory:";
@@ -8529,6 +8650,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
       readStdin = 0;
       break;
 #endif
+    }else if( strncmp(z, "-D", 2)==0 ){
+      /* Noop */
     }else{
       utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z);
       raw_printf(stderr,"Use -help for a list of options.\n");
@@ -8607,6 +8730,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   }
   sqlite3_free(data.zFreeOnClose);
   find_home_dir(1);
+  shell_bindings_clear();
   output_reset(&data);
   data.doXdgOpen = 0;
   clearTempFile(&data);

src/sqlite.h.in

@@ -7016,7 +7016,7 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_ALWAYS                  13
 #define SQLITE_TESTCTRL_RESERVE                 14
 #define SQLITE_TESTCTRL_OPTIMIZATIONS           15
-#define SQLITE_TESTCTRL_ISKEYWORD               16
+#define SQLITE_TESTCTRL_ISKEYWORD               16  /* NOT USED */
 #define SQLITE_TESTCTRL_SCRATCHMALLOC           17  /* NOT USED */
 #define SQLITE_TESTCTRL_LOCALTIME_FAULT         18
 #define SQLITE_TESTCTRL_EXPLAIN_STMT            19  /* NOT USED */
@@ -7030,6 +7030,57 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_PARSER_COVERAGE         26
 #define SQLITE_TESTCTRL_LAST                    26  /* Largest TESTCTRL */
 
+/*
+** CAPI3REF: SQL Keyword Checking
+**
+** These routines provide access to the set of SQL language keywords 
+** recognized by SQLite.  Applications can uses these routines to determine
+** whether or not a specific identifier needs to be escaped (for example,
+** by enclosing in double-quotes) so as not to confuse the parser.
+**
+** The sqlite3_keyword_count() interface returns the number of distinct
+** keywords understood by SQLite.
+**
+** The sqlite3_keyword_name(N,Z,L) interface finds the N-th keyword and
+** makes *Z point to that keyword expressed as UTF8 and writes the number
+** of bytes in the keyword into *L.  The string that *Z points to is not
+** zero-terminated.  The sqlite3_keyword_name(N,Z,L) routine returns
+** SQLITE_OK if N is within bounds and SQLITE_ERROR if not. If either Z
+** or L are NULL or invalid pointers then calls to
+** sqlite3_keyword_name(N,Z,L) result in undefined behavior.
+**
+** The sqlite3_keyword_check(Z,L) interface checks to see whether or not
+** the L-byte UTF8 identifier that Z points to is a keyword, returning non-zero
+** if it is and zero if not.
+**
+** The parser used by SQLite is forgiving.  It is often possible to use
+** a keyword as an identifier as long as such use does not result in a
+** parsing ambiguity.  For example, the statement
+** "CREATE TABLE BEGIN(REPLACE,PRAGMA,END);" is accepted by SQLite, and
+** creates a new table named "BEGIN" with three columns named
+** "REPLACE", "PRAGMA", and "END".  Nevertheless, best practice is to avoid
+** using keywords as identifiers.  Common techniques used to avoid keyword
+** name collisions include:
+** <ul>
+** <li> Put all indentifier names inside double-quotes.  This is the official
+**      SQL way to escape identifier names.
+** <li> Put identifier names inside &#91;...&#93;.  This is not standard SQL,
+**      but it is what SQL Server does and so lots of programmers use this
+**      technique.
+** <li> Begin every identifier with the letter "Z" as no SQL keywords start
+**      with "Z".
+** <li> Include a digit somewhere in every identifier name.
+** </ul>
+**
+** Note that the number of keywords understood by SQLite can depend on
+** compile-time options.  For example, "VACUUM" is not a keyword if
+** SQLite is compiled with the [-DSQLITE_OMIT_VACUUM] option.  Also,
+** new keywords may be added to future releases of SQLite.
+*/
+int sqlite3_keyword_count(void);
+int sqlite3_keyword_name(int,const char**,int*);
+int sqlite3_keyword_check(const char*,int);
+
 /*
 ** CAPI3REF: SQLite Runtime Status
 **

src/sqliteInt.h

@@ -1102,6 +1102,7 @@ typedef struct Trigger Trigger;
 typedef struct TriggerPrg TriggerPrg;
 typedef struct TriggerStep TriggerStep;
 typedef struct UnpackedRecord UnpackedRecord;
+typedef struct Upsert Upsert;
 typedef struct VTable VTable;
 typedef struct VtabCtx VtabCtx;
 typedef struct Walker Walker;
@@ -1361,7 +1362,7 @@ struct sqlite3 {
   u8 vtabOnConflict;            /* Value to return for s3_vtab_on_conflict() */
   u8 isTransactionSavepoint;    /* True if the outermost savepoint is a TS */
   u8 mTrace;                    /* zero or more SQLITE_TRACE flags */
-  u8 skipBtreeMutex;            /* True if no shared-cache backends */
+  u8 noSharedCache;             /* True if no shared-cache backends */
   u8 nSqlExec;                  /* Number of pending OP_SqlExec opcodes */
   int nextPagesize;             /* Pagesize after VACUUM if >0 */
   u32 magic;                    /* Magic number for detect library misuse */
@@ -1521,6 +1522,7 @@ struct sqlite3 {
 #define DBFLAG_SchemaChange   0x0001  /* Uncommitted Hash table changes */
 #define DBFLAG_PreferBuiltin  0x0002  /* Preference to built-in funcs */
 #define DBFLAG_Vacuum         0x0004  /* Currently in a VACUUM */
+#define DBFLAG_SchemaKnownOk  0x0008  /* Schema is known to be valid */
 
 /*
 ** Bits of the sqlite3.dbOptFlags field that are used by the
@@ -2054,13 +2056,12 @@ struct FKey {
 #define OE_Fail     3   /* Stop the operation but leave all prior changes */
 #define OE_Ignore   4   /* Ignore the error. Do not do the INSERT or UPDATE */
 #define OE_Replace  5   /* Delete existing record, then do INSERT or UPDATE */
-
-#define OE_Restrict 6   /* OE_Abort for IMMEDIATE, OE_Rollback for DEFERRED */
-#define OE_SetNull  7   /* Set the foreign key value to NULL */
-#define OE_SetDflt  8   /* Set the foreign key value to its default */
-#define OE_Cascade  9   /* Cascade the changes */
-
-#define OE_Default  10  /* Do whatever the default action is */
+#define OE_Update   6   /* Process as a DO UPDATE in an upsert */
+#define OE_Restrict 7   /* OE_Abort for IMMEDIATE, OE_Rollback for DEFERRED */
+#define OE_SetNull  8   /* Set the foreign key value to NULL */
+#define OE_SetDflt  9   /* Set the foreign key value to its default */
+#define OE_Cascade  10  /* Cascade the changes */
+#define OE_Default  11  /* Do whatever the default action is */
 
 
 /*
@@ -2691,8 +2692,11 @@ struct SrcList {
 struct NameContext {
   Parse *pParse;       /* The parser */
   SrcList *pSrcList;   /* One or more tables used to resolve names */
-  ExprList *pEList;    /* Optional list of result-set columns */
-  AggInfo *pAggInfo;   /* Information about aggregates at this level */
+  union {
+    ExprList *pEList;    /* Optional list of result-set columns */
+    AggInfo *pAggInfo;   /* Information about aggregates at this level */
+    Upsert *pUpsert;     /* ON CONFLICT clause information from an upsert */
+  } uNC;
   NameContext *pNext;  /* Next outer name context.  NULL for outermost */
   int nRef;            /* Number of names resolved by this context */
   int nErr;            /* Number of errors encountered while resolving names */
@@ -2714,9 +2718,42 @@ struct NameContext {
 #define NC_HasAgg    0x0010  /* One or more aggregate functions seen */
 #define NC_IdxExpr   0x0020  /* True if resolving columns of CREATE INDEX */
 #define NC_VarSelect 0x0040  /* A correlated subquery has been seen */
+#define NC_UEList    0x0080  /* True if uNC.pEList is used */
+#define NC_UAggInfo  0x0100  /* True if uNC.pAggInfo is used */
+#define NC_UUpsert   0x0200  /* True if uNC.pUpsert is used */
 #define NC_MinMaxAgg 0x1000  /* min/max aggregates seen.  See note above */
 #define NC_Complex   0x2000  /* True if a function or subquery seen */
 
+/*
+** An instance of the following object describes a single ON CONFLICT
+** clause in an upsert.
+**
+** The pUpsertTarget field is only set if the ON CONFLICT clause includes
+** conflict-target clause.  (In "ON CONFLICT(a,b)" the "(a,b)" is the
+** conflict-target clause.)  The pUpsertTargetWhere is the optional
+** WHERE clause used to identify partial unique indexes.
+**
+** pUpsertSet is the list of column=expr terms of the UPDATE statement. 
+** The pUpsertSet field is NULL for a ON CONFLICT DO NOTHING.  The
+** pUpsertWhere is the WHERE clause for the UPDATE and is NULL if the
+** WHERE clause is omitted.
+*/
+struct Upsert {
+  ExprList *pUpsertTarget;  /* Optional description of conflicting index */
+  Expr *pUpsertTargetWhere; /* WHERE clause for partial index targets */
+  ExprList *pUpsertSet;     /* The SET clause from an ON CONFLICT UPDATE */
+  Expr *pUpsertWhere;       /* WHERE clause for the ON CONFLICT UPDATE */
+  /* The fields above comprise the parse tree for the upsert clause.
+  ** The fields below are used to transfer information from the INSERT
+  ** processing down into the UPDATE processing while generating code.
+  ** Upsert owns the memory allocated above, but not the memory below. */
+  Index *pUpsertIdx;        /* Constraint that pUpsertTarget identifies */
+  SrcList *pUpsertSrc;      /* Table to be updated */
+  int regData;              /* First register holding array of VALUES */
+  int iDataCur;             /* Index of the data cursor */
+  int iIdxCur;              /* Index of the first index cursor */
+};
+
 /*
 ** An instance of the following structure contains all information
 ** needed to generate code for a single SELECT statement.
@@ -2745,6 +2782,7 @@ struct Select {
   int iLimit, iOffset;   /* Memory registers holding LIMIT & OFFSET counters */
 #if SELECTTRACE_ENABLED
   char zSelName[12];     /* Symbolic name of this SELECT use for debugging */
+  u32 iSelectId;         /* EXPLAIN QUERY PLAN select ID */
 #endif
   int addrOpenEphm[2];   /* OP_OpenEphem opcodes related to this select */
   SrcList *pSrc;         /* The FROM clause */
@@ -3216,8 +3254,9 @@ struct TriggerStep {
   Select *pSelect;     /* SELECT statement or RHS of INSERT INTO SELECT ... */
   char *zTarget;       /* Target table for DELETE, UPDATE, INSERT */
   Expr *pWhere;        /* The WHERE clause for DELETE or UPDATE steps */
-  ExprList *pExprList; /* SET clause for UPDATE. */
+  ExprList *pExprList; /* SET clause for UPDATE */
   IdList *pIdList;     /* Column names for INSERT */
+  Upsert *pUpsert;     /* Upsert clauses on an INSERT */
   char *zSpan;         /* Original SQL text of this command */
   TriggerStep *pNext;  /* Next in the link-list */
   TriggerStep *pLast;  /* Last element in link-list. Valid for 1st elem only */
@@ -3750,7 +3789,7 @@ void sqlite3DeleteTable(sqlite3*, Table*);
 # define sqlite3AutoincrementBegin(X)
 # define sqlite3AutoincrementEnd(X)
 #endif
-void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int);
+void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int, Upsert*);
 void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*);
 IdList *sqlite3IdListAppend(sqlite3*, IdList*, Token*);
 int sqlite3IdListIndex(IdList*,const char*);
@@ -3780,7 +3819,8 @@ void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
 Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,char*);
 #endif
 void sqlite3DeleteFrom(Parse*, SrcList*, Expr*, ExprList*, Expr*);
-void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*);
+void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*,
+                   Upsert*);
 WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int);
 void sqlite3WhereEnd(WhereInfo*);
 LogEst sqlite3WhereOutputRowCount(WhereInfo*);
@@ -3873,7 +3913,7 @@ void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int);
 int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int);
 void sqlite3ResolvePartIdxLabel(Parse*,int);
 void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int,
-                                     u8,u8,int,int*,int*);
+                                     u8,u8,int,int*,int*,Upsert*);
 #ifdef SQLITE_ENABLE_NULL_TRIM
   void sqlite3SetMakeRecordP5(Vdbe*,Table*);
 #else
@@ -3926,7 +3966,8 @@ void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int);
   TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*,
                                         const char*,const char*);
   TriggerStep *sqlite3TriggerInsertStep(sqlite3*,Token*, IdList*,
-                                        Select*,u8,const char*,const char*);
+                                        Select*,u8,Upsert*,
+                                        const char*,const char*);
   TriggerStep *sqlite3TriggerUpdateStep(sqlite3*,Token*,ExprList*, Expr*, u8,
                                         const char*,const char*);
   TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*,
@@ -4174,10 +4215,10 @@ char sqlite3IndexColumnAffinity(sqlite3*, Index*, int);
 ** The interface to the LEMON-generated parser
 */
 #ifndef SQLITE_AMALGAMATION
-  void *sqlite3ParserAlloc(void*(*)(u64));
+  void *sqlite3ParserAlloc(void*(*)(u64), Parse*);
   void sqlite3ParserFree(void*, void(*)(void*));
 #endif
-void sqlite3Parser(void*, int, Token, Parse*);
+void sqlite3Parser(void*, int, Token);
 #ifdef YYTRACKMAXSTACKDEPTH
   int sqlite3ParserStackPeak(void*);
 #endif
@@ -4265,6 +4306,18 @@ const char *sqlite3JournalModename(int);
 #define sqlite3WithPush(x,y,z)
 #define sqlite3WithDelete(x,y)
 #endif
+#ifndef SQLITE_OMIT_UPSERT
+  Upsert *sqlite3UpsertNew(sqlite3*,ExprList*,Expr*,ExprList*,Expr*);
+  void sqlite3UpsertDelete(sqlite3*,Upsert*);
+  Upsert *sqlite3UpsertDup(sqlite3*,Upsert*);
+  int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*);
+  void sqlite3UpsertDoUpdate(Parse*,Upsert*,Table*,Index*,int);
+#else
+#define sqlite3UpsertNew(v,w,x,y,z) ((Upsert*)0)
+#define sqlite3UpsertDelete(x,y)
+#define sqlite3UpsertDup(x,y)       ((Upsert*)0)
+#endif
+
 
 /* Declarations for functions in fkey.c. All of these are replaced by
 ** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign

src/tokenize.c

@@ -496,9 +496,9 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
   /* sqlite3ParserTrace(stdout, "parser: "); */
 #ifdef sqlite3Parser_ENGINEALWAYSONSTACK
   pEngine = &sEngine;
-  sqlite3ParserInit(pEngine);
+  sqlite3ParserInit(pEngine, pParse);
 #else
-  pEngine = sqlite3ParserAlloc(sqlite3Malloc);
+  pEngine = sqlite3ParserAlloc(sqlite3Malloc, pParse);
   if( pEngine==0 ){
     sqlite3OomFault(db);
     return SQLITE_NOMEM_BKPT;
@@ -542,7 +542,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
     }else{
       pParse->sLastToken.z = zSql;
       pParse->sLastToken.n = n;
-      sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
+      sqlite3Parser(pEngine, tokenType, pParse->sLastToken);
       lastTokenParsed = tokenType;
       zSql += n;
       if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break;

src/treeview.c

@@ -62,11 +62,13 @@ static void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){
     }
     sqlite3StrAccumAppend(&acc, p->bLine[i] ? "|-- " : "'-- ", 4);
   }
-  va_start(ap, zFormat);
-  sqlite3VXPrintf(&acc, zFormat, ap);
-  va_end(ap);
-  assert( acc.nChar>0 );
-  if( zBuf[acc.nChar-1]!='\n' ) sqlite3StrAccumAppend(&acc, "\n", 1);
+  if( zFormat!=0 ){
+    va_start(ap, zFormat);
+    sqlite3VXPrintf(&acc, zFormat, ap);
+    va_end(ap);
+    assert( acc.nChar>0 );
+    sqlite3StrAccumAppend(&acc, "\n", 1);
+  }
   sqlite3StrAccumFinish(&acc);
   fprintf(stdout,"%s", zBuf);
   fflush(stdout);
@@ -139,10 +141,10 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){
   do{
 #if SELECTTRACE_ENABLED
     sqlite3TreeViewLine(pView,
-      "SELECT%s%s (%s/%p) selFlags=0x%x nSelectRow=%d",
+      "SELECT%s%s (%s/%d/%p) selFlags=0x%x nSelectRow=%d",
       ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""),
       ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""),
-      p->zSelName, p, p->selFlags,
+      p->zSelName, p->iSelectId, p, p->selFlags,
       (int)p->nSelectRow
     );
 #else
@@ -536,16 +538,21 @@ void sqlite3TreeViewBareExprList(
     for(i=0; i<pList->nExpr; i++){
       int j = pList->a[i].u.x.iOrderByCol;
       char *zName = pList->a[i].zName;
+      int moreToFollow = i<pList->nExpr - 1;
       if( j || zName ){
-        sqlite3TreeViewPush(pView, 0);
+        sqlite3TreeViewPush(pView, moreToFollow);
+        moreToFollow = 0;
+        sqlite3TreeViewLine(pView, 0);
+        if( zName ){
+          fprintf(stdout, "AS %s ", zName);
+        }
+        if( j ){
+          fprintf(stdout, "iOrderByCol=%d", j);
+        }
+        fprintf(stdout, "\n");
+        fflush(stdout);
       }
-      if( zName ){
-        sqlite3TreeViewLine(pView, "AS %s", zName);
-      }
-      if( j ){
-        sqlite3TreeViewLine(pView, "iOrderByCol=%d", j);
-      }
-      sqlite3TreeViewExpr(pView, pList->a[i].pExpr, i<pList->nExpr-1);
+      sqlite3TreeViewExpr(pView, pList->a[i].pExpr, moreToFollow);
       if( j || zName ){
         sqlite3TreeViewPop(pView);
       }

src/trigger.c

@@ -25,6 +25,7 @@ void sqlite3DeleteTriggerStep(sqlite3 *db, TriggerStep *pTriggerStep){
     sqlite3ExprListDelete(db, pTmp->pExprList);
     sqlite3SelectDelete(db, pTmp->pSelect);
     sqlite3IdListDelete(db, pTmp->pIdList);
+    sqlite3UpsertDelete(db, pTmp->pUpsert);
     sqlite3DbFree(db, pTmp->zSpan);
 
     sqlite3DbFree(db, pTmp);
@@ -416,6 +417,7 @@ TriggerStep *sqlite3TriggerInsertStep(
   IdList *pColumn,    /* List of columns in pTableName to insert into */
   Select *pSelect,    /* A SELECT statement that supplies values */
   u8 orconf,          /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */
+  Upsert *pUpsert,    /* ON CONFLICT clauses for upsert */
   const char *zStart, /* Start of SQL text */
   const char *zEnd    /* End of SQL text */
 ){
@@ -427,9 +429,13 @@ TriggerStep *sqlite3TriggerInsertStep(
   if( pTriggerStep ){
     pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE);
     pTriggerStep->pIdList = pColumn;
+    pTriggerStep->pUpsert = pUpsert;
     pTriggerStep->orconf = orconf;
   }else{
+    testcase( pColumn );
     sqlite3IdListDelete(db, pColumn);
+    testcase( pUpsert );
+    sqlite3UpsertDelete(db, pUpsert);
   }
   sqlite3SelectDelete(db, pSelect);
 
@@ -746,7 +752,7 @@ static int codeTriggerProgram(
           targetSrcList(pParse, pStep),
           sqlite3ExprListDup(db, pStep->pExprList, 0), 
           sqlite3ExprDup(db, pStep->pWhere, 0), 
-          pParse->eOrconf, 0, 0
+          pParse->eOrconf, 0, 0, 0
         );
         break;
       }
@@ -755,7 +761,8 @@ static int codeTriggerProgram(
           targetSrcList(pParse, pStep),
           sqlite3SelectDup(db, pStep->pSelect, 0), 
           sqlite3IdListDup(db, pStep->pIdList), 
-          pParse->eOrconf
+          pParse->eOrconf,
+          sqlite3UpsertDup(db, pStep->pUpsert)
         );
         break;
       }

src/update.c

@@ -93,7 +93,8 @@ void sqlite3Update(
   Expr *pWhere,          /* The WHERE clause.  May be null */
   int onError,           /* How to handle constraint errors */
   ExprList *pOrderBy,    /* ORDER BY clause. May be null */
-  Expr *pLimit           /* LIMIT clause. May be null */
+  Expr *pLimit,          /* LIMIT clause. May be null */
+  Upsert *pUpsert        /* ON CONFLICT clause, or null */
 ){
   int i, j;              /* Loop counters */
   Table *pTab;           /* The table to be updated */
@@ -200,16 +201,23 @@ void sqlite3Update(
   ** need to occur right after the database cursor.  So go ahead and
   ** allocate enough space, just in case.
   */
-  pTabList->a[0].iCursor = iBaseCur = iDataCur = pParse->nTab++;
+  iBaseCur = iDataCur = pParse->nTab++;
   iIdxCur = iDataCur+1;
   pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab);
+  testcase( pPk!=0 && pPk!=pTab->pIndex );
   for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){
-    if( IsPrimaryKeyIndex(pIdx) && pPk!=0 ){
+    if( pPk==pIdx ){
       iDataCur = pParse->nTab;
-      pTabList->a[0].iCursor = iDataCur;
     }
     pParse->nTab++;
   }
+  if( pUpsert ){
+    /* On an UPSERT, reuse the same cursors already opened by INSERT */
+    iDataCur = pUpsert->iDataCur;
+    iIdxCur = pUpsert->iIdxCur;
+    pParse->nTab = iBaseCur;
+  }
+  pTabList->a[0].iCursor = iDataCur;
 
   /* Allocate space for aXRef[], aRegIdx[], and aToOpen[].  
   ** Initialize aXRef[] and aToOpen[] to their default values.
@@ -226,6 +234,8 @@ void sqlite3Update(
   memset(&sNC, 0, sizeof(sNC));
   sNC.pParse = pParse;
   sNC.pSrcList = pTabList;
+  sNC.uNC.pUpsert = pUpsert;
+  sNC.ncFlags = NC_UUpsert;
 
   /* Resolve the column names in all the expressions of the
   ** of the UPDATE statement.  Also find the column index
@@ -329,7 +339,7 @@ void sqlite3Update(
   v = sqlite3GetVdbe(pParse);
   if( v==0 ) goto update_cleanup;
   if( pParse->nested==0 ) sqlite3VdbeCountChanges(v);
-  sqlite3BeginWriteOperation(pParse, 1, iDb);
+  sqlite3BeginWriteOperation(pParse, pTrigger || hasFK, iDb);
 
   /* Allocate required registers. */
   if( !IsVirtual(pTab) ){
@@ -380,8 +390,16 @@ void sqlite3Update(
   }
 #endif
 
-  /* Initialize the count of updated rows */
-  if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab ){
+  /* Jump to labelBreak to abandon further processing of this UPDATE */
+  labelContinue = labelBreak = sqlite3VdbeMakeLabel(v);
+
+  /* Not an UPSERT.  Normal processing.  Begin by
+  ** initialize the count of updated rows */
+  if( (db->flags&SQLITE_CountRows)!=0
+   && !pParse->pTriggerTab
+   && !pParse->nested
+   && pUpsert==0
+  ){
     regRowCount = ++pParse->nMem;
     sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount);
   }
@@ -394,46 +412,61 @@ void sqlite3Update(
     iPk = pParse->nMem+1;
     pParse->nMem += nPk;
     regKey = ++pParse->nMem;
-    iEph = pParse->nTab++;
-
-    sqlite3VdbeAddOp3(v, OP_Null, 0, iPk, iPk+nPk-1);
-    addrOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEph, nPk);
-    sqlite3VdbeSetP4KeyInfo(pParse, pPk);
-  }
-
-  /* Begin the database scan. 
-  **
-  ** Do not consider a single-pass strategy for a multi-row update if
-  ** there are any triggers or foreign keys to process, or rows may
-  ** be deleted as a result of REPLACE conflict handling. Any of these
-  ** things might disturb a cursor being used to scan through the table
-  ** or index, causing a single-pass approach to malfunction.  */
-  flags = WHERE_ONEPASS_DESIRED|WHERE_SEEK_UNIQ_TABLE;
-  if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){
-    flags |= WHERE_ONEPASS_MULTIROW;
-  }
-  pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, flags, iIdxCur);
-  if( pWInfo==0 ) goto update_cleanup;
-
-  /* A one-pass strategy that might update more than one row may not
-  ** be used if any column of the index used for the scan is being
-  ** updated. Otherwise, if there is an index on "b", statements like
-  ** the following could create an infinite loop:
-  **
-  **   UPDATE t1 SET b=b+1 WHERE b>?
-  **
-  ** Fall back to ONEPASS_OFF if where.c has selected a ONEPASS_MULTI
-  ** strategy that uses an index for which one or more columns are being
-  ** updated.  */
-  eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
-  if( eOnePass==ONEPASS_MULTI ){
-    int iCur = aiCurOnePass[1];
-    if( iCur>=0 && iCur!=iDataCur && aToOpen[iCur-iBaseCur] ){
-      eOnePass = ONEPASS_OFF;
+    if( pUpsert==0 ){
+      iEph = pParse->nTab++;
+        sqlite3VdbeAddOp3(v, OP_Null, 0, iPk, iPk+nPk-1);
+      addrOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEph, nPk);
+      sqlite3VdbeSetP4KeyInfo(pParse, pPk);
     }
-    assert( iCur!=iDataCur || !HasRowid(pTab) );
   }
   
+  if( pUpsert ){
+    /* If this is an UPSERT, then all cursors have already been opened by
+    ** the outer INSERT and the data cursor should be pointing at the row
+    ** that is to be updated.  So bypass the code that searches for the
+    ** row(s) to be updated.
+    */
+    pWInfo = 0;
+    eOnePass = ONEPASS_SINGLE;
+    sqlite3ExprIfFalse(pParse, pWhere, labelBreak, SQLITE_JUMPIFNULL);
+  }else{
+    /* Begin the database scan. 
+    **
+    ** Do not consider a single-pass strategy for a multi-row update if
+    ** there are any triggers or foreign keys to process, or rows may
+    ** be deleted as a result of REPLACE conflict handling. Any of these
+    ** things might disturb a cursor being used to scan through the table
+    ** or index, causing a single-pass approach to malfunction.  */
+    flags = WHERE_ONEPASS_DESIRED|WHERE_SEEK_UNIQ_TABLE;
+    if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){
+      flags |= WHERE_ONEPASS_MULTIROW;
+    }
+    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, flags, iIdxCur);
+    if( pWInfo==0 ) goto update_cleanup;
+  
+    /* A one-pass strategy that might update more than one row may not
+    ** be used if any column of the index used for the scan is being
+    ** updated. Otherwise, if there is an index on "b", statements like
+    ** the following could create an infinite loop:
+    **
+    **   UPDATE t1 SET b=b+1 WHERE b>?
+    **
+    ** Fall back to ONEPASS_OFF if where.c has selected a ONEPASS_MULTI
+    ** strategy that uses an index for which one or more columns are being
+    ** updated.  */
+    eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
+    if( eOnePass!=ONEPASS_SINGLE ){
+      sqlite3MultiWrite(pParse);
+      if( eOnePass==ONEPASS_MULTI ){
+        int iCur = aiCurOnePass[1];
+        if( iCur>=0 && iCur!=iDataCur && aToOpen[iCur-iBaseCur] ){
+          eOnePass = ONEPASS_OFF;
+        }
+        assert( iCur!=iDataCur || !HasRowid(pTab) );
+      }
+    }
+  }
+
   if( HasRowid(pTab) ){
     /* Read the rowid of the current row of the WHERE scan. In ONEPASS_OFF
     ** mode, write the rowid into the FIFO. In either of the one-pass modes,
@@ -453,7 +486,7 @@ void sqlite3Update(
       sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur,pPk->aiColumn[i],iPk+i);
     }
     if( eOnePass ){
-      sqlite3VdbeChangeToNoop(v, addrOpen);
+      if( addrOpen ) sqlite3VdbeChangeToNoop(v, addrOpen);
       nKey = nPk;
       regKey = iPk;
     }else{
@@ -463,59 +496,58 @@ void sqlite3Update(
     }
   }
 
-  if( eOnePass!=ONEPASS_MULTI ){
-    sqlite3WhereEnd(pWInfo);
-  }
-
-  labelBreak = sqlite3VdbeMakeLabel(v);
-  if( !isView ){
-    int addrOnce = 0;
-
-    /* Open every index that needs updating. */
+  if( pUpsert==0 ){
+    if( eOnePass!=ONEPASS_MULTI ){
+      sqlite3WhereEnd(pWInfo);
+    }
+  
+    if( !isView ){
+      int addrOnce = 0;
+  
+      /* Open every index that needs updating. */
+      if( eOnePass!=ONEPASS_OFF ){
+        if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0;
+        if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0;
+      }
+  
+      if( eOnePass==ONEPASS_MULTI && (nIdx-(aiCurOnePass[1]>=0))>0 ){
+        addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
+      }
+      sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur,
+                                 aToOpen, 0, 0);
+      if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
+    }
+  
+    /* Top of the update loop */
     if( eOnePass!=ONEPASS_OFF ){
-      if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0;
-      if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0;
-    }
-
-    if( eOnePass==ONEPASS_MULTI && (nIdx-(aiCurOnePass[1]>=0))>0 ){
-      addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
-    }
-    sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, aToOpen,
-                               0, 0);
-    if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
-  }
-
-  /* Top of the update loop */
-  if( eOnePass!=ONEPASS_OFF ){
-    if( !isView && aiCurOnePass[0]!=iDataCur && aiCurOnePass[1]!=iDataCur ){
-      assert( pPk );
-      sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey);
-      VdbeCoverageNeverTaken(v);
-    }
-    if( eOnePass==ONEPASS_SINGLE ){
-      labelContinue = labelBreak;
-    }else{
+      if( !isView && aiCurOnePass[0]!=iDataCur && aiCurOnePass[1]!=iDataCur ){
+        assert( pPk );
+        sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey,nKey);
+        VdbeCoverageNeverTaken(v);
+      }
+      if( eOnePass!=ONEPASS_SINGLE ){
+        labelContinue = sqlite3VdbeMakeLabel(v);
+      }
+      sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak);
+      VdbeCoverageIf(v, pPk==0);
+      VdbeCoverageIf(v, pPk!=0);
+    }else if( pPk ){
       labelContinue = sqlite3VdbeMakeLabel(v);
+      sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v);
+      addrTop = sqlite3VdbeAddOp2(v, OP_RowData, iEph, regKey);
+      sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelContinue, regKey, 0);
+      VdbeCoverage(v);
+    }else{
+      labelContinue = sqlite3VdbeAddOp3(v, OP_RowSetRead, regRowSet,labelBreak,
+                               regOldRowid);
+      VdbeCoverage(v);
+      sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue, regOldRowid);
+      VdbeCoverage(v);
     }
-    sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak);
-    VdbeCoverageIf(v, pPk==0);
-    VdbeCoverageIf(v, pPk!=0);
-  }else if( pPk ){
-    labelContinue = sqlite3VdbeMakeLabel(v);
-    sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v);
-    addrTop = sqlite3VdbeAddOp2(v, OP_RowData, iEph, regKey);
-    sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelContinue, regKey, 0);
-    VdbeCoverage(v);
-  }else{
-    labelContinue = sqlite3VdbeAddOp3(v, OP_RowSetRead, regRowSet, labelBreak,
-                             regOldRowid);
-    VdbeCoverage(v);
-    sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue, regOldRowid);
-    VdbeCoverage(v);
   }
 
-  /* If the record number will change, set register regNewRowid to
-  ** contain the new value. If the record number is not being modified,
+  /* If the rowid value will change, set register regNewRowid to
+  ** contain the new value. If the rowid is not being modified,
   ** then regNewRowid is the same register as regOldRowid, which is
   ** already populated.  */
   assert( chngKey || pTrigger || hasFK || regOldRowid==regNewRowid );
@@ -579,6 +611,12 @@ void sqlite3Update(
         testcase( i==31 );
         testcase( i==32 );
         sqlite3ExprCodeGetColumnToReg(pParse, pTab, i, iDataCur, regNew+i);
+        if( tmask & TRIGGER_BEFORE ){
+          /* This value will be recomputed in After-BEFORE-trigger-reload-loop
+          ** below, so make sure that it is not cached and reused.
+          ** Ticket d85fffd6ffe856092ed8daefa811b1e399706b28. */
+          sqlite3ExprCacheRemove(pParse, regNew+i, 1);
+        }
       }else{
         sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i);
       }
@@ -607,10 +645,14 @@ void sqlite3Update(
       VdbeCoverage(v);
     }
 
-    /* If it did not delete it, the row-trigger may still have modified 
+    /* After-BEFORE-trigger-reload-loop:
+    ** If it did not delete it, the BEFORE trigger may still have modified 
     ** some of the columns of the row being updated. Load the values for 
-    ** all columns not modified by the update statement into their 
-    ** registers in case this has happened.
+    ** all columns not modified by the update statement into their registers
+    ** in case this has happened. Only unmodified columns are reloaded.
+    ** The values computed for modified columns use the values before the
+    ** BEFORE trigger runs.  See test case trigger1-18.0 (added 2018-04-26)
+    ** for an example.
     */
     for(i=0; i<pTab->nCol; i++){
       if( aXRef[i]<0 && i!=pTab->iPKey ){
@@ -626,7 +668,7 @@ void sqlite3Update(
     assert( regOldRowid>0 );
     sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur,
         regNewRowid, regOldRowid, chngKey, onError, labelContinue, &bReplace,
-        aXRef);
+        aXRef, 0);
 
     /* Do FK constraint checks. */
     if( hasFK ){
@@ -696,7 +738,7 @@ void sqlite3Update(
 
   /* Increment the row counter 
   */
-  if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab){
+  if( regRowCount ){
     sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1);
   }
 
@@ -723,16 +765,15 @@ void sqlite3Update(
   ** maximum rowid counter values recorded while inserting into
   ** autoincrement tables.
   */
-  if( pParse->nested==0 && pParse->pTriggerTab==0 ){
+  if( pParse->nested==0 && pParse->pTriggerTab==0 && pUpsert==0 ){
     sqlite3AutoincrementEnd(pParse);
   }
 
   /*
-  ** Return the number of rows that were changed. If this routine is 
-  ** generating code because of a call to sqlite3NestedParse(), do not
-  ** invoke the callback function.
+  ** Return the number of rows that were changed, if we are tracking
+  ** that information.
   */
-  if( (db->flags&SQLITE_CountRows) && !pParse->pTriggerTab && !pParse->nested ){
+  if( regRowCount ){
     sqlite3VdbeAddOp2(v, OP_ResultRow, regRowCount, 1);
     sqlite3VdbeSetNumCols(v, 1);
     sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows updated", SQLITE_STATIC);
@@ -853,15 +894,12 @@ static void updateVirtualTable(
 
   if( bOnePass ){
     /* If using the onepass strategy, no-op out the OP_OpenEphemeral coded
-    ** above. Also, if this is a top-level parse (not a trigger), clear the
-    ** multi-write flag so that the VM does not open a statement journal */
+    ** above. */
     sqlite3VdbeChangeToNoop(v, addr);
-    if( sqlite3IsToplevel(pParse) ){
-      pParse->isMultiWrite = 0;
-    }
   }else{
     /* Create a record from the argument register contents and insert it into
     ** the ephemeral table. */
+    sqlite3MultiWrite(pParse);
     sqlite3VdbeAddOp3(v, OP_MakeRecord, regArg, nArg, regRec);
 #ifdef SQLITE_DEBUG
     /* Signal an assert() within OP_MakeRecord that it is allowed to

src/upsert.c

@@ -0,0 +1,249 @@
+/*
+** 2018-04-12
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file contains code to implement various aspects of UPSERT
+** processing and handling of the Upsert object.
+*/
+#include "sqliteInt.h"
+
+#ifndef SQLITE_OMIT_UPSERT
+/*
+** Free a list of Upsert objects
+*/
+void sqlite3UpsertDelete(sqlite3 *db, Upsert *p){
+  if( p ){
+    sqlite3ExprListDelete(db, p->pUpsertTarget);
+    sqlite3ExprDelete(db, p->pUpsertTargetWhere);
+    sqlite3ExprListDelete(db, p->pUpsertSet);
+    sqlite3ExprDelete(db, p->pUpsertWhere);
+    sqlite3DbFree(db, p);
+  }
+}
+
+/*
+** Duplicate an Upsert object.
+*/
+Upsert *sqlite3UpsertDup(sqlite3 *db, Upsert *p){
+  if( p==0 ) return 0;
+  return sqlite3UpsertNew(db,
+           sqlite3ExprListDup(db, p->pUpsertTarget, 0),
+           sqlite3ExprDup(db, p->pUpsertTargetWhere, 0),
+           sqlite3ExprListDup(db, p->pUpsertSet, 0),
+           sqlite3ExprDup(db, p->pUpsertWhere, 0)
+         );
+}
+
+/*
+** Create a new Upsert object.
+*/
+Upsert *sqlite3UpsertNew(
+  sqlite3 *db,           /* Determines which memory allocator to use */
+  ExprList *pTarget,     /* Target argument to ON CONFLICT, or NULL */
+  Expr *pTargetWhere,    /* Optional WHERE clause on the target */
+  ExprList *pSet,        /* UPDATE columns, or NULL for a DO NOTHING */
+  Expr *pWhere           /* WHERE clause for the ON CONFLICT UPDATE */
+){
+  Upsert *pNew;
+  pNew = sqlite3DbMallocRaw(db, sizeof(Upsert));
+  if( pNew==0 ){
+    sqlite3ExprListDelete(db, pTarget);
+    sqlite3ExprDelete(db, pTargetWhere);
+    sqlite3ExprListDelete(db, pSet);
+    sqlite3ExprDelete(db, pWhere);
+    return 0;
+  }else{
+    pNew->pUpsertTarget = pTarget;
+    pNew->pUpsertTargetWhere = pTargetWhere;
+    pNew->pUpsertSet = pSet;
+    pNew->pUpsertWhere = pWhere;
+    pNew->pUpsertIdx = 0;
+  }
+  return pNew;
+}
+
+/*
+** Analyze the ON CONFLICT clause described by pUpsert.  Resolve all
+** symbols in the conflict-target.
+**
+** Return SQLITE_OK if everything works, or an error code is something
+** is wrong.
+*/
+int sqlite3UpsertAnalyzeTarget(
+  Parse *pParse,     /* The parsing context */
+  SrcList *pTabList, /* Table into which we are inserting */
+  Upsert *pUpsert    /* The ON CONFLICT clauses */
+){
+  Table *pTab;            /* That table into which we are inserting */
+  int rc;                 /* Result code */
+  int iCursor;            /* Cursor used by pTab */
+  Index *pIdx;            /* One of the indexes of pTab */
+  ExprList *pTarget;      /* The conflict-target clause */
+  Expr *pTerm;            /* One term of the conflict-target clause */
+  NameContext sNC;        /* Context for resolving symbolic names */
+  Expr sCol[2];           /* Index column converted into an Expr */
+
+  assert( pTabList->nSrc==1 );
+  assert( pTabList->a[0].pTab!=0 );
+  assert( pUpsert!=0 );
+  assert( pUpsert->pUpsertTarget!=0 );
+
+  /* Resolve all symbolic names in the conflict-target clause, which
+  ** includes both the list of columns and the optional partial-index
+  ** WHERE clause.
+  */
+  memset(&sNC, 0, sizeof(sNC));
+  sNC.pParse = pParse;
+  sNC.pSrcList = pTabList;
+  rc = sqlite3ResolveExprListNames(&sNC, pUpsert->pUpsertTarget);
+  if( rc ) return rc;
+  rc = sqlite3ResolveExprNames(&sNC, pUpsert->pUpsertTargetWhere);
+  if( rc ) return rc;
+
+  /* Check to see if the conflict target matches the rowid. */  
+  pTab = pTabList->a[0].pTab;
+  pTarget = pUpsert->pUpsertTarget;
+  iCursor = pTabList->a[0].iCursor;
+  if( HasRowid(pTab) 
+   && pTarget->nExpr==1
+   && (pTerm = pTarget->a[0].pExpr)->op==TK_COLUMN
+   && pTerm->iColumn==XN_ROWID
+  ){
+    /* The conflict-target is the rowid of the primary table */
+    assert( pUpsert->pUpsertIdx==0 );
+    return SQLITE_OK;
+  }
+
+  /* Initialize sCol[0..1] to be an expression parse tree for a
+  ** single column of an index.  The sCol[0] node will be the TK_COLLATE
+  ** operator and sCol[1] will be the TK_COLUMN operator.  Code below
+  ** will populate the specific collation and column number values
+  ** prior to comparing against the conflict-target expression.
+  */
+  memset(sCol, 0, sizeof(sCol));
+  sCol[0].op = TK_COLLATE;
+  sCol[0].pLeft = &sCol[1];
+  sCol[1].op = TK_COLUMN;
+  sCol[1].iTable = pTabList->a[0].iCursor;
+
+  /* Check for matches against other indexes */
+  for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
+    int ii, jj, nn;
+    if( !IsUniqueIndex(pIdx) ) continue;
+    if( pTarget->nExpr!=pIdx->nKeyCol ) continue;
+    if( pIdx->pPartIdxWhere ){
+      if( pUpsert->pUpsertTargetWhere==0 ) continue;
+      if( sqlite3ExprCompare(pParse, pUpsert->pUpsertTargetWhere,
+                             pIdx->pPartIdxWhere, iCursor)!=0 ){
+        continue;
+      }
+    }
+    nn = pIdx->nKeyCol;
+    for(ii=0; ii<nn; ii++){
+      Expr *pExpr;
+      sCol[0].u.zToken = (char*)pIdx->azColl[ii];
+      if( pIdx->aiColumn[ii]==XN_EXPR ){
+        assert( pIdx->aColExpr!=0 );
+        assert( pIdx->aColExpr->nExpr>ii );
+        pExpr = pIdx->aColExpr->a[ii].pExpr;
+        if( pExpr->op!=TK_COLLATE ){
+          sCol[0].pLeft = pExpr;
+          pExpr = &sCol[0];
+        }
+      }else{
+        sCol[0].pLeft = &sCol[1];
+        sCol[1].iColumn = pIdx->aiColumn[ii];
+        pExpr = &sCol[0];
+      }
+      for(jj=0; jj<nn; jj++){
+        if( sqlite3ExprCompare(pParse, pTarget->a[jj].pExpr, pExpr,iCursor)<2 ){
+          break;  /* Column ii of the index matches column jj of target */
+        }
+      }
+      if( jj>=nn ){
+        /* The target contains no match for column jj of the index */
+        break;
+      }
+    }
+    if( ii<nn ){
+      /* Column ii of the index did not match any term of the conflict target.
+      ** Continue the search with the next index. */
+      continue;
+    }
+    pUpsert->pUpsertIdx = pIdx;
+    return SQLITE_OK;
+  }
+  sqlite3ErrorMsg(pParse, "ON CONFLICT clause does not match any "
+                          "PRIMARY KEY or UNIQUE constraint");
+  return SQLITE_ERROR;
+}
+
+/*
+** Generate bytecode that does an UPDATE as part of an upsert.
+**
+** If pIdx is NULL, then the UNIQUE constraint that failed was the IPK.
+** In this case parameter iCur is a cursor open on the table b-tree that
+** currently points to the conflicting table row. Otherwise, if pIdx
+** is not NULL, then pIdx is the constraint that failed and iCur is a
+** cursor points to the conflicting row.
+*/
+void sqlite3UpsertDoUpdate(
+  Parse *pParse,        /* The parsing and code-generating context */
+  Upsert *pUpsert,      /* The ON CONFLICT clause for the upsert */
+  Table *pTab,          /* The table being updated */
+  Index *pIdx,          /* The UNIQUE constraint that failed */
+  int iCur              /* Cursor for pIdx (or pTab if pIdx==NULL) */
+){
+  Vdbe *v = pParse->pVdbe;
+  sqlite3 *db = pParse->db;
+  SrcList *pSrc;            /* FROM clause for the UPDATE */
+  int iDataCur = pUpsert->iDataCur;
+
+  assert( v!=0 );
+  VdbeNoopComment((v, "Begin DO UPDATE of UPSERT"));
+  if( pIdx && iCur!=iDataCur ){
+    if( HasRowid(pTab) ){
+      int regRowid = sqlite3GetTempReg(pParse);
+      sqlite3VdbeAddOp2(v, OP_IdxRowid, iCur, regRowid);
+      sqlite3VdbeAddOp3(v, OP_SeekRowid, iDataCur, 0, regRowid);
+      VdbeCoverage(v);
+      sqlite3ReleaseTempReg(pParse, regRowid);
+    }else{
+      Index *pPk = sqlite3PrimaryKeyIndex(pTab);
+      int nPk = pPk->nKeyCol;
+      int iPk = pParse->nMem+1;
+      int i;
+      pParse->nMem += nPk;
+      for(i=0; i<nPk; i++){
+        int k;
+        assert( pPk->aiColumn[i]>=0 );
+        k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[i]);
+        sqlite3VdbeAddOp3(v, OP_Column, iCur, k, iPk+i);
+        VdbeComment((v, "%s.%s", pIdx->zName,
+                    pTab->aCol[pPk->aiColumn[i]].zName));
+      }
+      i = sqlite3VdbeAddOp4Int(v, OP_Found, iDataCur, 0, iPk, nPk);
+      VdbeCoverage(v);
+      sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CORRUPT, OE_Abort, 0, 
+            "corrupt database", P4_STATIC);
+      sqlite3VdbeJumpHere(v, i);
+    }
+  }
+  /* pUpsert does not own pUpsertSrc - the outer INSERT statement does.  So
+  ** we have to make a copy before passing it down into sqlite3Update() */
+  pSrc = sqlite3SrcListDup(db, pUpsert->pUpsertSrc, 0);
+  sqlite3Update(pParse, pSrc, pUpsert->pUpsertSet,
+      pUpsert->pUpsertWhere, OE_Abort, 0, 0, pUpsert);
+  pUpsert->pUpsertSet = 0;    /* Will have been deleted by sqlite3Update() */
+  pUpsert->pUpsertWhere = 0;  /* Will have been deleted by sqlite3Update() */
+  VdbeNoopComment((v, "End DO UPDATE of UPSERT"));
+}
+
+#endif /* SQLITE_OMIT_UPSERT */

src/vdbe.h

@@ -220,6 +220,9 @@ void sqlite3VdbeClearObject(sqlite3*,Vdbe*);
 void sqlite3VdbeMakeReady(Vdbe*,Parse*);
 int sqlite3VdbeFinalize(Vdbe*);
 void sqlite3VdbeResolveLabel(Vdbe*, int);
+#ifdef SQLITE_COVERAGE_TEST
+  int sqlite3VdbeLabelHasBeenResolved(Vdbe*,int);
+#endif
 int sqlite3VdbeCurrentAddr(Vdbe*);
 #ifdef SQLITE_DEBUG
   int sqlite3VdbeAssertMayAbort(Vdbe *, int);

src/vdbeaux.c

@@ -392,10 +392,29 @@ void sqlite3VdbeResolveLabel(Vdbe *v, int x){
   assert( j<p->nLabel );
   assert( j>=0 );
   if( p->aLabel ){
+#ifdef SQLITE_DEBUG
+    if( p->db->flags & SQLITE_VdbeAddopTrace ){
+      printf("RESOLVE LABEL %d to %d\n", x, v->nOp);
+    }
+#endif
+    assert( p->aLabel[j]==(-1) ); /* Labels may only be resolved once */
     p->aLabel[j] = v->nOp;
   }
 }
 
+#ifdef SQLITE_COVERAGE_TEST
+/*
+** Return TRUE if and only if the label x has already been resolved.
+** Return FALSE (zero) if label x is still unresolved.
+**
+** This routine is only used inside of testcase() macros, and so it
+** only exists when measuring test coverage.
+*/
+int sqlite3VdbeLabelHasBeenResolved(Vdbe *v, int x){
+  return v->pParse->aLabel && v->pParse->aLabel[ADDR(x)]>=0;
+}
+#endif /* SQLITE_COVERAGE_TEST */
+
 /*
 ** Mark the VDBE as one that can only be run one time.
 */
@@ -1626,6 +1645,9 @@ void sqlite3VdbeFrameDelete(VdbeFrame *p){
 ** p->explain==2, only OP_Explain instructions are listed and these
 ** are shown in a different format.  p->explain==2 is used to implement
 ** EXPLAIN QUERY PLAN.
+** 2018-04-24:  In p->explain==2 mode, the OP_Init opcodes of triggers
+** are also shown, so that the boundaries between the main program and
+** each trigger are clear.
 **
 ** When p->explain==1, first the main program is listed, then each of
 ** the trigger subprograms are listed one by one.
@@ -1688,7 +1710,7 @@ int sqlite3VdbeList(
     }
   }
 
-  do{
+  while(1){  /* Loop exits via break */
     i = p->pc++;
     if( i>=nRow ){
       p->rc = SQLITE_OK;
@@ -1734,7 +1756,10 @@ int sqlite3VdbeList(
         nRow += pOp->p4.pProgram->nOp;
       }
     }
-  }while( p->explain==2 && pOp->opcode!=OP_Explain );
+    if( p->explain<2 ) break;
+    if( pOp->opcode==OP_Explain ) break;
+    if( pOp->opcode==OP_Init && p->pc>1 ) break;
+  }
 
   if( rc==SQLITE_OK ){
     if( db->u1.isInterrupted ){

src/vdbemem.c

@@ -1478,12 +1478,16 @@ static int valueFromExpr(
                          0, SQLITE_DYNAMIC);
   }
 #endif
-
 #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
   else if( op==TK_FUNCTION && pCtx!=0 ){
     rc = valueFromFunction(db, pExpr, enc, affinity, &pVal, pCtx);
   }
 #endif
+  else if( op==TK_TRUEFALSE ){
+     pVal = valueNew(db, pCtx);
+     pVal->flags = MEM_Int;
+     pVal->u.i = pExpr->u.zToken[4]==0;
+  }
 
   *ppVal = pVal;
   return rc;

src/where.c

@@ -2420,15 +2420,12 @@ static int whereLoopAddBtreeIndex(
     ** to mix with a lower range bound from some other source */
     if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue;
 
-    /* Do not allow IS constraints from the WHERE clause to be used by the
+    /* Do not allow constraints from the WHERE clause to be used by the
     ** right table of a LEFT JOIN.  Only constraints in the ON clause are
     ** allowed */
     if( (pSrc->fg.jointype & JT_LEFT)!=0
      && !ExprHasProperty(pTerm->pExpr, EP_FromJoin)
-     && (eOp & (WO_IS|WO_ISNULL))!=0
     ){
-      testcase( eOp & WO_IS );
-      testcase( eOp & WO_ISNULL );
       continue;
     }
 

src/wherecode.c

@@ -1215,6 +1215,9 @@ Bitmask sqlite3WhereCodeOneLoopStart(
   ** initialize a memory cell that records if this table matches any
   ** row of the left table of the join.
   */
+  assert( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)
+       || pLevel->iFrom>0 || (pTabItem[0].fg.jointype & JT_LEFT)==0
+  );
   if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){
     pLevel->iLeftJoin = ++pParse->nMem;
     sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin);
@@ -1748,9 +1751,16 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     /* If pIdx is an index on one or more expressions, then look through
     ** all the expressions in pWInfo and try to transform matching expressions
     ** into reference to index columns.
+    **
+    ** Do not do this for the RHS of a LEFT JOIN. This is because the 
+    ** expression may be evaluated after OP_NullRow has been executed on
+    ** the cursor. In this case it is important to do the full evaluation,
+    ** as the result of the expression may not be NULL, even if all table
+    ** column values are.  https://www.sqlite.org/src/info/7fa8049685b50b5a
     */
-    whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo);
-
+    if( pLevel->iLeftJoin==0 ){
+      whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo);
+    }
 
     /* Record the instruction used to terminate the loop. */
     if( pLoop->wsFlags & WHERE_ONEROW ){
@@ -1906,7 +1916,6 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       for(iTerm=0; iTerm<pWC->nTerm; iTerm++){
         Expr *pExpr = pWC->a[iTerm].pExpr;
         if( &pWC->a[iTerm] == pTerm ) continue;
-        if( ExprHasProperty(pExpr, EP_FromJoin) ) continue;
         testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL );
         testcase( pWC->a[iTerm].wtFlags & TERM_CODED );
         if( (pWC->a[iTerm].wtFlags & (TERM_VIRTUAL|TERM_CODED))!=0 ) continue;
@@ -1931,7 +1940,10 @@ Bitmask sqlite3WhereCodeOneLoopStart(
         WhereInfo *pSubWInfo;           /* Info for single OR-term scan */
         Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */
         int jmp1 = 0;                   /* Address of jump operation */
-        if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){
+        assert( (pTabItem[0].fg.jointype & JT_LEFT)==0 
+             || ExprHasProperty(pOrExpr, EP_FromJoin) 
+        );
+        if( pAndExpr ){
           pAndExpr->pLeft = pOrExpr;
           pOrExpr = pAndExpr;
         }
@@ -2114,7 +2126,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       }
       pE = pTerm->pExpr;
       assert( pE!=0 );
-      if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){
+      if( (pTabItem->fg.jointype&JT_LEFT) && !ExprHasProperty(pE,EP_FromJoin) ){
         continue;
       }
       

src/whereexpr.c

@@ -818,7 +818,6 @@ static void exprAnalyzeOrTerm(
       }else{
         sqlite3ExprListDelete(db, pList);
       }
-      pTerm->eOperator = WO_NOOP;  /* case 1 trumps case 3 */
     }
   }
 }

test/autoinc.test

@@ -675,6 +675,13 @@ do_execsql_test autoinc-10.1 {
   SELECT * FROM sqlite_sequence;
 } {t10a 888 t10b 888}
 
-
+# 2018-04-21 autoincrement does not cause problems for upsert
+#
+do_execsql_test autoinc-11.1 {
+  CREATE TABLE t11(a INTEGER PRIMARY KEY AUTOINCREMENT,b UNIQUE);
+  INSERT INTO t11(a,b) VALUES(2,3),(5,6),(4,3),(1,2)
+    ON CONFLICT(b) DO UPDATE SET a=a+1000;
+  SELECT seq FROM sqlite_sequence WHERE name='t11';
+} {5}
 
 finish_test

test/bestindex1.test

@@ -59,7 +59,6 @@ do_eqp_test 1.2 {
   SELECT * FROM x1 WHERE a IN ('abc', 'def');
 } {
   0 0 0 {SCAN TABLE x1 VIRTUAL TABLE INDEX 555:eq!}
-  0 0 0 {EXECUTE LIST SUBQUERY 1}
 }
 
 #-------------------------------------------------------------------------
@@ -146,17 +145,14 @@ foreach {tn mode} {
 
   set plan(use) {
     0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x WHERE a='%1%'}
-    0 0 0 {EXECUTE LIST SUBQUERY 1}
     0 0 0 {USE TEMP B-TREE FOR ORDER BY}
   }
   set plan(omit) {
     0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x WHERE a='%1%'}
-    0 0 0 {EXECUTE LIST SUBQUERY 1}
     0 0 0 {USE TEMP B-TREE FOR ORDER BY}
   }
   set plan(use2) {
     0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x}
-    0 0 0 {EXECUTE LIST SUBQUERY 1}
     0 0 0 {USE TEMP B-TREE FOR ORDER BY}
   }
 

test/csv01.test

@@ -140,5 +140,12 @@ do_catchsql_test 4.2 {
       );
 } {1 {vtable constructor failed: t5}}
 
+# 2018-04-24
+# Memory leak reported on the sqlite-users mailing list by Ralf Junker.
+#
+do_catchsql_test 4.3 {
+  CREATE VIRTUAL TABLE IF NOT EXISTS temp.t1
+  USING csv(filename='FileDoesNotExist.csv');
+} {1 {cannot open 'FileDoesNotExist.csv' for reading}}
 
 finish_test

test/delete.test

@@ -403,5 +403,17 @@ do_execsql_test delete-10.2 {
   SELECT * FROM t1 WHERE a='1' AND b='2';
 }
 
+do_execsql_test delete-11.0 {
+  CREATE TABLE t11(a INTEGER PRIMARY KEY, b INT);
+  WITH RECURSIVE cnt(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM cnt WHERE x<20)
+    INSERT INTO t11(a,b) SELECT x, (x*17)%100 FROM cnt;
+  SELECT * FROM t11;
+} {1 17 2 34 3 51 4 68 5 85 6 2 7 19 8 36 9 53 10 70 11 87 12 4 13 21 14 38 15 55 16 72 17 89 18 6 19 23 20 40}
+do_execsql_test delete-11.1 {
+  DELETE FROM t11 AS xyz
+   WHERE EXISTS(SELECT 1 FROM t11 WHERE t11.a>xyz.a AND t11.b<=xyz.b);
+  SELECT * FROM t11;
+} {6 2 12 4 18 6 19 23 20 40}
+
 
 finish_test

test/fuzz_malloc.test

@@ -59,7 +59,12 @@ proc do_fuzzy_malloc_test {testname args} {
     # puts fuzyy-sql=\[$::sql\]; flush stdout
     foreach {rc res} [catchsql "$::sql"] {}
     if {$rc==0} {
+      set nErr1 [set_test_counter errors]
       do_malloc_test $testname-$ii -sqlbody $::sql -sqlprep $::prep
+      if {[set_test_counter errors]>$nErr1} {
+        puts "Previous fuzzy-sql=\[$::sql\]"
+        flush stdout
+      }
     } else {
       incr ii -1
     }

test/istrue.test

@@ -143,4 +143,19 @@ foreach {tn val} [list 1 NaN 2 -NaN 3 NaN0 4 -NaN0 5 Inf 6 -Inf] {
   } {0}
 }
 
+do_execsql_test istrue-700 {
+  CREATE TABLE t7(
+    a INTEGER PRIMARY KEY,
+    b BOOLEAN DEFAULT false,
+    c BOOLEAN DEFAULT true
+  );
+  INSERT INTO t7(a) VALUES(1);
+  INSERT INTO t7(a,b,c) VALUES(2,true,false);
+  ALTER TABLE t7 ADD COLUMN d BOOLEAN DEFAULT false;
+  ALTER TABLE t7 ADD COLUMN e BOOLEAN DEFAULT true;
+  INSERT INTO t7(a,b,c) VALUES(3,true,false);
+  INSERT INTO t7 VALUES(4,false,true,true,false);
+  SELECT *,'x' FROM t7 ORDER BY a;
+} {1 0 1 0 1 x 2 1 0 0 1 x 3 1 0 0 1 x 4 0 1 1 0 x}
+
 finish_test

test/join5.test

@@ -212,4 +212,82 @@ do_execsql_test 5.5 {
   SELECT * FROM ( SELECT * FROM y1 ) LEFT JOIN y2 ON x
 } {0 0 1 {}}
 
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 6.1 {
+  CREATE TABLE t1(x); 
+  INSERT INTO t1 VALUES(1);
+
+  CREATE TABLE t2(y INTEGER PRIMARY KEY,a,b);
+  INSERT INTO t2 VALUES(1,2,3);
+  CREATE INDEX t2a ON t2(a); 
+  CREATE INDEX t2b ON t2(b); 
+}
+
+do_execsql_test 6.2 {
+  SELECT * FROM t1 LEFT JOIN t2 ON a=2 OR b=3 WHERE y IS NULL;
+} {}
+
+do_execsql_test 6.3.1 {
+  CREATE TABLE t3(x);
+  INSERT INTO t3 VALUES(1);
+  CREATE TABLE t4(y, z);
+  SELECT ifnull(z, '!!!') FROM t3 LEFT JOIN t4 ON (x=y);
+} {!!!}
+
+do_execsql_test 6.3.2 {
+  CREATE INDEX t4i ON t4(y, ifnull(z, '!!!'));
+  SELECT ifnull(z, '!!!') FROM t3 LEFT JOIN t4 ON (x=y);
+} {!!!}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 7.0 {
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES(1);
+}
+
+do_execsql_test 7.1 {
+  CREATE TABLE t2(x, y, z);
+  CREATE INDEX t2xy ON t2(x, y);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50000
+  )
+  INSERT INTO t2 SELECT i/10, i, NULL FROM s;
+  ANALYZE;
+}
+
+do_eqp_test 7.2 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (
+    t2.x = t1.x AND (t2.y=? OR (t2.y=? AND t2.z IS NOT NULL))
+  );
+} {
+  0 0 0 {SCAN TABLE t1} 
+  0 1 1 {SEARCH TABLE t2 USING INDEX t2xy (x=? AND y=?)} 
+  0 1 1 {SEARCH TABLE t2 USING INDEX t2xy (x=? AND y=?)}
+}
+
+do_execsql_test 7.3 {
+  CREATE TABLE t3(x);
+
+  CREATE TABLE t4(x, y, z);
+  CREATE INDEX t4xy ON t4(x, y);
+  CREATE INDEX t4xz ON t4(x, z);
+
+  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50000)
+  INSERT INTO t4 SELECT i/10, i, i FROM s;
+
+  ANALYZE;
+}
+
+do_eqp_test 7.4 {
+  SELECT * FROM t3 LEFT JOIN t4 ON (t4.x = t3.x) WHERE (t4.y = ? OR t4.z = ?);
+} {
+  0 0 0 {SCAN TABLE t3} 
+  0 1 1 {SEARCH TABLE t4 USING INDEX t4xz (x=?)}
+} 
+
 finish_test
+

test/trigger1.test

@@ -728,4 +728,44 @@ do_execsql_test trigger1-17.0 {
   PRAGMA integrity_check;
 } {ok}
 
+# 2018-04-26
+# When a BEFORE UPDATE trigger changes a column value in a row being
+# updated, and that column value is used by the UPDATE to change other
+# column, the value used to compute the update is from before the trigger.
+# In the example that follows, the value of "b" in "c=b" is 2 (the value
+# prior to running the BEFORE UPDATE trigger) not 1000.
+#
+do_execsql_test trigger1-18.0 {
+  CREATE TABLE t18(a PRIMARY KEY,b,c);
+  INSERT INTO t18(a,b,c) VALUES(1,2,3);
+  CREATE TRIGGER t18r1 BEFORE UPDATE ON t18 BEGIN
+    UPDATE t18 SET b=1000 WHERE a=old.a;
+  END;
+  UPDATE t18 SET c=b WHERE a=1;
+  SELECT * FROM t18;
+} {1 1000 2}  ;# Not: 1 1000 1000 
+do_execsql_test trigger1-18.1 {
+  DELETE FROM t18;
+  INSERT INTO t18(a,b,c) VALUES(1,2,3);
+  UPDATE t18 SET c=b, b=b+1 WHERE a=1;
+  SELECT * FROM t18;
+} {1 3 2}     ;# Not: 1 1001 1000
+
+# 2018-04-26 ticket [https://www.sqlite.org/src/tktview/d85fffd6ffe856092e]
+# VDBE Program uses an expired value.
+#
+do_execsql_test trigger1-19.0 {
+  CREATE TABLE t19(a INT PRIMARY KEY, b, c)WITHOUT ROWID;
+  INSERT INTO t19(a,b,c) VALUES(1,2,3);
+  CREATE TRIGGER t19r3 BEFORE UPDATE ON t19 BEGIN SELECT new.b; END;
+  UPDATE t19 SET c=b WHERE a=1;
+  SELECT * FROM t19;
+} {1 2 2}
+do_execsql_test trigger1-19.1 {
+  DELETE FROM t19;
+  INSERT INTO t19(a,b,c) VALUES(1,2,3);
+  UPDATE t19 SET c=CASE WHEN b=2 THEN b ELSE b+99 END WHERE a=1;
+  SELECT * FROM t19;
+} {1 2 2}
+
 finish_test

test/update.test

@@ -508,6 +508,18 @@ ifcapable subquery {
       SELECT a,e FROM t1;
     }
   } {1 15 2 8}
+  do_test update-11.3 {
+    execsql {
+      UPDATE t1 AS xyz SET e=e+1 WHERE xyz.a IN (SELECT a FROM t1);
+      SELECT a,e FROM t1;
+    }
+  } {1 16 2 9}
+  do_test update-11.4 {
+    execsql {
+      UPDATE t1 AS xyz SET e=e+1 WHERE EXISTS(SELECT 1 FROM t1 WHERE t1.a<xyz.a);
+      SELECT a,e FROM t1;
+    }
+  } {1 16 2 10}
 }
 
 integrity_check update-12.1
@@ -618,5 +630,15 @@ do_execsql_test update-15.1 {
   SELECT a,b,c,'|' FROM t15 ORDER BY a;
 } {5 zyx y5 | 10 abc y10 | 15 wvu y15 | 20 def y20 | 25 tsr y25 | 30 ghi y30 | 35 qpo y35 |}
 
+# Unreleased bug in UPDATE caused by the UPSERT changes.
+# Found by OSSFuzz as soon as the UPSERT changes landed on trunk.
+# Never released into the wild.  2018-04-19.
+#
+do_execsql_test update-16.1 {
+  CREATE TABLE t16(a INTEGER PRIMARY KEY ON CONFLICT REPLACE, b UNIQUE);
+  INSERT INTO t16(a,b) VALUES(1,2),(3,4),(5,6);
+  UPDATE t16 SET a=a;
+  SELECT * FROM t16 ORDER BY +a;
+} {1 2 3 4 5 6}
 
 finish_test

test/upsert1.test

@@ -0,0 +1,115 @@
+# 2018-04-12
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test cases for UPSERT
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix zipfile
+
+do_execsql_test upsert1-100 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c DEFAULT 0);
+  CREATE UNIQUE INDEX t1x1 ON t1(b);
+  INSERT INTO t1(a,b) VALUES(1,2) ON CONFLICT DO NOTHING;
+  INSERT INTO t1(a,b) VALUES(1,99),(99,2) ON CONFLICT DO NOTHING;
+  SELECT * FROM t1;
+} {1 2 0}
+do_execsql_test upsert1-101 {
+  DELETE FROM t1;
+  INSERT INTO t1(a,b) VALUES(2,3) ON CONFLICT(a) DO NOTHING;
+  INSERT INTO t1(a,b) VALUES(2,99) ON CONFLICT(a) DO NOTHING;
+  SELECT * FROM t1;
+} {2 3 0}
+do_execsql_test upsert1-102 {
+  DELETE FROM t1;
+  INSERT INTO t1(a,b) VALUES(3,4) ON CONFLICT(b) DO NOTHING;
+  INSERT INTO t1(a,b) VALUES(99,4) ON CONFLICT(b) DO NOTHING;
+  SELECT * FROM t1;
+} {3 4 0}
+do_catchsql_test upsert1-110 {
+  INSERT INTO t1(a,b) VALUES(5,6) ON CONFLICT(x) DO NOTHING;
+  SELECT * FROM t1;
+} {1 {no such column: x}}
+do_catchsql_test upsert1-120 {
+  INSERT INTO t1(a,b) VALUES(5,6) ON CONFLICT(c) DO NOTHING;
+  SELECT * FROM t1;
+} {1 {ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint}}
+breakpoint
+do_catchsql_test upsert1-130 {
+  INSERT INTO t1(a,b) VALUES(5,6) ON CONFLICT(b COLLATE nocase) DO NOTHING;
+  SELECT * FROM t1;
+} {1 {ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint}}
+do_execsql_test upsert1-140 {
+  DELETE FROM t1;
+  INSERT INTO t1(a,b) VALUES(5,6) ON CONFLICT(b COLLATE binary) DO NOTHING;
+  SELECT * FROM t1;
+} {5 6 0}
+
+do_catchsql_test upsert1-200 {
+  DROP TABLE t1;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b INT, c DEFAULT 0);
+  CREATE UNIQUE INDEX t1x1 ON t1(a+b);
+  INSERT INTO t1(a,b) VALUES(7,8) ON CONFLICT(a+b) DO NOTHING;
+  INSERT INTO t1(a,b) VALUES(8,7),(9,6) ON CONFLICT(a+b) DO NOTHING;
+  SELECT * FROM t1;
+} {0 {7 8 0}}
+do_catchsql_test upsert1-201 {
+  INSERT INTO t1(a,b) VALUES(8,7),(9,6) ON CONFLICT(a) DO NOTHING;
+} {1 {UNIQUE constraint failed: index 't1x1'}}
+do_catchsql_test upsert1-210 {
+  DELETE FROM t1;
+  INSERT INTO t1(a,b) VALUES(9,10) ON CONFLICT(a+(+b)) DO NOTHING;
+  SELECT * FROM t1;
+} {1 {ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint}}
+
+do_catchsql_test upsert1-300 {
+  DROP INDEX t1x1;
+  DELETE FROM t1;
+  CREATE UNIQUE INDEX t1x1 ON t1(b) WHERE b>10;
+  INSERT INTO t1(a,b) VALUES(1,2),(3,2) ON CONFLICT(b) DO NOTHING;
+  SELECT * FROM t1;
+} {1 {ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint}}
+do_catchsql_test upsert1-310 {
+  DELETE FROM t1;
+  INSERT INTO t1(a,b) VALUES(1,2),(3,2) ON CONFLICT(b) WHERE b!=10 DO NOTHING;
+  SELECT * FROM t1;
+} {1 {ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint}}
+do_execsql_test upsert1-320 {
+  DELETE FROM t1;
+  INSERT INTO t1(a,b) VALUES(1,2),(3,2),(4,20),(5,20)
+         ON CONFLICT(b) WHERE b>10 DO NOTHING;
+  SELECT *, 'x' FROM t1 ORDER BY b, a;
+} {1 2 0 x 3 2 0 x 4 20 0 x}
+
+# Upsert works with count_changes=on;
+do_execsql_test upsert1-400 {
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t2(a TEXT UNIQUE, b INT DEFAULT 1);
+  INSERT INTO t2(a) VALUES('one'),('two'),('three');
+  PRAGMA count_changes=ON;
+  INSERT INTO t2(a) VALUES('one'),('one'),('three'),('four')
+      ON CONFLICT(a) DO UPDATE SET b=b+1;
+} {1}
+do_execsql_test upsert1-410 {
+  PRAGMA count_changes=OFF;
+  SELECT a, b FROM t2 ORDER BY a;
+} {four 1 one 3 three 2 two 1}
+
+# Problem found by AFL prior to any release
+do_execsql_test upsert1-500 {
+  DROP TABLE t1;
+  CREATE TABLE t1(x INTEGER PRIMARY KEY, y INT UNIQUE);
+  INSERT INTO t1(x,y) SELECT 1,2 WHERE true
+    ON CONFLICT(x) DO UPDATE SET y=max(t1.y,excluded.y) AND true;
+  SELECT * FROM t1;
+} {1 2}
+
+finish_test

test/upsert2.test

@@ -0,0 +1,170 @@
+# 2018-04-17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test cases for UPSERT
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix zipfile
+
+do_execsql_test upsert2-100 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b int, c DEFAULT 0);
+  INSERT INTO t1(a,b) VALUES(1,2),(3,4);
+  INSERT INTO t1(a,b) VALUES(1,8),(2,11),(3,1)
+    ON CONFLICT(a) DO UPDATE SET b=excluded.b, c=c+1 WHERE t1.b<excluded.b;
+  SELECT *, 'x' FROM t1 ORDER BY a;
+} {1 8 1 x 2 11 0 x 3 4 0 x}
+do_execsql_test upsert2-110 {
+  DROP TABLE t1;
+  CREATE TABLE t1(a INT PRIMARY KEY, b int, c DEFAULT 0) WITHOUT ROWID;
+  INSERT INTO t1(a,b) VALUES(1,2),(3,4);
+  INSERT INTO t1(a,b) VALUES(1,8),(2,11),(3,1)
+    ON CONFLICT(a) DO UPDATE SET b=excluded.b, c=c+1 WHERE t1.b<excluded.b;
+  SELECT *, 'x' FROM t1 ORDER BY a;
+} {1 8 1 x 2 11 0 x 3 4 0 x}
+
+do_execsql_test upsert2-200 {
+  DROP TABLE t1;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b int, c DEFAULT 0);
+  INSERT INTO t1(a,b) VALUES(1,2),(3,4);
+  WITH nx(a,b) AS (VALUES(1,8),(2,11),(3,1),(2,15),(1,4),(1,99))
+  INSERT INTO t1(a,b) SELECT a, b FROM nx WHERE true
+    ON CONFLICT(a) DO UPDATE SET b=excluded.b, c=c+1 WHERE t1.b<excluded.b;
+  SELECT *, 'x' FROM t1 ORDER BY a;
+} {1 99 2 x 2 15 1 x 3 4 0 x}
+do_execsql_test upsert2-201 {
+  DELETE FROM t1;
+  INSERT INTO t1(a,b) VALUES(1,2),(3,4);
+  WITH nx(a,b) AS (VALUES(1,8),(2,11),(3,1),(2,15),(1,4),(1,99))
+  INSERT INTO main.t1 AS t2(a,b) SELECT a, b FROM nx WHERE true
+    ON CONFLICT(a) DO UPDATE SET b=excluded.b, c=t2.c+1 WHERE t2.b<excluded.b;
+  SELECT *, 'x' FROM t1 ORDER BY a;
+} {1 99 2 x 2 15 1 x 3 4 0 x}
+do_catchsql_test upsert2-202 {
+  WITH nx(a,b) AS (VALUES(1,8),(2,11),(3,1),(2,15),(1,4),(1,99))
+  INSERT INTO t1 AS t2(a,b) SELECT a, b FROM nx WHERE true
+    ON CONFLICT(a) DO UPDATE SET b=excluded.b, c=t1.c+1 WHERE t1.b<excluded.b;
+} {1 {no such column: t1.c}}
+do_execsql_test upsert2-210 {
+  DROP TABLE t1;
+  CREATE TABLE t1(a INT PRIMARY KEY, b int, c DEFAULT 0) WITHOUT ROWID;
+  INSERT INTO t1(a,b) VALUES(1,2),(3,4);
+  WITH nx(a,b) AS (VALUES(1,8),(2,11),(3,1),(2,15),(1,4),(1,99))
+  INSERT INTO t1(a,b) SELECT a, b FROM nx WHERE true
+    ON CONFLICT(a) DO UPDATE SET b=excluded.b, c=c+1 WHERE t1.b<excluded.b;
+  SELECT *, 'x' FROM t1 ORDER BY a;
+} {1 99 2 x 2 15 1 x 3 4 0 x}
+
+# On an ON CONFLICT DO UPDATE, the before-insert, before-update, and
+# after-update triggers fire.
+#
+do_execsql_test upsert2-300 {
+  DROP TABLE t1;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b int, c DEFAULT 0);
+  CREATE TABLE record(x TEXT, y TEXT);
+  CREATE TRIGGER r1 BEFORE INSERT ON t1 BEGIN
+    INSERT INTO record(x,y)
+        VALUES('before-insert',printf('%d,%d,%d',new.a,new.b,new.c));
+  END;
+  CREATE TRIGGER r2 AFTER INSERT ON t1 BEGIN
+    INSERT INTO record(x,y)
+        VALUES('after-insert',printf('%d,%d,%d',new.a,new.b,new.c));
+  END;
+  CREATE TRIGGER r3 BEFORE UPDATE ON t1 BEGIN
+    INSERT INTO record(x,y)
+        VALUES('before-update',printf('%d,%d,%d/%d,%d,%d',
+                                      old.a,old.b,old.c,new.a,new.b,new.c));
+  END;
+  CREATE TRIGGER r4 AFTER UPDATE ON t1 BEGIN
+    INSERT INTO record(x,y)
+        VALUES('after-update',printf('%d,%d,%d/%d,%d,%d',
+                                      old.a,old.b,old.c,new.a,new.b,new.c));
+  END;
+  INSERT INTO t1(a,b) VALUES(1,2);
+  DELETE FROM record;
+  INSERT INTO t1(a,b) VALUES(1,2)
+    ON CONFLICT(a) DO UPDATE SET c=t1.c+1;
+  SELECT * FROM record
+} {before-insert 1,2,0 before-update 1,2,0/1,2,1 after-update 1,2,0/1,2,1}
+
+# On an ON CONFLICT DO NOTHING, only the before-insert trigger fires.
+#
+do_execsql_test upsert2-310 {
+  DELETE FROM record;
+  INSERT INTO t1(a,b) VALUES(1,2) ON CONFLICT DO NOTHING;
+  SELECT * FROM record;
+} {before-insert 1,2,0}
+
+# With ON CONFLICT DO UPDATE and a failed WHERE, only the before-insert
+# trigger fires.
+#
+do_execsql_test upsert2-320 {
+  DELETE FROM record;
+  INSERT INTO t1(a,b) VALUES(1,2)
+    ON CONFLICT(a) DO UPDATE SET c=c+1 WHERE c<0;
+  SELECT * FROM record;
+} {before-insert 1,2,0}
+do_execsql_test upsert2-321 {
+  SELECT * FROM t1;
+} {1 2 1}
+
+# Trigger tests repeated for a WITHOUT ROWID table.
+#
+do_execsql_test upsert2-400 {
+  DROP TABLE t1;
+  CREATE TABLE t1(a INT PRIMARY KEY, b int, c DEFAULT 0) WITHOUT ROWID;
+  CREATE TRIGGER r1 BEFORE INSERT ON t1 BEGIN
+    INSERT INTO record(x,y)
+        VALUES('before-insert',printf('%d,%d,%d',new.a,new.b,new.c));
+  END;
+  CREATE TRIGGER r2 AFTER INSERT ON t1 BEGIN
+    INSERT INTO record(x,y)
+        VALUES('after-insert',printf('%d,%d,%d',new.a,new.b,new.c));
+  END;
+  CREATE TRIGGER r3 BEFORE UPDATE ON t1 BEGIN
+    INSERT INTO record(x,y)
+        VALUES('before-update',printf('%d,%d,%d/%d,%d,%d',
+                                      old.a,old.b,old.c,new.a,new.b,new.c));
+  END;
+  CREATE TRIGGER r4 AFTER UPDATE ON t1 BEGIN
+    INSERT INTO record(x,y)
+        VALUES('after-update',printf('%d,%d,%d/%d,%d,%d',
+                                      old.a,old.b,old.c,new.a,new.b,new.c));
+  END;
+  INSERT INTO t1(a,b) VALUES(1,2);
+  DELETE FROM record;
+  INSERT INTO t1(a,b) VALUES(1,2)
+    ON CONFLICT(a) DO UPDATE SET c=t1.c+1;
+  SELECT * FROM record
+} {before-insert 1,2,0 before-update 1,2,0/1,2,1 after-update 1,2,0/1,2,1}
+
+# On an ON CONFLICT DO NOTHING, only the before-insert trigger fires.
+#
+do_execsql_test upsert2-410 {
+  DELETE FROM record;
+  INSERT INTO t1(a,b) VALUES(1,2) ON CONFLICT DO NOTHING;
+  SELECT * FROM record;
+} {before-insert 1,2,0}
+
+# With ON CONFLICT DO UPDATE and a failed WHERE, only the before-insert
+# trigger fires.
+#
+do_execsql_test upsert2-420 {
+  DELETE FROM record;
+  INSERT INTO t1(a,b) VALUES(1,2)
+    ON CONFLICT(a) DO UPDATE SET c=c+1 WHERE c<0;
+  SELECT * FROM record;
+} {before-insert 1,2,0}
+do_execsql_test upsert2-421 {
+  SELECT * FROM t1;
+} {1 2 1}
+
+finish_test

test/upsert3.test

@@ -0,0 +1,58 @@
+# 2018-04-17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test cases for UPSERT
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix zipfile
+
+do_execsql_test upsert3-100 {
+  CREATE TABLE t1(k int, v text);
+  CREATE UNIQUE INDEX x1 ON t1(k, v);
+} {}
+do_catchsql_test upsert3-110 {
+  INSERT INTO t1 VALUES(0,'abcdefghij')
+     ON CONFLICT(k) DO NOTHING;
+} {1 {ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint}}
+do_catchsql_test upsert3-120 {
+  INSERT INTO t1 VALUES(0,'abcdefghij')
+     ON CONFLICT(v) DO NOTHING;
+} {1 {ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint}}
+
+do_execsql_test upsert3-130 {
+  INSERT INTO t1 VALUES(0, 'abcdefghij')
+      ON CONFLICT(k,v) DO NOTHING;
+  SELECT * FROM t1;
+} {0 abcdefghij}
+do_execsql_test upsert3-140 {
+  INSERT INTO t1 VALUES(0, 'abcdefghij')
+      ON CONFLICT(v,k) DO NOTHING;
+  SELECT * FROM t1;
+} {0 abcdefghij}
+
+do_execsql_test upsert3-200 {
+  CREATE TABLE excluded(a INT, b INT, c INT DEFAULT 0);
+  CREATE UNIQUE INDEX excludedab ON excluded(a,b);
+  INSERT INTO excluded(a,b) VALUES(1,2),(1,2),(3,4),(1,2),(5,6),(3,4)
+    ON CONFLICT(b,a) DO UPDATE SET c=excluded.c+1;
+  SELECT *, 'x' FROM excluded ORDER BY a;
+} {1 2 2 x 3 4 1 x 5 6 0 x}
+do_execsql_test upsert3-210 {
+  INSERT INTO excluded AS base(a,b,c) VALUES(1,2,8),(1,2,3)
+    ON CONFLICT(b,a) DO UPDATE SET c=excluded.c+1 WHERE base.c<excluded.c;
+  SELECT *, 'x' FROM excluded ORDER BY a;
+} {1 2 9 x 3 4 1 x 5 6 0 x}
+
+
+
+finish_test

test/upsert4.test

@@ -0,0 +1,407 @@
+# 2018-04-17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test cases for UPSERT
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix upsert4
+
+foreach {tn sql} {
+  1 { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c UNIQUE) }
+  2 { CREATE TABLE t1(a INT PRIMARY KEY, b, c UNIQUE) }
+  3 { CREATE TABLE t1(a INT PRIMARY KEY, b, c UNIQUE) WITHOUT ROWID}
+} {
+  reset_db
+  execsql $sql
+
+  do_execsql_test 1.$tn.0 {
+    INSERT INTO t1 VALUES(1, NULL, 'one');
+    INSERT INTO t1 VALUES(2, NULL, 'two');
+    INSERT INTO t1 VALUES(3, NULL, 'three');
+  }
+  
+  do_execsql_test 1.$tn.1 {
+    INSERT INTO t1 VALUES(1, NULL, 'xyz') ON CONFLICT DO NOTHING;
+    SELECT * FROM t1;
+  } {
+    1 {} one 2 {} two 3 {} three
+  }
+  
+  do_execsql_test 1.$tn.2 {
+    INSERT INTO t1 VALUES(4, NULL, 'two') ON CONFLICT DO NOTHING;
+    SELECT * FROM t1;
+  } {
+    1 {} one 2 {} two 3 {} three
+  }
+  
+  do_execsql_test 1.$tn.3 {
+    INSERT INTO t1 VALUES(4, NULL, 'two') ON CONFLICT (c) DO UPDATE SET b = 1;
+    SELECT * FROM t1;
+  } {
+    1 {} one 2 1 two 3 {} three
+  }
+  
+  do_execsql_test 1.$tn.4 {
+    INSERT INTO t1 VALUES(2, NULL, 'zero') ON CONFLICT (a) DO UPDATE SET b=2;
+    SELECT * FROM t1;
+  } {1 {} one 2 2 two 3 {} three}
+
+  do_catchsql_test 1.$tn.5 {
+    INSERT INTO t1 VALUES(2, NULL, 'zero') ON CONFLICT (a) 
+      DO UPDATE SET c = 'one';
+  } {1 {UNIQUE constraint failed: t1.c}}
+
+  do_execsql_test 1.$tn.6 {
+    SELECT * FROM t1;
+  } {1 {} one 2 2 two 3 {} three}
+
+  do_execsql_test 1.$tn.7 {
+    INSERT INTO t1 VALUES(2, NULL, 'zero') ON CONFLICT (a) 
+      DO UPDATE SET (b, c) = (SELECT 'x', 'y');
+    SELECT * FROM t1;
+  } {1 {} one 2 x y 3 {} three}
+
+  do_execsql_test 1.$tn.8 {
+    INSERT INTO t1 VALUES(1, NULL, NULL) ON CONFLICT (a) 
+      DO UPDATE SET (c, a) = ('four', 4);
+    SELECT * FROM t1 ORDER BY 1;
+  } {2 x y 3 {} three 4 {} four}
+}
+
+#-------------------------------------------------------------------------
+# Test target analysis.
+#
+set rtbl(0) {0 {}}
+set rtbl(1) {/1 .*failed.*/}
+set rtbl(2) {1 {ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint}}
+
+foreach {tn sql} {
+  1 { 
+      CREATE TABLE xyz(a INTEGER PRIMARY KEY, b, c, d);
+      CREATE UNIQUE INDEX xyz1 ON xyz(d, c, b COLLATE nocase);
+  }
+
+  2 { 
+      CREATE TABLE xyz(a INT PRIMARY KEY, b, c, d);
+      CREATE UNIQUE INDEX xyz1 ON xyz(d, c, b COLLATE nocase);
+  }
+
+  3 { 
+      CREATE TABLE xyz(a INT PRIMARY KEY, b, c, d) WITHOUT ROWID;
+      CREATE UNIQUE INDEX xyz1 ON xyz(d, c, b COLLATE nocase);
+  }
+} {
+  reset_db
+  execsql $sql
+  do_execsql_test 2.$tn.1 {
+    INSERT INTO xyz VALUES(10, 1, 1, 'one');
+  }
+
+
+  foreach {tn2 oc res} {
+    1 "ON CONFLICT (b COLLATE nocase, c, d) DO NOTHING"   0
+    2 "ON CONFLICT (b, c, d) DO NOTHING"                  0
+    3 "ON CONFLICT (b, c COLLATE nocase, d) DO NOTHING"   2
+    4 "ON CONFLICT (a) DO NOTHING"                        1
+    5 "ON CONFLICT DO NOTHING"                            0
+    6 "ON CONFLICT (b, c, d) WHERE a!=0 DO NOTHING"       0
+    7 "ON CONFLICT (d, c, c) WHERE a!=0 DO NOTHING"       2
+    8 "ON CONFLICT (b COLLATE nocase, c COLLATE nocase, d) DO NOTHING"   2
+    9 "ON CONFLICT (b, c, d) WHERE b==45 DO NOTHING"      0
+  } {
+
+    do_catchsql_test 2.$tn.2.$tn2 "
+      INSERT INTO xyz VALUES(11, 1, 1, 'one') $oc
+    " $rtbl($res)
+  }
+
+  do_execsql_test 2.$tn.3 {
+    SELECT * FROM xyz;
+  } {10 1 1 one}
+}
+
+foreach {tn sql} {
+  1 {
+    CREATE TABLE abc(a INTEGER PRIMARY KEY, x, y);
+    CREATE UNIQUE INDEX abc1 ON abc(('x' || x) COLLATE nocase);
+  }
+  2 {
+    CREATE TABLE abc(a INT PRIMARY KEY, x, y);
+    CREATE UNIQUE INDEX abc1 ON abc(('x' || x) COLLATE nocase);
+  }
+  3 { 
+    CREATE TABLE abc(a INT PRIMARY KEY, x, y) WITHOUT ROWID;
+    CREATE UNIQUE INDEX abc1 ON abc(('x' || x) COLLATE nocase);
+  }
+} {
+  reset_db
+  execsql $sql
+  do_execsql_test 3.$tn.1 {
+    INSERT INTO abc VALUES(1, 'one', 'two');
+  }
+
+  foreach {tn2 oc res} {
+    1 "ON CONFLICT DO NOTHING"                             0
+    2 "ON CONFLICT ('x' || x) DO NOTHING"                  0
+    3 "ON CONFLICT (('x' || x) COLLATE nocase) DO NOTHING" 0
+    4 "ON CONFLICT (('x' || x) COLLATE binary) DO NOTHING" 2
+    5 "ON CONFLICT (x || 'x') DO NOTHING"                  2
+    6 "ON CONFLICT ((('x' || x))) DO NOTHING"              0
+  } {
+    do_catchsql_test 3.$tn.2.$tn2 "
+      INSERT INTO abc VALUES(2, 'one', NULL) $oc;
+    " $rtbl($res)
+  }
+
+  do_execsql_test 3.$tn.3 {
+    SELECT * FROM abc
+  } {1 one two}
+}
+
+foreach {tn sql} {
+  1 {
+    CREATE TABLE abc(a INTEGER PRIMARY KEY, x, y);
+    CREATE UNIQUE INDEX abc1 ON abc(x) WHERE y>0;
+    CREATE UNIQUE INDEX abc2 ON abc(y) WHERE x='xyz' COLLATE nocase;
+  }
+} {
+  reset_db
+  execsql $sql
+  do_execsql_test 4.$tn.1 {
+    INSERT INTO abc VALUES(1, 'one', 1);
+    INSERT INTO abc VALUES(2, 'two', 2);
+    INSERT INTO abc VALUES(3, 'xyz', 3);
+    INSERT INTO abc VALUES(4, 'XYZ', 4);
+  }
+
+  foreach {tn2 oc res} {
+    1 "ON CONFLICT DO NOTHING"                                 0
+    2 "ON CONFLICT(x) WHERE y>0 DO NOTHING"                    0
+    3 "ON CONFLICT(x) DO NOTHING"                              2
+    4 "ON CONFLICT(x) WHERE y>=0 DO NOTHING"                   2
+    5 "ON CONFLICT(y) WHERE x='xyz' COLLATE nocase DO NOTHING" 1
+  } {
+    do_catchsql_test 4.$tn.2.$tn2 "
+      INSERT INTO abc VALUES(5, 'one', 10) $oc
+    " $rtbl($res)
+  }
+
+  do_execsql_test 4.$tn.3 {
+    SELECT * FROM abc
+  } {1 one 1 2 two 2 3 xyz 3 4 XYZ 4}
+
+  foreach {tn2 oc res} {
+    1 "ON CONFLICT DO NOTHING"                                 0
+    2 "ON CONFLICT(y) WHERE x='xyz' COLLATE nocase DO NOTHING" 0
+    3 "ON CONFLICT(y) WHERE x='xyz' COLLATE binary DO NOTHING" 2
+    4 "ON CONFLICT(x) WHERE y>0 DO NOTHING"                    1
+  } {
+    do_catchsql_test 4.$tn.2.$tn2 "
+      INSERT INTO abc VALUES(5, 'xYz', 3) $oc
+    " $rtbl($res)
+  }
+}
+
+do_catchsql_test 5.0 {
+  CREATE TABLE w1(a INT PRIMARY KEY, x, y);
+  CREATE UNIQUE INDEX w1expr ON w1(('x' || x));
+  INSERT INTO w1 VALUES(2, 'one', NULL)
+    ON CONFLICT (('x' || x) COLLATE nocase) DO NOTHING;
+} {1 {ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint}}
+
+#-------------------------------------------------------------------------
+# Test that ON CONFLICT constraint processing occurs before any REPLACE
+# constraint processing.
+#
+foreach {tn sql} {
+  1 {
+    CREATE TABLE t1(a INTEGER PRIMARY KEY, b UNIQUE, c);
+  }
+  2 {
+    CREATE TABLE t1(a INT PRIMARY KEY, b UNIQUE, c);
+  }
+  3 {
+    CREATE TABLE t1(a INT PRIMARY KEY, b UNIQUE, c) WITHOUT ROWID;
+  }
+} {
+  reset_db
+  execsql $sql
+  do_execsql_test 6.1.$tn {
+    INSERT INTO t1 VALUES(1, 1, 'one');
+    INSERT INTO t1 VALUES(2, 2, 'two');
+    INSERT OR REPLACE INTO t1 VALUES(1, 2, 'two') ON CONFLICT(b) DO NOTHING;
+    PRAGMA integrity_check;
+  } {ok}
+}
+
+foreach {tn sql} {
+  1 {
+    CREATE TABLE t1(a INTEGER PRIMARY KEY, b UNIQUE, c UNIQUE);
+  }
+} {
+  reset_db
+  execsql $sql
+
+  do_execsql_test 6.2.$tn.1 {
+    INSERT INTO t1 VALUES(1, 1, 1);
+    INSERT INTO t1 VALUES(2, 2, 2);
+  }
+
+  do_execsql_test 6.2.$tn.2 {
+    INSERT OR REPLACE INTO t1 VALUES(3, 1, 1) ON CONFLICT(b) DO NOTHING;
+    SELECT * FROM t1;
+    PRAGMA integrity_check;
+  } {1 1 1 2 2 2 ok}
+
+  do_execsql_test 6.2.$tn.3 {
+    INSERT OR REPLACE INTO t1 VALUES(3, 2, 2) ON CONFLICT(c) DO NOTHING;
+    SELECT * FROM t1;
+    PRAGMA integrity_check;
+  } {1 1 1 2 2 2 ok}
+
+  do_execsql_test 6.2.$tn.2 {
+    INSERT OR REPLACE INTO t1 VALUES(3, 1, 1) ON CONFLICT(b) 
+      DO UPDATE SET b=b||'x';
+    SELECT * FROM t1;
+    PRAGMA integrity_check;
+  } {1 1x 1 2 2 2 ok}
+
+  do_execsql_test 6.2.$tn.2 {
+    INSERT OR REPLACE INTO t1 VALUES(3, 2, 2) ON CONFLICT(c) 
+      DO UPDATE SET c=c||'x';
+    SELECT * FROM t1;
+    PRAGMA integrity_check;
+  } {1 1x 1 2 2 2x ok}
+}
+
+#-------------------------------------------------------------------------
+# Test references to "excluded". And using an alias in an INSERT 
+# statement.
+#
+foreach {tn sql} {
+  1 {
+    CREATE TABLE t1(w, x, y, z, PRIMARY KEY(x, y));
+    CREATE UNIQUE INDEX zz ON t1(z);
+  }
+  2 {
+    CREATE TABLE t1(w, x, y, z, PRIMARY KEY(x, y)) WITHOUT ROWID;
+    CREATE UNIQUE INDEX zz ON t1(z);
+  }
+} {
+  reset_db
+  execsql $sql
+  do_execsql_test 7.$tn.0 {
+    INSERT INTO t1 VALUES('a', 1, 1, 1);
+    INSERT INTO t1 VALUES('b', 2, 2, 2);
+  }
+
+  do_execsql_test 7.$tn.1 {
+    INSERT INTO t1 VALUES('c', 3, 3, 1) ON CONFLICT(z) 
+      DO UPDATE SET w = excluded.w;
+    SELECT * FROM t1;
+  } {c 1 1 1 b 2 2 2}
+
+  do_execsql_test 7.$tn.2 {
+    INSERT INTO t1 VALUES('c', 2, 2, 3) ON CONFLICT(y, x) 
+      DO UPDATE SET w = w||w;
+    SELECT * FROM t1;
+  } {c 1 1 1 bb 2 2 2}
+
+  do_execsql_test 7.$tn.3 {
+    INSERT INTO t1 VALUES('c', 2, 2, 3) ON CONFLICT(y, x) 
+      DO UPDATE SET w = w||t1.w;
+    SELECT * FROM t1;
+  } {c 1 1 1 bbbb 2 2 2}
+
+  do_execsql_test 7.$tn.4 {
+    INSERT INTO t1 AS tbl VALUES('c', 2, 2, 3) ON CONFLICT(y, x) 
+      DO UPDATE SET w = w||tbl.w;
+    SELECT * FROM t1;
+  } {c 1 1 1 bbbbbbbb 2 2 2}
+}
+
+foreach {tn sql} {
+  1 {
+    CREATE TABLE excluded(w, x INTEGER, 'a b', z, PRIMARY KEY(x, 'a b'));
+    CREATE UNIQUE INDEX zz ON excluded(z);
+    CREATE INDEX zz2 ON excluded(z);
+  }
+  2 {
+    CREATE TABLE excluded(w, x, 'a b', z, PRIMARY KEY(x, 'a b')) WITHOUT ROWID;
+    CREATE UNIQUE INDEX zz ON excluded(z);
+    CREATE INDEX zz2 ON excluded(z);
+  }
+} {
+  reset_db
+  execsql $sql
+  do_execsql_test 8.$tn.0 {
+    INSERT INTO excluded VALUES('a', 1, 1, 1);
+    INSERT INTO excluded VALUES('b', 2, 2, 2);
+  }
+
+  # Note: An error in Postgres: "table reference "excluded" is ambiguous".
+  #
+  do_execsql_test 8.$tn.1 {
+    INSERT INTO excluded VALUES('hello', 1, 1, NULL) ON CONFLICT(x, "a b")
+      DO UPDATE SET w=excluded.w;
+    SELECT * FROM excluded;
+  } {a 1 1 1 b 2 2 2}
+
+  do_execsql_test 8.$tn.2 {
+    INSERT INTO excluded AS x1 VALUES('hello', 1, 1, NULL) ON CONFLICT(x, [a b])
+      DO UPDATE SET w=excluded.w;
+    SELECT * FROM excluded;
+  } {hello 1 1 1 b 2 2 2}
+
+  do_execsql_test 8.$tn.3 {
+    INSERT INTO excluded AS x1 VALUES('hello', 1, 1, NULL) ON CONFLICT(x, [a b])
+      DO UPDATE SET w=w||w WHERE excluded.w!='hello';
+    SELECT * FROM excluded;
+  } {hello 1 1 1 b 2 2 2}
+
+  do_execsql_test 8.$tn.4 {
+    INSERT INTO excluded AS x1 VALUES('hello', 1, 1, NULL) ON CONFLICT(x, [a b])
+      DO UPDATE SET w=w||w WHERE excluded.x=1;
+    SELECT * FROM excluded;
+  } {hellohello 1 1 1 b 2 2 2}
+
+  do_catchsql_test 8.$tn.5 {
+    INSERT INTO excluded AS x1 VALUES('hello', 1, 1, NULL) 
+      ON CONFLICT(x, [a b]) WHERE y=1
+      DO UPDATE SET w=w||w WHERE excluded.x=1;
+  } {1 {no such column: y}}
+}
+
+#--------------------------------------------------------------------------
+#
+do_execsql_test 9.0 {
+  CREATE TABLE v(x INTEGER);
+  CREATE TABLE hist(x INTEGER PRIMARY KEY, cnt INTEGER);
+  CREATE TRIGGER vt AFTER INSERT ON v BEGIN
+    INSERT INTO hist VALUES(new.x, 1) ON CONFLICT(x) DO
+      UPDATE SET cnt=cnt+1;
+  END;
+}
+
+do_execsql_test 9.1 {
+  INSERT INTO v VALUES(1), (4), (1), (5), (5), (8), (9), (1);
+  SELECT * FROM hist;
+} {
+  1 3
+  4 1
+  5 2
+  8 1
+  9 1
+}
+
+
+finish_test

test/upsertfault.test

@@ -0,0 +1,38 @@
+# 2018-04-17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test cases for UPSERT
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix upsertfault
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a PRIMARY KEY, b, c, d, UNIQUE(b, c));
+  INSERT INTO t1 VALUES(1, 1, 1, 1);
+  INSERT INTO t1 VALUES(2, 2, 2, 2);
+}
+faultsim_save_and_close
+
+do_faultsim_test 1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  db eval { SELECT * FROM sqlite_master } 
+} -body {
+  execsql {
+     INSERT INTO t1 VALUES(3, 2, 2, NULL) ON CONFLICT(b, c) DO
+       UPDATE SET d=d+1;
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+
+finish_test

test/where.test

@@ -1367,5 +1367,50 @@ do_execsql_test where-18.6 {
   SELECT DISTINCT a FROM t181 LEFT JOIN t182 ON a=b ORDER BY +a, +c IS NULL;
 } {1 2}
 
+# Make sure the OR optimization works on a JOIN
+#
+do_execsql_test where-19.0 {
+  CREATE TABLE t191(a INT UNIQUE NOT NULL, b INT UNIQUE NOT NULL,c,d);
+  CREATE INDEX t191a ON t1(a);
+  CREATE INDEX t191b ON t1(b);
+  CREATE TABLE t192(x INTEGER PRIMARY KEY,y INT, z INT);
+
+  EXPLAIN QUERY PLAN
+  SELECT t191.rowid FROM t192, t191 WHERE (a=y OR b=y) AND x=?1;
+} {/.* sqlite_autoindex_t191_1 .* sqlite_autoindex_t191_2 .*/}
+
+# 2018-04-24 ticket [https://www.sqlite.org/src/info/4ba5abf65c5b0f9a]
+# Index on expressions leads to an incorrect answer for a LEFT JOIN
+#
+do_execsql_test where-20.0 {
+  CREATE TABLE t201(x);
+  CREATE TABLE t202(y, z);
+  INSERT INTO t201 VALUES('key');
+  INSERT INTO t202 VALUES('key', -1);
+  CREATE INDEX t202i ON t202(y, ifnull(z, 0));
+  SELECT count(*) FROM t201 LEFT JOIN t202 ON (x=y) WHERE ifnull(z, 0) >=0;
+} {0}
+
+do_execsql_test where-21.0 {
+  CREATE TABLE t12(a, b, c);
+  CREATE TABLE t13(x);
+  CREATE INDEX t12ab ON t12(b, a);
+  CREATE INDEX t12ac ON t12(c, a);
+
+  INSERT INTO t12 VALUES(4, 0, 1);
+  INSERT INTO t12 VALUES(4, 1, 0);
+  INSERT INTO t12 VALUES(5, 0, 1);
+  INSERT INTO t12 VALUES(5, 1, 0);
+
+  INSERT INTO t13 VALUES(1), (2), (3), (4);
+}
+do_execsql_test where-21.1 {
+  SELECT * FROM t12 WHERE 
+  a = (SELECT * FROM (SELECT count(*) FROM t13 LIMIT 5) ORDER BY 1 LIMIT 10) 
+  AND (b=1 OR c=1);
+} {
+  4 1 0
+  4 0 1
+}
 
 finish_test

test/where8.test

@@ -16,6 +16,11 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
+if {[permutation]=="sorterref"} {
+  finish_test
+  return
+}
+
 # Test organization:
 #
 #   where8-1.*: Tests to demonstrate simple cases work with a single table

test/wordcount.c

@@ -28,6 +28,10 @@
 **    (1) REPLACE INTO wordcount
 **        VALUES($new,ifnull((SELECT cnt FROM wordcount WHERE word=$new),0)+1);
 **
+** Upsert mode means:
+**    (1) INSERT INTO wordcount VALUES($new,1)
+**            ON CONFLICT(word) DO UPDATE SET cnt=cnt+1
+**
 ** Select mode means:
 **    (1) SELECT 1 FROM wordcount WHERE word=$new
 **    (2) INSERT INTO wordcount VALUES($new,1) -- if (1) returns nothing
@@ -90,6 +94,7 @@ const char zHelp[] =
 " --timer              Time the operation of this program\n"
 " --trace              Enable sqlite3_trace() output.\n"
 " --update             Use UPDATE mode\n"
+" --upsert             Use UPSERT mode\n"
 " --without-rowid      Use a WITHOUT ROWID table to store the words.\n"
 ;
 
@@ -208,17 +213,19 @@ static void checksumFinalize(sqlite3_context *context){
 /* Define operating modes */
 #define MODE_INSERT     0
 #define MODE_REPLACE    1
-#define MODE_SELECT     2
-#define MODE_UPDATE     3
-#define MODE_DELETE     4
-#define MODE_QUERY      5
-#define MODE_COUNT      6
+#define MODE_UPSERT     2
+#define MODE_SELECT     3
+#define MODE_UPDATE     4
+#define MODE_DELETE     5
+#define MODE_QUERY      6
+#define MODE_COUNT      7
 #define MODE_ALL      (-1)
 
 /* Mode names */
 static const char *azMode[] = {
   "--insert",
   "--replace",
+  "--upsert",
   "--select",
   "--update",
   "--delete",
@@ -292,6 +299,8 @@ int main(int argc, char **argv){
         useWithoutRowid = 1;
       }else if( strcmp(z,"replace")==0 ){
         iMode = MODE_REPLACE;
+      }else if( strcmp(z,"upsert")==0 ){
+        iMode = MODE_UPSERT;
       }else if( strcmp(z,"select")==0 ){
         iMode = MODE_SELECT;
       }else if( strcmp(z,"insert")==0 ){
@@ -467,6 +476,14 @@ int main(int argc, char **argv){
       if( rc ) fatal_error("Could not prepare the REPLACE statement: %s\n",
                             sqlite3_errmsg(db));
     }
+    if( iMode2==MODE_UPSERT ){
+      rc = sqlite3_prepare_v2(db,
+          "INSERT INTO wordcount(word,cnt) VALUES(?1,1) "
+          "ON CONFLICT(word) DO UPDATE SET cnt=cnt+1",
+          -1, &pInsert, 0);
+      if( rc ) fatal_error("Could not prepare the UPSERT statement: %s\n",
+                            sqlite3_errmsg(db));
+    }
     if( iMode2==MODE_DELETE ){
       rc = sqlite3_prepare_v2(db,
             "DELETE FROM wordcount WHERE word=?1",

test/zipfile2.test

@@ -52,17 +52,15 @@ do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE fff USING zipfile('test''zip');
 }
 
-if {$::tcl_platform(platform)=="windows"} {
-  set res {1 {cannot open file: testdir}}
-} else {
-  set res {1 {error in fread()}}
-}
 do_test 2.0 {
   forcedelete testdir
   file mkdir testdir
   execsql { CREATE VIRTUAL TABLE hhh USING zipfile('testdir') }
-  catchsql { SELECT * FROM hhh } 
-} $res
+  lindex [catchsql { 
+    SELECT * FROM hhh;
+    INSERT INTO hhh(name, data) VALUES('1.txt', 'file data');
+  }] 0 
+} 1
 
 
 set archive {

tool/lemon.c

@@ -270,6 +270,8 @@ struct symbol {
   int dtnum;               /* The data type number.  In the parser, the value
                            ** stack is a union.  The .yy%d element of this
                            ** union is the correct data type for this object */
+  int bContent;            /* True if this symbol ever carries content - if
+                           ** it is ever more than just syntax */
   /* The following fields are used by MULTITERMINALs only */
   int nsubsym;             /* Number of constituent symbols in the MULTI */
   struct symbol **subsym;  /* Array of constituent symbols */
@@ -396,6 +398,7 @@ struct lemon {
   struct symbol *wildcard; /* Token that matches anything */
   char *name;              /* Name of the generated parser */
   char *arg;               /* Declaration of the 3th argument to parser */
+  char *ctx;               /* Declaration of 2nd argument to constructor */
   char *tokentype;         /* Type of terminal symbols in the parser stack */
   char *vartype;           /* The default type of non-terminal symbols */
   char *start;             /* Name of the start symbol for the grammar */
@@ -1535,6 +1538,18 @@ static void handle_D_option(char *z){
   *z = 0;
 }
 
+/* Rember the name of the output directory 
+*/
+static char *outputDir = NULL;
+static void handle_d_option(char *z){
+  outputDir = (char *) malloc( lemonStrlen(z)+1 );
+  if( outputDir==0 ){
+    fprintf(stderr,"out of memory\n");
+    exit(1);
+  }
+  lemon_strcpy(outputDir, z);
+}
+
 static char *user_templatename = NULL;
 static void handle_T_option(char *z){
   user_templatename = (char *) malloc( lemonStrlen(z)+1 );
@@ -1616,9 +1631,11 @@ int main(int argc, char **argv)
   static int mhflag = 0;
   static int nolinenosflag = 0;
   static int noResort = 0;
+  
   static struct s_options options[] = {
     {OPT_FLAG, "b", (char*)&basisflag, "Print only the basis in report."},
     {OPT_FLAG, "c", (char*)&compress, "Don't compress the action table."},
+    {OPT_FSTR, "d", (char*)&handle_d_option, "Output directory.  Default '.'"},
     {OPT_FSTR, "D", (char*)handle_D_option, "Define an %ifdef macro."},
     {OPT_FSTR, "f", 0, "Ignored.  (Placeholder for -f compiler options.)"},
     {OPT_FLAG, "g", (char*)&rpflag, "Print grammar without actions."},
@@ -1663,8 +1680,6 @@ int main(int argc, char **argv)
   lem.basisflag = basisflag;
   lem.nolinenosflag = nolinenosflag;
   Symbol_new("$");
-  lem.errsym = Symbol_new("error");
-  lem.errsym->useCnt = 0;
 
   /* Parse the input file */
   Parse(&lem);
@@ -1673,6 +1688,7 @@ int main(int argc, char **argv)
     fprintf(stderr,"Empty grammar.\n");
     exit(1);
   }
+  lem.errsym = Symbol_find("error");
 
   /* Count and index the symbols of the grammar */
   Symbol_new("{default}");
@@ -2363,6 +2379,7 @@ to follow the previous rule.");
           for(i=0; i<psp->nrhs; i++){
             rp->rhs[i] = psp->rhs[i];
             rp->rhsalias[i] = psp->alias[i];
+            if( rp->rhsalias[i]!=0 ){ rp->rhs[i]->bContent = 1; }
           }
           rp->lhs = psp->lhs;
           rp->lhsalias = psp->lhsalias;
@@ -2480,6 +2497,9 @@ to follow the previous rule.");
         }else if( strcmp(x,"extra_argument")==0 ){
           psp->declargslot = &(psp->gp->arg);
           psp->insertLineMacro = 0;
+        }else if( strcmp(x,"extra_context")==0 ){
+          psp->declargslot = &(psp->gp->ctx);
+          psp->insertLineMacro = 0;
         }else if( strcmp(x,"token_type")==0 ){
           psp->declargslot = &(psp->gp->tokentype);
           psp->insertLineMacro = 0;
@@ -3026,13 +3046,28 @@ PRIVATE char *file_makename(struct lemon *lemp, const char *suffix)
 {
   char *name;
   char *cp;
+  char *filename = lemp->filename;
+  int sz;
 
-  name = (char*)malloc( lemonStrlen(lemp->filename) + lemonStrlen(suffix) + 5 );
+  if( outputDir ){
+    cp = strrchr(filename, '/');
+    if( cp ) filename = cp + 1;
+  }
+  sz = lemonStrlen(filename);
+  sz += lemonStrlen(suffix);
+  if( outputDir ) sz += lemonStrlen(outputDir) + 1;
+  sz += 5;
+  name = (char*)malloc( sz );
   if( name==0 ){
     fprintf(stderr,"Can't allocate space for a filename.\n");
     exit(1);
   }
-  lemon_strcpy(name,lemp->filename);
+  name[0] = 0;
+  if( outputDir ){
+    lemon_strcpy(name, outputDir);
+    lemon_strcat(name, "/");
+  }
+  lemon_strcat(name,filename);
   cp = strrchr(name,'.');
   if( cp ) *cp = 0;
   lemon_strcat(name,suffix);
@@ -3250,7 +3285,7 @@ int PrintAction(
 /* Generate the "*.out" log file */
 void ReportOutput(struct lemon *lemp)
 {
-  int i;
+  int i, n;
   struct state *stp;
   struct config *cfp;
   struct action *ap;
@@ -3290,6 +3325,7 @@ void ReportOutput(struct lemon *lemp)
   }
   fprintf(fp, "----------------------------------------------------\n");
   fprintf(fp, "Symbols:\n");
+  fprintf(fp, "The first-set of non-terminals is shown after the name.\n\n");
   for(i=0; i<lemp->nsymbol; i++){
     int j;
     struct symbol *sp;
@@ -3311,6 +3347,26 @@ void ReportOutput(struct lemon *lemp)
     fprintf(fp, "\n");
   }
   fprintf(fp, "----------------------------------------------------\n");
+  fprintf(fp, "Syntax-only Symbols:\n");
+  fprintf(fp, "The following symbols never carry semantic content.\n\n");
+  for(i=n=0; i<lemp->nsymbol; i++){
+    int w;
+    struct symbol *sp = lemp->symbols[i];
+    if( sp->bContent ) continue;
+    w = (int)strlen(sp->name);
+    if( n>0 && n+w>75 ){
+      fprintf(fp,"\n");
+      n = 0;
+    }
+    if( n>0 ){
+      fprintf(fp, " ");
+      n++;
+    }
+    fprintf(fp, "%s", sp->name);
+    n += w;
+  }
+  if( n>0 ) fprintf(fp, "\n");
+  fprintf(fp, "----------------------------------------------------\n");
   fprintf(fp, "Rules:\n");
   for(rp=lemp->rule; rp; rp=rp->next){
     fprintf(fp, "%4d: ", rp->iRule);
@@ -3994,7 +4050,7 @@ void print_stack_union(
     fprintf(out,"  %s yy%d;\n",types[i],i+1); lineno++;
     free(types[i]);
   }
-  if( lemp->errsym->useCnt ){
+  if( lemp->errsym && lemp->errsym->useCnt ){
     fprintf(out,"  int yy%d;\n",lemp->errsym->dtnum); lineno++;
   }
   free(stddt);
@@ -4144,8 +4200,8 @@ void ReportTable(
 
   /* Generate the defines */
   fprintf(out,"#define YYCODETYPE %s\n",
-    minimum_size_type(0, lemp->nsymbol+1, &szCodeType)); lineno++;
-  fprintf(out,"#define YYNOCODE %d\n",lemp->nsymbol+1);  lineno++;
+    minimum_size_type(0, lemp->nsymbol, &szCodeType)); lineno++;
+  fprintf(out,"#define YYNOCODE %d\n",lemp->nsymbol);  lineno++;
   fprintf(out,"#define YYACTIONTYPE %s\n",
     minimum_size_type(0,lemp->maxAction,&szActionType)); lineno++;
   if( lemp->wildcard ){
@@ -4170,20 +4226,40 @@ void ReportTable(
     while( i>=1 && (ISALNUM(lemp->arg[i-1]) || lemp->arg[i-1]=='_') ) i--;
     fprintf(out,"#define %sARG_SDECL %s;\n",name,lemp->arg);  lineno++;
     fprintf(out,"#define %sARG_PDECL ,%s\n",name,lemp->arg);  lineno++;
-    fprintf(out,"#define %sARG_FETCH %s = yypParser->%s\n",
+    fprintf(out,"#define %sARG_PARAM ,%s\n",name,&lemp->arg[i]);  lineno++;
+    fprintf(out,"#define %sARG_FETCH %s=yypParser->%s;\n",
                  name,lemp->arg,&lemp->arg[i]);  lineno++;
-    fprintf(out,"#define %sARG_STORE yypParser->%s = %s\n",
+    fprintf(out,"#define %sARG_STORE yypParser->%s=%s;\n",
                  name,&lemp->arg[i],&lemp->arg[i]);  lineno++;
   }else{
-    fprintf(out,"#define %sARG_SDECL\n",name);  lineno++;
-    fprintf(out,"#define %sARG_PDECL\n",name);  lineno++;
+    fprintf(out,"#define %sARG_SDECL\n",name); lineno++;
+    fprintf(out,"#define %sARG_PDECL\n",name); lineno++;
+    fprintf(out,"#define %sARG_PARAM\n",name); lineno++;
     fprintf(out,"#define %sARG_FETCH\n",name); lineno++;
     fprintf(out,"#define %sARG_STORE\n",name); lineno++;
   }
+  if( lemp->ctx && lemp->ctx[0] ){
+    i = lemonStrlen(lemp->ctx);
+    while( i>=1 && ISSPACE(lemp->ctx[i-1]) ) i--;
+    while( i>=1 && (ISALNUM(lemp->ctx[i-1]) || lemp->ctx[i-1]=='_') ) i--;
+    fprintf(out,"#define %sCTX_SDECL %s;\n",name,lemp->ctx);  lineno++;
+    fprintf(out,"#define %sCTX_PDECL ,%s\n",name,lemp->ctx);  lineno++;
+    fprintf(out,"#define %sCTX_PARAM ,%s\n",name,&lemp->ctx[i]);  lineno++;
+    fprintf(out,"#define %sCTX_FETCH %s=yypParser->%s;\n",
+                 name,lemp->ctx,&lemp->ctx[i]);  lineno++;
+    fprintf(out,"#define %sCTX_STORE yypParser->%s=%s;\n",
+                 name,&lemp->ctx[i],&lemp->ctx[i]);  lineno++;
+  }else{
+    fprintf(out,"#define %sCTX_SDECL\n",name); lineno++;
+    fprintf(out,"#define %sCTX_PDECL\n",name); lineno++;
+    fprintf(out,"#define %sCTX_PARAM\n",name); lineno++;
+    fprintf(out,"#define %sCTX_FETCH\n",name); lineno++;
+    fprintf(out,"#define %sCTX_STORE\n",name); lineno++;
+  }
   if( mhflag ){
     fprintf(out,"#endif\n"); lineno++;
   }
-  if( lemp->errsym->useCnt ){
+  if( lemp->errsym && lemp->errsym->useCnt ){
     fprintf(out,"#define YYERRORSYMBOL %d\n",lemp->errsym->index); lineno++;
     fprintf(out,"#define YYERRSYMDT yy%d\n",lemp->errsym->dtnum); lineno++;
   }

tool/lempar.c

@@ -66,8 +66,10 @@
 **                       zero the stack is dynamically sized using realloc()
 **    ParseARG_SDECL     A static variable declaration for the %extra_argument
 **    ParseARG_PDECL     A parameter declaration for the %extra_argument
+**    ParseARG_PARAM     Code to pass %extra_argument as a subroutine parameter
 **    ParseARG_STORE     Code to store %extra_argument into yypParser
 **    ParseARG_FETCH     Code to extract %extra_argument from yypParser
+**    ParseCTX_*         As ParseARG_ except for %extra_context
 **    YYERRORSYMBOL      is the code number of the error symbol.  If not
 **                       defined, then do no error processing.
 **    YYNSTATE           the combined number of states.
@@ -211,6 +213,7 @@ struct yyParser {
   int yyerrcnt;                 /* Shifts left before out of the error */
 #endif
   ParseARG_SDECL                /* A place to hold %extra_argument */
+  ParseCTX_SDECL                /* A place to hold %extra_context */
 #if YYSTACKDEPTH<=0
   int yystksz;                  /* Current side of the stack */
   yyStackEntry *yystack;        /* The parser's stack */
@@ -315,28 +318,29 @@ static int yyGrowStack(yyParser *p){
 
 /* Initialize a new parser that has already been allocated.
 */
-void ParseInit(void *yypParser){
-  yyParser *pParser = (yyParser*)yypParser;
+void ParseInit(void *yypRawParser ParseCTX_PDECL){
+  yyParser *yypParser = (yyParser*)yypRawParser;
+  ParseCTX_STORE
 #ifdef YYTRACKMAXSTACKDEPTH
-  pParser->yyhwm = 0;
+  yypParser->yyhwm = 0;
 #endif
 #if YYSTACKDEPTH<=0
-  pParser->yytos = NULL;
-  pParser->yystack = NULL;
-  pParser->yystksz = 0;
-  if( yyGrowStack(pParser) ){
-    pParser->yystack = &pParser->yystk0;
-    pParser->yystksz = 1;
+  yypParser->yytos = NULL;
+  yypParser->yystack = NULL;
+  yypParser->yystksz = 0;
+  if( yyGrowStack(yypParser) ){
+    yypParser->yystack = &yypParser->yystk0;
+    yypParser->yystksz = 1;
   }
 #endif
 #ifndef YYNOERRORRECOVERY
-  pParser->yyerrcnt = -1;
+  yypParser->yyerrcnt = -1;
 #endif
-  pParser->yytos = pParser->yystack;
-  pParser->yystack[0].stateno = 0;
-  pParser->yystack[0].major = 0;
+  yypParser->yytos = yypParser->yystack;
+  yypParser->yystack[0].stateno = 0;
+  yypParser->yystack[0].major = 0;
 #if YYSTACKDEPTH>0
-  pParser->yystackEnd = &pParser->yystack[YYSTACKDEPTH-1];
+  yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1];
 #endif
 }
 
@@ -353,11 +357,14 @@ void ParseInit(void *yypParser){
 ** A pointer to a parser.  This pointer is used in subsequent calls
 ** to Parse and ParseFree.
 */
-void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){
-  yyParser *pParser;
-  pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) );
-  if( pParser ) ParseInit(pParser);
-  return pParser;
+void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) ParseCTX_PDECL){
+  yyParser *yypParser;
+  yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) );
+  if( yypParser ){
+    ParseCTX_STORE
+    ParseInit(yypParser ParseCTX_PARAM);
+  }
+  return (void*)yypParser;
 }
 #endif /* Parse_ENGINEALWAYSONSTACK */
 
@@ -374,7 +381,8 @@ static void yy_destructor(
   YYCODETYPE yymajor,     /* Type code for object to destroy */
   YYMINORTYPE *yypminor   /* The object to be destroyed */
 ){
-  ParseARG_FETCH;
+  ParseARG_FETCH
+  ParseCTX_FETCH
   switch( yymajor ){
     /* Here is inserted the actions which take place when a
     ** terminal or non-terminal is destroyed.  This can happen
@@ -497,13 +505,12 @@ int ParseCoverage(FILE *out){
 ** Find the appropriate action for a parser given the terminal
 ** look-ahead token iLookAhead.
 */
-static unsigned int yy_find_shift_action(
-  yyParser *pParser,        /* The parser */
-  YYCODETYPE iLookAhead     /* The look-ahead token */
+static YYACTIONTYPE yy_find_shift_action(
+  YYCODETYPE iLookAhead,    /* The look-ahead token */
+  YYACTIONTYPE stateno      /* Current state number */
 ){
   int i;
-  int stateno = pParser->yytos->stateno;
- 
+
   if( stateno>YY_MAX_SHIFT ) return stateno;
   assert( stateno <= YY_SHIFT_COUNT );
 #if defined(YYCOVERAGE)
@@ -567,7 +574,7 @@ static unsigned int yy_find_shift_action(
 ** look-ahead token iLookAhead.
 */
 static int yy_find_reduce_action(
-  int stateno,              /* Current state number */
+  YYACTIONTYPE stateno,     /* Current state number */
   YYCODETYPE iLookAhead     /* The look-ahead token */
 ){
   int i;
@@ -596,7 +603,8 @@ static int yy_find_reduce_action(
 ** The following routine is called if the stack overflows.
 */
 static void yyStackOverflow(yyParser *yypParser){
-   ParseARG_FETCH;
+   ParseARG_FETCH
+   ParseCTX_FETCH
 #ifndef NDEBUG
    if( yyTraceFILE ){
      fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt);
@@ -608,7 +616,8 @@ static void yyStackOverflow(yyParser *yypParser){
 /******** Begin %stack_overflow code ******************************************/
 %%
 /******** End %stack_overflow code ********************************************/
-   ParseARG_STORE; /* Suppress warning about unused %extra_argument var */
+   ParseARG_STORE /* Suppress warning about unused %extra_argument var */
+   ParseCTX_STORE
 }
 
 /*
@@ -637,8 +646,8 @@ static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){
 */
 static void yy_shift(
   yyParser *yypParser,          /* The parser to be shifted */
-  int yyNewState,               /* The new state to shift in */
-  int yyMajor,                  /* The major token to shift in */
+  YYACTIONTYPE yyNewState,      /* The new state to shift in */
+  YYCODETYPE yyMajor,           /* The major token to shift in */
   ParseTOKENTYPE yyMinor        /* The minor token to shift in */
 ){
   yyStackEntry *yytos;
@@ -668,8 +677,8 @@ static void yy_shift(
     yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
   }
   yytos = yypParser->yytos;
-  yytos->stateno = (YYACTIONTYPE)yyNewState;
-  yytos->major = (YYCODETYPE)yyMajor;
+  yytos->stateno = yyNewState;
+  yytos->major = yyMajor;
   yytos->minor.yy0 = yyMinor;
   yyTraceShift(yypParser, yyNewState, "Shift");
 }
@@ -696,17 +705,18 @@ static void yy_accept(yyParser*);  /* Forward Declaration */
 ** only called from one place, optimizing compilers will in-line it, which
 ** means that the extra parameters have no performance impact.
 */
-static void yy_reduce(
+static YYACTIONTYPE yy_reduce(
   yyParser *yypParser,         /* The parser */
   unsigned int yyruleno,       /* Number of the rule by which to reduce */
   int yyLookahead,             /* Lookahead token, or YYNOCODE if none */
   ParseTOKENTYPE yyLookaheadToken  /* Value of the lookahead token */
+  ParseCTX_PDECL                   /* %extra_context */
 ){
   int yygoto;                     /* The next state */
   int yyact;                      /* The next action */
   yyStackEntry *yymsp;            /* The top of the parser's stack */
   int yysize;                     /* Amount to pop the stack */
-  ParseARG_FETCH;
+  ParseARG_FETCH
   (void)yyLookahead;
   (void)yyLookaheadToken;
   yymsp = yypParser->yytos;
@@ -737,13 +747,19 @@ static void yy_reduce(
 #if YYSTACKDEPTH>0 
     if( yypParser->yytos>=yypParser->yystackEnd ){
       yyStackOverflow(yypParser);
-      return;
+      /* The call to yyStackOverflow() above pops the stack until it is
+      ** empty, causing the main parser loop to exit.  So the return value
+      ** is never used and does not matter. */
+      return 0;
     }
 #else
     if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){
       if( yyGrowStack(yypParser) ){
         yyStackOverflow(yypParser);
-        return;
+        /* The call to yyStackOverflow() above pops the stack until it is
+        ** empty, causing the main parser loop to exit.  So the return value
+        ** is never used and does not matter. */
+        return 0;
       }
       yymsp = yypParser->yytos;
     }
@@ -780,6 +796,7 @@ static void yy_reduce(
   yymsp->stateno = (YYACTIONTYPE)yyact;
   yymsp->major = (YYCODETYPE)yygoto;
   yyTraceShift(yypParser, yyact, "... then shift");
+  return yyact;
 }
 
 /*
@@ -789,7 +806,8 @@ static void yy_reduce(
 static void yy_parse_failed(
   yyParser *yypParser           /* The parser */
 ){
-  ParseARG_FETCH;
+  ParseARG_FETCH
+  ParseCTX_FETCH
 #ifndef NDEBUG
   if( yyTraceFILE ){
     fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt);
@@ -801,7 +819,8 @@ static void yy_parse_failed(
 /************ Begin %parse_failure code ***************************************/
 %%
 /************ End %parse_failure code *****************************************/
-  ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
+  ParseARG_STORE /* Suppress warning about unused %extra_argument variable */
+  ParseCTX_STORE
 }
 #endif /* YYNOERRORRECOVERY */
 
@@ -813,12 +832,14 @@ static void yy_syntax_error(
   int yymajor,                   /* The major type of the error token */
   ParseTOKENTYPE yyminor         /* The minor type of the error token */
 ){
-  ParseARG_FETCH;
+  ParseARG_FETCH
+  ParseCTX_FETCH
 #define TOKEN yyminor
 /************ Begin %syntax_error code ****************************************/
 %%
 /************ End %syntax_error code ******************************************/
-  ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
+  ParseARG_STORE /* Suppress warning about unused %extra_argument variable */
+  ParseCTX_STORE
 }
 
 /*
@@ -827,7 +848,8 @@ static void yy_syntax_error(
 static void yy_accept(
   yyParser *yypParser           /* The parser */
 ){
-  ParseARG_FETCH;
+  ParseARG_FETCH
+  ParseCTX_FETCH
 #ifndef NDEBUG
   if( yyTraceFILE ){
     fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt);
@@ -842,7 +864,8 @@ static void yy_accept(
 /*********** Begin %parse_accept code *****************************************/
 %%
 /*********** End %parse_accept code *******************************************/
-  ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
+  ParseARG_STORE /* Suppress warning about unused %extra_argument variable */
+  ParseCTX_STORE
 }
 
 /* The main parser program.
@@ -871,45 +894,47 @@ void Parse(
   ParseARG_PDECL               /* Optional %extra_argument parameter */
 ){
   YYMINORTYPE yyminorunion;
-  unsigned int yyact;   /* The parser action. */
+  YYACTIONTYPE yyact;   /* The parser action. */
 #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
   int yyendofinput;     /* True if we are at the end of input */
 #endif
 #ifdef YYERRORSYMBOL
   int yyerrorhit = 0;   /* True if yymajor has invoked an error */
 #endif
-  yyParser *yypParser;  /* The parser */
+  yyParser *yypParser = (yyParser*)yyp;  /* The parser */
+  ParseCTX_FETCH
+  ParseARG_STORE
 
-  yypParser = (yyParser*)yyp;
   assert( yypParser->yytos!=0 );
 #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
   yyendofinput = (yymajor==0);
 #endif
-  ParseARG_STORE;
 
+  yyact = yypParser->yytos->stateno;
 #ifndef NDEBUG
   if( yyTraceFILE ){
-    int stateno = yypParser->yytos->stateno;
-    if( stateno < YY_MIN_REDUCE ){
+    if( yyact < YY_MIN_REDUCE ){
       fprintf(yyTraceFILE,"%sInput '%s' in state %d\n",
-              yyTracePrompt,yyTokenName[yymajor],stateno);
+              yyTracePrompt,yyTokenName[yymajor],yyact);
     }else{
       fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n",
-              yyTracePrompt,yyTokenName[yymajor],stateno-YY_MIN_REDUCE);
+              yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE);
     }
   }
 #endif
 
   do{
-    yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor);
+    assert( yyact==yypParser->yytos->stateno );
+    yyact = yy_find_shift_action(yymajor,yyact);
     if( yyact >= YY_MIN_REDUCE ){
-      yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,yyminor);
+      yyact = yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,
+                        yyminor ParseCTX_PARAM);
     }else if( yyact <= YY_MAX_SHIFTREDUCE ){
       yy_shift(yypParser,yyact,yymajor,yyminor);
 #ifndef YYNOERRORRECOVERY
       yypParser->yyerrcnt--;
 #endif
-      yymajor = YYNOCODE;
+      break;
     }else if( yyact==YY_ACCEPT_ACTION ){
       yypParser->yytos--;
       yy_accept(yypParser);
@@ -980,6 +1005,8 @@ void Parse(
       }
       yypParser->yyerrcnt = 3;
       yyerrorhit = 1;
+      if( yymajor==YYNOCODE ) break;
+      yyact = yypParser->yytos->stateno;
 #elif defined(YYNOERRORRECOVERY)
       /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to
       ** do any kind of error recovery.  Instead, simply invoke the syntax
@@ -990,8 +1017,7 @@ void Parse(
       */
       yy_syntax_error(yypParser,yymajor, yyminor);
       yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
-      yymajor = YYNOCODE;
-      
+      break;
 #else  /* YYERRORSYMBOL is not defined */
       /* This is what we do if the grammar does not define ERROR:
       **
@@ -1013,10 +1039,10 @@ void Parse(
         yypParser->yyerrcnt = -1;
 #endif
       }
-      yymajor = YYNOCODE;
+      break;
 #endif
     }
-  }while( yymajor!=YYNOCODE && yypParser->yytos>yypParser->yystack );
+  }while( yypParser->yytos>yypParser->yystack );
 #ifndef NDEBUG
   if( yyTraceFILE ){
     yyStackEntry *i;

tool/mkkeywordhash.c

@@ -143,6 +143,11 @@ struct Keyword {
 #else
 #  define CTE        0x00040000
 #endif
+#ifdef SQLITE_OMIT_UPSERT
+#  define UPSERT     0
+#else
+#  define UPSERT     0x00080000
+#endif
 
 /*
 ** These are the keywords
@@ -186,6 +191,7 @@ static Keyword aKeywordTable[] = {
   { "DESC",             "TK_DESC",         ALWAYS                 },
   { "DETACH",           "TK_DETACH",       ATTACH                 },
   { "DISTINCT",         "TK_DISTINCT",     ALWAYS                 },
+  { "DO",               "TK_DO",           UPSERT                 },
   { "DROP",             "TK_DROP",         ALWAYS                 },
   { "END",              "TK_END",          ALWAYS                 },
   { "EACH",             "TK_EACH",         TRIGGER                },
@@ -226,6 +232,7 @@ static Keyword aKeywordTable[] = {
   { "NATURAL",          "TK_JOIN_KW",      ALWAYS                 },
   { "NO",               "TK_NO",           FKEY                   },
   { "NOT",              "TK_NOT",          ALWAYS                 },
+  { "NOTHING",          "TK_NOTHING",      UPSERT                 },
   { "NOTNULL",          "TK_NOTNULL",      ALWAYS                 },
   { "NULL",             "TK_NULL",         ALWAYS                 },
   { "OF",               "TK_OF",           ALWAYS                 },
@@ -610,6 +617,16 @@ int main(int argc, char **argv){
   printf("  return id;\n");
   printf("}\n");
   printf("#define SQLITE_N_KEYWORD %d\n", nKeyword);
+  printf("int sqlite3_keyword_name(int i,const char **pzName,int *pnName){\n");
+  printf("  if( i<0 || i>=SQLITE_N_KEYWORD ) return SQLITE_ERROR;\n");
+  printf("  *pzName = zKWText + aKWOffset[i];\n");
+  printf("  *pnName = aKWLen[i];\n");
+  printf("  return SQLITE_OK;\n");
+  printf("}\n");
+  printf("int sqlite3_keyword_count(void){ return SQLITE_N_KEYWORD; }\n");
+  printf("int sqlite3_keyword_check(const char *zName, int nName){\n");
+  printf("  return TK_ID!=sqlite3KeywordCode((const u8*)zName, nName);\n");
+  printf("}\n");
 
   return 0;
 }

tool/mksqlite3c.tcl

@@ -363,6 +363,7 @@ foreach file {
    table.c
    trigger.c
    update.c
+   upsert.c
    vacuum.c
    vtab.c
    wherecode.c

tool/sqldiff.c

@@ -134,28 +134,8 @@ static void strPrintf(Str *p, const char *zFormat, ...){
 ** needed.
 */
 static char *safeId(const char *zId){
-  /* All SQLite keywords, in alphabetical order */
-  static const char *azKeywords[] = {
-    "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
-    "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
-    "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
-    "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
-    "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
-    "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
-    "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
-    "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
-    "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
-    "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
-    "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
-    "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
-    "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
-    "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
-    "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
-    "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
-    "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
-    "WITH", "WITHOUT",
-  };
-  int lwr, upr, mid, c, i, x;
+  int i, x;
+  char c;
   if( zId[0]==0 ) return sqlite3_mprintf("\"\"");
   for(i=x=0; (c = zId[i])!=0; i++){
     if( !isalpha(c) && c!='_' ){
@@ -166,20 +146,10 @@ static char *safeId(const char *zId){
       }
     }
   }
-  if( x ) return sqlite3_mprintf("%s", zId);
-  lwr = 0;
-  upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1;
-  while( lwr<=upr ){
-    mid = (lwr+upr)/2;
-    c = sqlite3_stricmp(azKeywords[mid], zId);
-    if( c==0 ) return sqlite3_mprintf("\"%w\"", zId);
-    if( c<0 ){
-      lwr = mid+1;
-    }else{
-      upr = mid-1;
-    }
+  if( x || !sqlite3_keyword_check(zId,i) ){
+    return sqlite3_mprintf("%s", zId);
   }
-  return sqlite3_mprintf("%s", zId);
+  return sqlite3_mprintf("\"%w\"", zId);
 }
 
 /*