Merge pull request #237 from servicetitan/upstream/SqlTruncateTable

Optimize work with temporary tables when translating `.In()` extension

- SqlInsert supports insertion of multiple rows at once;
- SqlInsert Values dictionary became obsolete, all usages of it are eliminated,
  add values as rows instead;
- use of Truncate instead of Delete for temporary table clean-up;
- improves population of temporary tables by using 2 grades of multi-row inserts;
- new property DomainConfiguration.MaxNumberOfConditions can define limit when .In()
  extension with IncludeAlgorithm.Auto will decide to use temporary table 
  instead of IN clause, Default value is 256 (as it was before) and can be found
  among WellKnown constants. Old WellKnown.MaxNumberOfConditions is Obsolete
  Use WellKnown.DefaultMaxNumberOfConditions or DomainConfiguration.MaxNumberOfConditions.

Author: alex-kulakov

Extensions/Xtensive.Orm.BulkOperations/Internals/AddValueContext.cs

@@ -1,3 +1,4 @@
+using System.Collections.Generic;
 using System.Linq.Expressions;
 using Xtensive.Linq;
 using Xtensive.Orm.Model;
@@ -14,9 +15,9 @@ internal class AddValueContext
     public LambdaExpression Lambda { get; set; }
     public SetStatement Statement { get; set; }
 
-    public FieldInfo Field { get; set; }
+    public FieldInfo Field => Descriptor.Field;
 
-    public bool SubqueryExists { get; set; }
+    public Dictionary<SqlColumn, SqlExpression> Values { get; set; }
 
     public object EvalLambdaBody() =>
       Lambda.Body is ConstantExpression ce

Extensions/Xtensive.Orm.BulkOperations/Internals/BaseSqlVisitor.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
+using System.Linq;
 using Xtensive.Sql;
 using Xtensive.Sql.Ddl;
 using Xtensive.Sql.Dml;
@@ -247,6 +248,10 @@ public virtual void Visit(SqlDropView node)
     {
     }
 
+    public virtual void Visit(SqlTruncateTable node)
+    {
+    }
+
     public virtual void Visit(SqlDynamicFilter node)
     {
       foreach (SqlExpression expression in node.Expressions)
@@ -314,9 +319,12 @@ public virtual void Visit(SqlInsert node)
     {
       VisitInternal(node.From);
       VisitInternal(node.Into);
-      foreach (var pair in node.Values) {
-        VisitInternal(pair.Key);
-        VisitInternal(pair.Value);
+      
+      foreach(var row in node.ValueRows) {
+        foreach(var columnvalue in row.Zip(node.ValueRows.Columns)) {
+          VisitInternal(columnvalue.Second);
+          VisitInternal(columnvalue.First);
+        }
       }
     }
 

Extensions/Xtensive.Orm.BulkOperations/Internals/SetOperation.cs

@@ -1,4 +1,5 @@
-using System;
+using System;
+using System.Collections.Generic;
 using Xtensive.Sql;
 using Xtensive.Sql.Dml;
 
@@ -20,9 +21,9 @@ public override SqlTable Table
         get { return insert.Into; }
       }
 
-      public override void AddValue(SqlTableColumn column, SqlExpression value)
+      public override void AddValues(Dictionary<SqlColumn, SqlExpression> values)
       {
-        insert.Values.Add(column, value);
+        insert.ValueRows.Add(values);
       }
     }
 
@@ -42,30 +43,33 @@ public override SqlTable Table
         get { return update.Update; }
       }
 
-      public override void AddValue(SqlTableColumn column, SqlExpression value)
+      public override void AddValues(Dictionary<SqlColumn, SqlExpression> values)
       {
-        update.Values.Add(column, value);
+        if (update.Values.Count!=0) {
+          throw new InvalidOperationException("Update values have already been initialized");
+        }
+        foreach (var keyValue in values) {
+          update.Values.Add((SqlTableColumn)keyValue.Key, keyValue.Value);
+        }
       }
     }
 
     #endregion
 
     private SqlQueryStatement statement;
 
-    public static SetStatement Create(SqlQueryStatement statement)
+    public static SetStatement Create(SqlUpdate updateStatement)
     {
-      SetStatement result;
-      if (statement is SqlUpdate)
-        result = new Update();
-      else if (statement is SqlInsert)
-        result = new Insert();
-      else
-        throw new InvalidOperationException("Statement must be SqlUpdate or SqlInsert");
-      result.statement = statement;
-      return result;
+      return new Update() { statement = updateStatement };
+    }
+
+    public static SetStatement Create(SqlInsert insertStatement)
+    {
+      return new Insert() { statement = insertStatement };
     }
 
     public abstract SqlTable Table { get; }
-    public abstract void AddValue(SqlTableColumn column, SqlExpression value);
+
+    public abstract void AddValues(Dictionary<SqlColumn, SqlExpression> values);
   }
 }

Extensions/Xtensive.Orm.BulkOperations/Internals/SetStatement.cs

@@ -1,4 +1,4 @@
-// Copyright (C) 2020 Xtensive LLC.
+// Copyright (C) 2020-2023 Xtensive LLC.
 // This code is distributed under MIT license terms.
 // See the License.txt file in the project root for more information.
 
@@ -16,7 +16,9 @@ internal static class WellKnownMembers
     public static readonly Type FuncOfTArgTResultType = typeof(Func<,>);
 
     public static readonly Type IncludeAlgorithmType = typeof(IncludeAlgorithm);
+    public static readonly Type QueryableType = typeof(Queryable);
     public static readonly Type QueryableExtensionsType = typeof(QueryableExtensions);
+
     public const string InMethodName = nameof(QueryableExtensions.In);
 
     public static readonly MethodInfo TranslateQueryMethod =

Extensions/Xtensive.Orm.BulkOperations/Internals/WellKnownMembers.cs

@@ -40,15 +40,15 @@ public override EntityInfo GetTriggerInfo()
     {
       EntityInfo triggerInfo = new EntityInfo();
       triggerInfo.MaxIdentifierLength = MaxIdentifierLength;
-      triggerInfo.AllowedDdlStatements = DdlStatements.All;
+      triggerInfo.AllowedDdlStatements = DdlStatements.All & ~DdlStatements.Truncate;
       return triggerInfo;
     }
 
     public override EntityInfo GetStoredProcedureInfo()
     {
       EntityInfo procedureInfo = new EntityInfo();
       procedureInfo.MaxIdentifierLength = MaxIdentifierLength;
-      procedureInfo.AllowedDdlStatements = DdlStatements.All;
+      procedureInfo.AllowedDdlStatements = DdlStatements.All & ~DdlStatements.Truncate;
       return procedureInfo;
     }
 
@@ -61,7 +61,7 @@ public override EntityInfo GetDatabaseInfo()
     {
       EntityInfo databaseInfo = new EntityInfo();
       databaseInfo.MaxIdentifierLength = MaxIdentifierLength;
-      databaseInfo.AllowedDdlStatements = DdlStatements.All;
+      databaseInfo.AllowedDdlStatements = DdlStatements.All & ~DdlStatements.Truncate;
       return databaseInfo;
     }
 
@@ -70,23 +70,23 @@ public override ColumnInfo GetColumnInfo()
       ColumnInfo columnInfo = new ColumnInfo();
       columnInfo.MaxIdentifierLength = MaxIdentifierLength;
       columnInfo.Features = ColumnFeatures.Identity | ColumnFeatures.Computed;
-      columnInfo.AllowedDdlStatements = DdlStatements.All;
+      columnInfo.AllowedDdlStatements = DdlStatements.All & ~DdlStatements.Truncate;
       return columnInfo;
     }
 
     public override EntityInfo GetViewInfo()
     {
       EntityInfo viewInfo = new EntityInfo();
       viewInfo.MaxIdentifierLength = MaxIdentifierLength;
-      viewInfo.AllowedDdlStatements = DdlStatements.All;
+      viewInfo.AllowedDdlStatements = DdlStatements.All & ~DdlStatements.Truncate;
       return viewInfo;
     }
 
     public override EntityInfo GetSchemaInfo()
     {
       EntityInfo schemaInfo = new EntityInfo();
       schemaInfo.MaxIdentifierLength = MaxIdentifierLength;
-      schemaInfo.AllowedDdlStatements = DdlStatements.All;
+      schemaInfo.AllowedDdlStatements = DdlStatements.All & ~DdlStatements.Truncate;
       return schemaInfo;
     }
 
@@ -114,7 +114,7 @@ public override CheckConstraintInfo GetCheckConstraintInfo()
       checkConstraintInfo.MaxIdentifierLength = MaxIdentifierLength;
       checkConstraintInfo.MaxExpressionLength = 4000;
       checkConstraintInfo.Features = CheckConstraintFeatures.None;
-      checkConstraintInfo.AllowedDdlStatements = DdlStatements.All;
+      checkConstraintInfo.AllowedDdlStatements = DdlStatements.All & ~DdlStatements.Truncate;
       return checkConstraintInfo;
     }
 
@@ -123,7 +123,7 @@ public override PrimaryKeyConstraintInfo GetPrimaryKeyInfo()
       var primaryKeyInfo = new PrimaryKeyConstraintInfo();
       primaryKeyInfo.MaxIdentifierLength = MaxIdentifierLength;
       primaryKeyInfo.Features = PrimaryKeyConstraintFeatures.Clustered;
-      primaryKeyInfo.AllowedDdlStatements = DdlStatements.All;
+      primaryKeyInfo.AllowedDdlStatements = DdlStatements.All & ~DdlStatements.Truncate;
       return primaryKeyInfo;
     }
 
@@ -132,7 +132,7 @@ public override UniqueConstraintInfo GetUniqueConstraintInfo()
       var uniqueConstraintInfo = new UniqueConstraintInfo();
       uniqueConstraintInfo.MaxIdentifierLength = MaxIdentifierLength;
       uniqueConstraintInfo.Features = UniqueConstraintFeatures.Clustered | UniqueConstraintFeatures.Nullable;
-      uniqueConstraintInfo.AllowedDdlStatements = DdlStatements.All;
+      uniqueConstraintInfo.AllowedDdlStatements = DdlStatements.All & ~DdlStatements.Truncate;
       return uniqueConstraintInfo;
     }
 
@@ -142,7 +142,7 @@ public override IndexInfo GetIndexInfo()
       indexInfo.MaxIdentifierLength = MaxIdentifierLength;
       indexInfo.MaxNumberOfColumns = 16;
       indexInfo.MaxLength = 900;
-      indexInfo.AllowedDdlStatements = DdlStatements.All;
+      indexInfo.AllowedDdlStatements = DdlStatements.All & ~DdlStatements.Truncate;
       indexInfo.Features =
         IndexFeatures.Clustered |
         IndexFeatures.FillFactor |
@@ -163,7 +163,7 @@ public override ForeignKeyConstraintInfo GetForeignKeyConstraintInfo()
         ForeignKeyConstraintActions.Cascade |
         ForeignKeyConstraintActions.SetDefault |
         ForeignKeyConstraintActions.SetNull;
-      referenceConstraintInfo.AllowedDdlStatements = DdlStatements.All;
+      referenceConstraintInfo.AllowedDdlStatements = DdlStatements.All & ~DdlStatements.Truncate;
       return referenceConstraintInfo;
     }
 

Orm/Xtensive.Orm.SqlServer/Sql.Drivers.SqlServer/v09/ServerInfoProvider.cs

@@ -73,23 +73,23 @@ public override EntityInfo GetViewInfo()
     {
       EntityInfo viewInfo = new EntityInfo();
       viewInfo.MaxIdentifierLength = MaxIdentifierLength;
-      viewInfo.AllowedDdlStatements = DdlStatements.All;
+      viewInfo.AllowedDdlStatements = DdlStatements.All & ~DdlStatements.Truncate;
       return viewInfo;
     }
 
     public override EntityInfo GetSchemaInfo()
     {
       EntityInfo schemaInfo = new EntityInfo();
       schemaInfo.MaxIdentifierLength = MaxIdentifierLength;
-      schemaInfo.AllowedDdlStatements = DdlStatements.All;
+      schemaInfo.AllowedDdlStatements = DdlStatements.All & ~DdlStatements.Truncate;
       return schemaInfo;
     }
 
     public override TableInfo GetTableInfo()
     {
       var tableInfo = new TableInfo();
       tableInfo.MaxIdentifierLength = MaxIdentifierLength;
-      tableInfo.AllowedDdlStatements = DdlStatements.All;
+      tableInfo.AllowedDdlStatements = DdlStatements.All & ~DdlStatements.Truncate;
       tableInfo.PartitionMethods = PartitionMethods.List | PartitionMethods.Range | PartitionMethods.Hash;
       return tableInfo;
     }
@@ -98,7 +98,7 @@ public override TemporaryTableInfo GetTemporaryTableInfo()
     {
       var temporaryTableInfo = new TemporaryTableInfo();
       temporaryTableInfo.MaxIdentifierLength = MaxIdentifierLength;
-      temporaryTableInfo.AllowedDdlStatements = DdlStatements.All;
+      temporaryTableInfo.AllowedDdlStatements = DdlStatements.All & ~DdlStatements.Truncate;
       temporaryTableInfo.Features = TemporaryTableFeatures.Local;
       return temporaryTableInfo;
     }

Orm/Xtensive.Orm.Sqlite/Sql.Drivers.Sqlite/v3/ServerInfoProvider.cs

@@ -1,11 +1,14 @@
-// Copyright (C) 2008-2021 Xtensive LLC.
+// Copyright (C) 2008-2023 Xtensive LLC.
 // This code is distributed under MIT license terms.
 // See the License.txt file in the project root for more information.
 // Created by: Alex Yakunin
 // Created:    2008.02.09
 
 using System;
+using System.Collections.Generic;
 using System.Threading;
+using Xtensive.Sql.Dml;
+
 
 namespace Xtensive.Orm.Tests
 {
@@ -166,6 +169,17 @@ public static TimeOnly AdjustTimeOnlyForProvider(this TimeOnly origin, StoragePr
       var newTicks = ticks - (ticks % divider.Value);
       return new TimeOnly(newTicks);
     }
+
+    public static void AddValueRow(this SqlInsert insert, in (SqlColumn column, SqlExpression value) first, params (SqlColumn column, SqlExpression value)[] additional)
+    {
+      var additional1 = additional ?? Array.Empty<(SqlColumn,SqlExpression)>();
+      var row = new Dictionary<SqlColumn, SqlExpression>(1 + additional1.Length);
+      row.Add(first.column, first.value);
+      foreach (var keyValue in additional1) {
+        row.Add(keyValue.column, keyValue.value);
+      }
+      insert.ValueRows.Add(row);
+    }
 #endif
   }
 }

Orm/Xtensive.Orm.Tests.Framework/TestHelper.cs

@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2022 Xtensive LLC.
+// Copyright (C) 2019-2023 Xtensive LLC.
 // This code is distributed under MIT license terms.
 // See the License.txt file in the project root for more information.
 // Created by: Alexey Kulakov
@@ -325,13 +325,14 @@ private static void ExecuteInsert(SqlConnection connection, Table table, Diction
     {
       var tableRef = SqlDml.TableRef(table);
       var insertQuery = SqlDml.Insert(tableRef);
+      var row = new Dictionary<SqlColumn, SqlExpression>(values.Count);
       foreach (var nameValue in values) {
         var value = nameValue.Value != null
           ? (SqlExpression) SqlDml.Literal(nameValue.Value)
           : SqlDml.Null;
-        insertQuery.Values.Add(tableRef[nameValue.Key], value);
+        row.Add(tableRef[nameValue.Key], value);
       }
-
+      insertQuery.ValueRows.Add(row);
       using var command = connection.CreateCommand(insertQuery);
       Console.WriteLine(command.CommandText);
       _ = command.ExecuteNonQuery();

Orm/Xtensive.Orm.Tests.Sql/ChinookSchemaCreator.cs

@@ -1,9 +1,10 @@
-// Copyright (C) 2003-2010 Xtensive LLC.
-// All rights reserved.
-// For conditions of distribution and use, see license.
+// Copyright (C) 2003-2023 Xtensive LLC.
+// This code is distributed under MIT license terms.
+// See the License.txt file in the project root for more information.
 
 using NUnit.Framework;
 using System;
+using System.Linq;
 using System.Collections.Generic;
 using Xtensive.Sql;
 using Xtensive.Sql.Dml;
@@ -597,18 +598,19 @@ public void SqlInsertCloneTest()
     {
       SqlTableRef t = SqlDml.TableRef(table1);
       SqlInsert i = SqlDml.Insert(t);
-      i.Values[t[0]] = 1;
-      i.Values[t[1]] = "Anonym";
+      i.AddValueRow(( t[0], 1 ), ( t[1], "Anonym" ));
       i.Hints.Add(SqlDml.FastFirstRowsHint(10));
       SqlInsert iClone = (SqlInsert)i.Clone();
 
       Assert.AreNotEqual(i, iClone);
       Assert.AreNotEqual(i.Into, iClone.Into);
       Assert.AreEqual(i.NodeType, iClone.NodeType);
-      Assert.AreEqual(i.Values.Count, iClone.Values.Count);
-      foreach (KeyValuePair<SqlColumn, SqlExpression> p in i.Values) {
-        Assert.IsFalse(iClone.Values.ContainsKey(p.Key));
-        Assert.IsFalse(iClone.Values.ContainsValue(p.Value));
+      Assert.AreEqual(i.ValueRows.Count, iClone.ValueRows.Count);
+      Assert.AreEqual(i.ValueRows.Columns.Count, iClone.ValueRows.Columns.Count);
+      for(var indx = 0; indx < i.ValueRows.Count; indx++) {
+        var iRow = i.ValueRows[indx];
+        var iClonedRow = iClone.ValueRows[indx];
+        Assert.IsFalse(iRow.SequenceEqual(iClonedRow));
       }
       Assert.AreEqual(i.Hints.Count, iClone.Hints.Count);
     }