优化导入数据中间关系表查询，内存数据最小化拼接数据行

src/hosts/ZhonTai.Host/Configs/dbconfig.json

@@ -32,13 +32,13 @@
   "syncStructure": true,
   //同步数据，只新增数据不修改数据，想要修改数据需开启sysUpdateData
   "syncData": true,
-  //同步更新数据，注意生产环境谨慎开启，确定要修改表数据是最新数据再开启。如不想更新某些表的数据，可以先删除对应表的data.json，再执行数据更新操作
+  //同步更新数据，注意生产环境谨慎开启，确定要修改表数据是最新数据再开启。如不想更新某些表的数据，可以先配置同步数据排除表syncDataExcludeTables，再执行数据更新操作
   "sysUpdateData": false,
   //同步数据地址
   //"SyncDataPath": "InitData/Admin/Vue2",
-  //同步数据包含表
+  //同步数据包含表，指定表同步，不填同步所有表
   "syncDataIncludeTables": [],
-  //同步数据排除表
+  //同步数据排除表，指定表不同步
   "syncDataExcludeTables": [],
 
   //项目初始化不开启生成数据，发布生产环境前，如果开发环境有配置数据需要更新数据包，可以开启生成数据包，使用完记得关闭

src/platform/ZhonTai.Admin/Core/Db/Data/SyncData.cs

@@ -32,6 +32,36 @@ public abstract class SyncData
         return isIdentity;
     }
 
+    /// <summary>
+    /// 获得表名
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    /// <returns></returns>
+    protected static string GetTableName<T>() where T : class, new()
+    {
+        var table = typeof(T).GetCustomAttributes(typeof(TableAttribute), false).FirstOrDefault() as TableAttribute;
+        return table.Name;
+    }
+
+    protected static bool IsSyncData(string tableName, DbConfig dbConfig)
+    {
+        var isSyncData = true;
+
+        var hasDataIncludeTables = dbConfig.SyncDataIncludeTables?.Length > 0;
+        if (hasDataIncludeTables && !dbConfig.SyncDataIncludeTables.Contains(tableName))
+        {
+            isSyncData = false;
+        }
+
+        var hasSyncDataExcludeTables = dbConfig.SyncDataExcludeTables?.Length > 0;
+        if (hasSyncDataExcludeTables && dbConfig.SyncDataExcludeTables.Contains(tableName))
+        {
+            isSyncData = false;
+        }
+
+        return isSyncData;
+    }
+
     /// <summary>
     /// 初始化数据表数据
     /// </summary>

src/platform/ZhonTai.Admin/Repositories/CustomSyncData.cs

@@ -18,44 +18,16 @@ using ZhonTai.Admin.Core.Db.Data;
 using ZhonTai.Admin.Domain.UserOrg;
 using System.Linq;
 using ZhonTai.Common.Extensions;
-using FreeSql.DataAnnotations;
 using System;
 using FreeSql;
+using ZhonTai.Admin.Core.Auth;
+using Mapster;
+using System.Collections.Generic;
 
 namespace ZhonTai.Admin.Repositories;
 
 public class CustomSyncData : SyncData, ISyncData
 {
-    /// <summary>
-    /// 获得表名
-    /// </summary>
-    /// <typeparam name="T"></typeparam>
-    /// <returns></returns>
-    private static string GetTableName<T>() where T : class, new()
-    {
-        var table = typeof(T).GetCustomAttributes(typeof(TableAttribute), false).FirstOrDefault() as TableAttribute;
-        return table.Name;
-    }
-
-    private static bool IsSyncData(string tableName, DbConfig dbConfig)
-    {
-        var isSyncData = true;
-
-        var hasDataIncludeTables = dbConfig.SyncDataIncludeTables?.Length > 0;
-        if (hasDataIncludeTables && !dbConfig.SyncDataIncludeTables.Contains(tableName))
-        {
-            isSyncData = false;
-        }
-
-        var hasSyncDataExcludeTables = dbConfig.SyncDataExcludeTables?.Length > 0;
-        if (hasSyncDataExcludeTables && dbConfig.SyncDataExcludeTables.Contains(tableName))
-        {
-            isSyncData = false;
-        }
-
-        return isSyncData;
-    }
-
     /// <summary>
     /// 初始化字典类型
     /// </summary>
@@ -160,6 +132,7 @@ public class CustomSyncData : SyncData, ISyncData
                 var updateDataList = dataList.Where(a => recordIds.Contains(a.Id));
                 await rep.UpdateAsync(updateDataList);
             }
+
             Console.WriteLine($"table: {tableName} sync data succeed");
         }
         catch (Exception ex)
@@ -217,6 +190,7 @@ public class CustomSyncData : SyncData, ISyncData
                 var updateDataList = dataList.Where(a => recordIds.Contains(a.Id));
                 await rep.UpdateAsync(updateDataList);
             }
+
             Console.WriteLine($"table: {tableName} sync data succeed");
         }
         catch (Exception ex)
@@ -274,6 +248,7 @@ public class CustomSyncData : SyncData, ISyncData
                 var updateDataList = dataList.Where(a => recordIds.Contains(a.Id));
                 await rep.UpdateAsync(updateDataList);
             }
+
             Console.WriteLine($"table: {tableName} sync data succeed");
         }
         catch (Exception ex)
@@ -332,6 +307,7 @@ public class CustomSyncData : SyncData, ISyncData
                 var updateDataList = dataList.Where(a => recordIds.Contains(a.Id));
                 await rep.UpdateAsync(updateDataList);
             }
+
             Console.WriteLine($"table: {tableName} sync data succeed");
         }
         catch (Exception ex)
@@ -389,6 +365,7 @@ public class CustomSyncData : SyncData, ISyncData
                 var updateDataList = dataList.Where(a => recordIds.Contains(a.Id));
                 await rep.UpdateAsync(updateDataList);
             }
+
             Console.WriteLine($"table: {tableName} sync data succeed");
         }
         catch (Exception ex)
@@ -446,6 +423,7 @@ public class CustomSyncData : SyncData, ISyncData
                 var updateDataList = dataList.Where(a => recordIds.Contains(a.Id));
                 await rep.UpdateAsync(updateDataList);
             }
+
             Console.WriteLine($"table: {tableName} sync data succeed");
         }
         catch (Exception ex)
@@ -503,6 +481,7 @@ public class CustomSyncData : SyncData, ISyncData
                 var updateDataList = dataList.Where(a => recordIds.Contains(a.Id));
                 await rep.UpdateAsync(updateDataList);
             }
+
             Console.WriteLine($"table: {tableName} sync data succeed");
         }
         catch (Exception ex)
@@ -560,6 +539,7 @@ public class CustomSyncData : SyncData, ISyncData
                 var updateDataList = dataList.Where(a => recordIds.Contains(a.Id));
                 await rep.UpdateAsync(updateDataList);
             }
+
             Console.WriteLine($"table: {tableName} sync data succeed");
         }
         catch (Exception ex)
@@ -599,18 +579,15 @@ public class CustomSyncData : SyncData, ISyncData
             }
 
             //查询
-            var permissionIds = dataList.Select(a => a.PermissionId).ToList();
-            var apiIds = dataList.Select(a => a.ApiId).ToList();
-            var records = await rep.Where(a => permissionIds.Contains(a.PermissionId) && apiIds.Contains(a.ApiId)).ToListAsync();
+            var recordList = await rep.Where(a => rep.Select.WithMemory(dataList).Where(b => b.PermissionId == a.PermissionId && b.ApiId == a.ApiId).Any()).ToListAsync();
 
             //新增
-            var recordPermissionIds = dataList.Select(a => a.PermissionId).ToList();
-            var recordApiIds = dataList.Select(a => a.ApiId).ToList();
-            var insertDataList = dataList.Where(a => !(recordPermissionIds.Contains(a.PermissionId) && recordApiIds.Contains(a.ApiId)));
+            var insertDataList = dataList.Where(a => !recordList.Where(b => a.PermissionId == b.PermissionId && a.ApiId == b.ApiId).Any()).ToList();
             if (insertDataList.Any())
             {
                 await rep.InsertAsync(insertDataList);
             }
+
             Console.WriteLine($"table: {tableName} sync data succeed");
         }
         catch (Exception ex)
@@ -620,6 +597,13 @@ public class CustomSyncData : SyncData, ISyncData
         }
     }
 
+    /// <summary>
+    /// 用户角色记录
+    /// </summary>
+    /// <param name="UserId"></param>
+    /// <param name="RoleId"></param>
+    record UserRoleRecord(long UserId, long RoleId);
+
     /// <summary>
     /// 初始化用户角色
     /// </summary>
@@ -650,18 +634,16 @@ public class CustomSyncData : SyncData, ISyncData
             }
 
             //查询
-            var userIds = dataList.Select(a => a.UserId).ToList();
-            var roleIds = dataList.Select(a => a.RoleId).ToList();
-            var records = await rep.Where(a => userIds.Contains(a.UserId) && roleIds.Contains(a.RoleId)).ToListAsync();
+            var userRoleRecordList = dataList.Adapt<List<UserRoleRecord>>();
+            var recordList = await rep.Where(a => rep.Orm.Select<UserRoleRecord>().WithMemory(userRoleRecordList).Where(b => b.UserId == a.UserId && b.RoleId == a.RoleId).Any()).ToListAsync();
 
             //新增
-            var recordUserIds = dataList.Select(a => a.UserId).ToList();
-            var recordRoleIds = dataList.Select(a => a.RoleId).ToList();
-            var insertDataList = dataList.Where(a => !(recordUserIds.Contains(a.UserId) && recordRoleIds.Contains(a.RoleId)));
+            var insertDataList = dataList.Where(a => !recordList.Where(b => a.UserId == b.UserId && a.RoleId == b.RoleId).Any()).ToList();
             if (insertDataList.Any())
             {
                 await rep.InsertAsync(insertDataList);
             }
+
             Console.WriteLine($"table: {tableName} sync data succeed");
         }
         catch (Exception ex)
@@ -671,6 +653,13 @@ public class CustomSyncData : SyncData, ISyncData
         }
     }
 
+    /// <summary>
+    /// 用户部门记录
+    /// </summary>
+    /// <param name="UserId"></param>
+    /// <param name="OrgId"></param>
+    record UserOrgRecord(long UserId, long OrgId);
+
     /// <summary>
     /// 初始化用户部门
     /// </summary>
@@ -701,18 +690,16 @@ public class CustomSyncData : SyncData, ISyncData
             }
 
             //查询
-            var userIds = dataList.Select(a => a.UserId).ToList();
-            var orgIds = dataList.Select(a => a.OrgId).ToList();
-            var records = await rep.Where(a => userIds.Contains(a.UserId) && orgIds.Contains(a.OrgId)).ToListAsync();
+            var userOrgRecordList = dataList.Adapt<List<UserOrgRecord>>();
+            var recordList = await rep.Where(a => rep.Orm.Select<UserOrgRecord>().WithMemory(userOrgRecordList).Where(b => b.UserId == a.UserId && b.OrgId == a.OrgId).Any()).ToListAsync();
 
             //新增
-            var recordUserIds = dataList.Select(a => a.UserId).ToList();
-            var recordOrgIds = dataList.Select(a => a.OrgId).ToList();
-            var insertDataList = dataList.Where(a => !(recordUserIds.Contains(a.UserId) && recordOrgIds.Contains(a.OrgId)));
+            var insertDataList = dataList.Where(a => !recordList.Where(b => a.UserId == b.UserId && a.OrgId == b.OrgId).Any()).ToList();
             if (insertDataList.Any())
             {
                 await rep.InsertAsync(insertDataList);
             }
+
             Console.WriteLine($"table: {tableName} sync data succeed");
         }
         catch (Exception ex)
@@ -722,6 +709,13 @@ public class CustomSyncData : SyncData, ISyncData
         }
     }
 
+    /// <summary>
+    /// 角色权限记录
+    /// </summary>
+    /// <param name="RoleId"></param>
+    /// <param name="PermissionId"></param>
+    record RolePermissionRecord(long RoleId, long PermissionId);
+
     /// <summary>
     /// 初始化角色权限
     /// </summary>
@@ -752,18 +746,16 @@ public class CustomSyncData : SyncData, ISyncData
             }
 
             //查询
-            var roleIds = dataList.Select(a => a.RoleId).ToList();
-            var permissionIds = dataList.Select(a => a.PermissionId).ToList();
-            var records = await rep.Where(a => roleIds.Contains(a.RoleId) && permissionIds.Contains(a.PermissionId)).ToListAsync();
+            var rolePermissionRecordList = dataList.Adapt<List<RolePermissionRecord>>();
+            var recordList = await rep.Where(a => rep.Orm.Select<RolePermissionRecord>().WithMemory(rolePermissionRecordList).Where(b => b.RoleId == a.RoleId && b.PermissionId == a.PermissionId).Any()).ToListAsync();
 
             //新增
-            var recordRoleIds = dataList.Select(a => a.RoleId).ToList();
-            var recordPermissionIds = dataList.Select(a => a.PermissionId).ToList();
-            var insertDataList = dataList.Where(a => !(recordRoleIds.Contains(a.RoleId) && recordPermissionIds.Contains(a.PermissionId)));
+            var insertDataList = dataList.Where(a => !recordList.Where(b => a.RoleId == b.RoleId && a.PermissionId == b.PermissionId).Any()).ToList();
             if (insertDataList.Any())
             {
                 await rep.InsertAsync(insertDataList);
             }
+
             Console.WriteLine($"table: {tableName} sync data succeed");
         }
         catch (Exception ex)
@@ -820,6 +812,7 @@ public class CustomSyncData : SyncData, ISyncData
                 var updateDataList = dataList.Where(a => recordIds.Contains(a.Id));
                 await rep.UpdateAsync(updateDataList);
             }
+
             Console.WriteLine($"table: {tableName} sync data succeed");
         }
         catch (Exception ex)
@@ -829,6 +822,13 @@ public class CustomSyncData : SyncData, ISyncData
         }
     }
 
+    /// <summary>
+    /// 租户权限记录
+    /// </summary>
+    /// <param name="TenantId"></param>
+    /// <param name="PermissionId"></param>
+    record TenantPermissionRecord(long TenantId, long PermissionId);
+
     /// <summary>
     /// 初始化租户权限
     /// </summary>
@@ -859,18 +859,16 @@ public class CustomSyncData : SyncData, ISyncData
             }
 
             //查询
-            var tenantIds = dataList.Select(a => a.TenantId).ToList();
-            var permissionIds = dataList.Select(a => a.PermissionId).ToList();
-            var records = await rep.Where(a => tenantIds.Contains(a.TenantId) && permissionIds.Contains(a.PermissionId)).ToListAsync();
+            var tenantPermissionRecordList = dataList.Adapt<List<TenantPermissionRecord>>();
+            var recordList = await rep.Where(a => rep.Orm.Select<TenantPermissionRecord>().WithMemory(tenantPermissionRecordList).Where(b => b.TenantId == a.TenantId && b.PermissionId == a.PermissionId).Any()).ToListAsync();
 
             //新增
-            var recordTenantIds = dataList.Select(a => a.TenantId).ToList();
-            var recordPermissionIds = dataList.Select(a => a.PermissionId).ToList();
-            var insertDataList = dataList.Where(a => !(tenantIds.Contains(a.TenantId) && recordPermissionIds.Contains(a.PermissionId)));
+            var insertDataList = dataList.Where(a => !recordList.Where(b => a.TenantId == b.TenantId && a.PermissionId == b.PermissionId).Any()).ToList();
             if (insertDataList.Any())
             {
                 await rep.InsertAsync(insertDataList);
             }
+
             Console.WriteLine($"table: {tableName} sync data succeed");
         }
         catch (Exception ex)

src/platform/ZhonTai.Admin/ZhonTai.Admin.xml

@@ -1057,6 +1057,13 @@
             <typeparam name="T"></typeparam>
             <returns></returns>
         </member>
+        <member name="M:ZhonTai.Admin.Core.Db.Data.SyncData.GetTableName``1">
+            <summary>
+            获得表名
+            </summary>
+            <typeparam name="T"></typeparam>
+            <returns></returns>
+        </member>
         <member name="M:ZhonTai.Admin.Core.Db.Data.SyncData.InitDataAsync``1(IFreeSql,System.Data.Common.DbTransaction,``0[],ZhonTai.Admin.Core.Configs.DbConfig)">
             <summary>
             初始化数据表数据
@@ -3582,13 +3589,6 @@
             </summary>
             <typeparam name="TEntity"></typeparam>
         </member>
-        <member name="M:ZhonTai.Admin.Repositories.CustomSyncData.GetTableName``1">
-            <summary>
-            获得表名
-            </summary>
-            <typeparam name="T"></typeparam>
-            <returns></returns>
-        </member>
         <member name="M:ZhonTai.Admin.Repositories.CustomSyncData.InitDictTypeAsync(IFreeSql,FreeSql.IRepositoryUnitOfWork,ZhonTai.Admin.Core.Configs.DbConfig)">
             <summary>
             初始化字典类型
@@ -3683,6 +3683,26 @@
             <param name="dbConfig"></param>
             <returns></returns>
         </member>
+        <member name="T:ZhonTai.Admin.Repositories.CustomSyncData.UserRoleRecord">
+            <summary>
+            用户角色记录
+            </summary>
+            <param name="UserId"></param>
+            <param name="RoleId"></param>
+        </member>
+        <member name="M:ZhonTai.Admin.Repositories.CustomSyncData.UserRoleRecord.#ctor(System.Int64,System.Int64)">
+            <summary>
+            用户角色记录
+            </summary>
+            <param name="UserId"></param>
+            <param name="RoleId"></param>
+        </member>
+        <member name="P:ZhonTai.Admin.Repositories.CustomSyncData.UserRoleRecord.UserId">
+            <summary></summary>
+        </member>
+        <member name="P:ZhonTai.Admin.Repositories.CustomSyncData.UserRoleRecord.RoleId">
+            <summary></summary>
+        </member>
         <member name="M:ZhonTai.Admin.Repositories.CustomSyncData.InitUserRoleAsync(IFreeSql,FreeSql.IRepositoryUnitOfWork,ZhonTai.Admin.Core.Configs.DbConfig)">
             <summary>
             初始化用户角色
@@ -3692,6 +3712,26 @@
             <param name="dbConfig"></param>
             <returns></returns>
         </member>
+        <member name="T:ZhonTai.Admin.Repositories.CustomSyncData.UserOrgRecord">
+            <summary>
+            用户部门记录
+            </summary>
+            <param name="UserId"></param>
+            <param name="OrgId"></param>
+        </member>
+        <member name="M:ZhonTai.Admin.Repositories.CustomSyncData.UserOrgRecord.#ctor(System.Int64,System.Int64)">
+            <summary>
+            用户部门记录
+            </summary>
+            <param name="UserId"></param>
+            <param name="OrgId"></param>
+        </member>
+        <member name="P:ZhonTai.Admin.Repositories.CustomSyncData.UserOrgRecord.UserId">
+            <summary></summary>
+        </member>
+        <member name="P:ZhonTai.Admin.Repositories.CustomSyncData.UserOrgRecord.OrgId">
+            <summary></summary>
+        </member>
         <member name="M:ZhonTai.Admin.Repositories.CustomSyncData.InitUserOrgAsync(IFreeSql,FreeSql.IRepositoryUnitOfWork,ZhonTai.Admin.Core.Configs.DbConfig)">
             <summary>
             初始化用户部门
@@ -3701,6 +3741,26 @@
             <param name="dbConfig"></param>
             <returns></returns>
         </member>
+        <member name="T:ZhonTai.Admin.Repositories.CustomSyncData.RolePermissionRecord">
+            <summary>
+            角色权限记录
+            </summary>
+            <param name="RoleId"></param>
+            <param name="PermissionId"></param>
+        </member>
+        <member name="M:ZhonTai.Admin.Repositories.CustomSyncData.RolePermissionRecord.#ctor(System.Int64,System.Int64)">
+            <summary>
+            角色权限记录
+            </summary>
+            <param name="RoleId"></param>
+            <param name="PermissionId"></param>
+        </member>
+        <member name="P:ZhonTai.Admin.Repositories.CustomSyncData.RolePermissionRecord.RoleId">
+            <summary></summary>
+        </member>
+        <member name="P:ZhonTai.Admin.Repositories.CustomSyncData.RolePermissionRecord.PermissionId">
+            <summary></summary>
+        </member>
         <member name="M:ZhonTai.Admin.Repositories.CustomSyncData.InitRolePermissionAsync(IFreeSql,FreeSql.IRepositoryUnitOfWork,ZhonTai.Admin.Core.Configs.DbConfig)">
             <summary>
             初始化角色权限
@@ -3719,6 +3779,26 @@
             <param name="dbConfig"></param>
             <returns></returns>
         </member>
+        <member name="T:ZhonTai.Admin.Repositories.CustomSyncData.TenantPermissionRecord">
+            <summary>
+            租户权限记录
+            </summary>
+            <param name="TenantId"></param>
+            <param name="PermissionId"></param>
+        </member>
+        <member name="M:ZhonTai.Admin.Repositories.CustomSyncData.TenantPermissionRecord.#ctor(System.Int64,System.Int64)">
+            <summary>
+            租户权限记录
+            </summary>
+            <param name="TenantId"></param>
+            <param name="PermissionId"></param>
+        </member>
+        <member name="P:ZhonTai.Admin.Repositories.CustomSyncData.TenantPermissionRecord.TenantId">
+            <summary></summary>
+        </member>
+        <member name="P:ZhonTai.Admin.Repositories.CustomSyncData.TenantPermissionRecord.PermissionId">
+            <summary></summary>
+        </member>
         <member name="M:ZhonTai.Admin.Repositories.CustomSyncData.InitTenantPermissionAsync(IFreeSql,FreeSql.IRepositoryUnitOfWork,ZhonTai.Admin.Core.Configs.DbConfig)">
             <summary>
             初始化租户权限