michal/tbfDBBackup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add test_Block_LageInsert.sql resource for database tests

Browse Source 
Introduced a new SQL test resource under `tbfDBBackup.Tests` to support database testing scenarios involving procedures and history operations.
This commit is contained in:
Michal Buzik 2025-09-19 08:04:40 +02:00
parent 2b86358a57
commit a00994cd5c
11 changed files with 768 additions and 292 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tbfDBBackup.Tests/Resources/schema-mapping.json
View File

@ -820,6 +820,7 @@
"ColumnDef": {
"Name": "Id",
"Type": "int(11)",
"IsPrimaryKey": true,
"Nullable": "NO"
}
},
@ -886,6 +887,7 @@
"ColumnDef": {
"Name": "Id",
"Type": "int(11)",
"IsPrimaryKey": true,
"Nullable": "NO"
}
},
@ -934,6 +936,7 @@
"ColumnDef": {
"Name": "Id",
"Type": "int(11)",
"IsPrimaryKey": true,
"Nullable": "NO"
}
},
@ -1024,6 +1027,7 @@
"ColumnDef": {
"Name": "Id",
"Type": "int(11)",
"IsPrimaryKey": true,
"Nullable": "NO"
}
},

1
tbfDBBackup.Tests/Resources/test_Block_LageInsert.sql Normal file
View File

File diff suppressed because one or more lines are too long

30
tbfDBBackup.Tests/Resources/test_Block_LargeCreate.sql Normal file
View File

@ -0,0 +1,30 @@
CREATE TABLE `procedure` (
`Id` int(11) NOT NULL AUTO_INCREMENT,
`ItemNr` int(11) DEFAULT NULL,
`Name` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`Description` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`CreationUser` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`CreationTime` datetime DEFAULT NULL,
`LastChgUser` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`LastChgTime` datetime DEFAULT NULL,
`LongDescription` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`ProcedureState` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`Revision` int(11) DEFAULT NULL,
`PredecessorId` int(11) DEFAULT NULL,
`ObtainedByCopy` tinyint(1) DEFAULT NULL,
`MetersKind` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`Watermeters` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`ProtocolTitle` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`Protected` tinyint(1) NOT NULL DEFAULT '0',
`MustBeComplete` tinyint(1) NOT NULL DEFAULT '0',
`ManualCtrlDisabled` tinyint(1) NOT NULL DEFAULT '0',
`DataEntry` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`ResultsPrinter` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`ResultsWriter` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`EventTriggers` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`TransitionStart` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`TransitionEnd` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`AltProcName` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`AltProcPeriod` int(11) NOT NULL DEFAULT '0',
PRIMARY KEY (`Id`)
) ENGINE=InnoDB AUTO_INCREMENT=4549 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;

65
tbfDBBackup.Tests/compare_dumps/Parserer/SqlDumpFileParserTest.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using JetBrains.Annotations;
@ -60,12 +61,25 @@ public class SqlDumpFileParserTest
public void SqlDumpParserFile_2()
{
string basePath = GetProjectRoot();
string sql = Path.Combine(basePath,"Resources/slm50-250709-1549.sql");
string sql = Path.Combine(basePath,"Resources","slm50-250709-1549.sql");
ValidateSqlDumpParserResults(sql);
ValidateSqlDumpParserResults(sql, false);
SqlDumpFileParser parser = new();
parser.SqlDumpParserFile(sql);
List<ForeignKeyDef> foreignKeyDefList = new();
foreach (var parserTableDefinition in parser.TableDefinitions)
{
List<ForeignKeyDef> foreignKeyDefs = parser.GetAllForeignKeysByTableName(parserTableDefinition.Key);
Console.WriteLine($" Table {parserTableDefinition.Key} has foreign keys count: {foreignKeyDefs.Count}");
foreignKeyDefList.AddRange(foreignKeyDefs);
}
Assert.IsTrue(foreignKeyDefList.Count > 0);
}
private static void ValidateSqlDumpParserResults(string sql)
private static void ValidateSqlDumpParserResults(string sql, bool testForineKeys = true)
{
SqlDumpFileParser parser = new();
parser.SqlDumpParserFile(sql);
@ -95,26 +109,41 @@ public class SqlDumpFileParserTest
Assert.IsTrue(lastCharacters.Contains(';'));
}
}
Assert.IsTrue(parser.TableForeignKeys.Count > 1);
foreach (var table in parser.TableForeignKeys)
if (testForineKeys)
{
Console.WriteLine(table.Key);
foreach (var row in table.Value)
Assert.IsTrue(parser.TableForeignKeys.Count > 1);
foreach (var table in parser.TableForeignKeys)
{
//check if it contains alter table
Assert.IsTrue(row.Substring(0, Math.Min(20, row.Length-1)).Contains("ALTER TABLE"));
//check rest of string for alter table - non valid sql
string restString = row.Substring(20);
if (restString.Contains("ALTER TABLE"))
Console.WriteLine(table.Key);
foreach (var row in table.Value)
{
Assert.Fail($"Alter contains insert more than once, see: {row}");
//check if it contains alter table
Assert.IsTrue(row.Substring(0, Math.Min(20, row.Length - 1)).Contains("ALTER TABLE"));
//check rest of string for alter table - non valid sql
string restString = row.Substring(20);
if (restString.Contains("ALTER TABLE"))
{
Assert.Fail($"Alter contains insert more than once, see: {row}");
}
//check if it ends with ;
string lastCharacters = row.Length >= 5 ? row.Substring(row.Length - 5) : "";
Assert.IsTrue(lastCharacters.Contains(';'));
}
//check if it ends with ;
string lastCharacters = row.Length >= 5 ? row.Substring(row.Length - 5) : "";
Assert.IsTrue(lastCharacters.Contains(';'));
}
}
}
[TestMethod]
public void ExtractForeignKeysFromCreateTable_Test()
{
string sql = "CREATE TABLE `usersgroups` (\n`Group_id` int(11) NOT NULL,\n`User_id` int(11) NOT NULL,\nKEY `User_id` (`User_id`),\nKEY `Group_id` (`Group_id`),\nCONSTRAINT `FKEC3AF23373767C99` FOREIGN KEY (`User_id`) REFERENCES `user` (`Id`),\nCONSTRAINT `FKEC3AF23393C4061C` FOREIGN KEY (`Group_id`) REFERENCES `group` (`Id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;";
List<ForeignKeyDef> list = SqlDumpFileParser.ExtractForeignKeysFromCreateTable(sql);
list.ForEach(FK => Console.WriteLine(FK.ToString()));
Assert.IsTrue(list.Count == 2);
}
}

39
tbfDBBackup.Tests/compare_dumps/SqlDumpParserTest.cs
View File

@ -51,6 +51,25 @@ public class SqlDumpParserTest
Console.WriteLine($" {col.Key} = {col.Value.Substring(0, Math.Min(80, col.Value.Length))}...");
}
}
[TestMethod]
public void ParseInsertBlock_File_LargeInsert()
{
string projectRoot = GetProjectRoot();
string sql = File.ReadAllText( Path.Combine(projectRoot, "Resources","test_Block_LageInsert.sql"));
string sqlCreate = File.ReadAllText( Path.Combine(projectRoot, "Resources","test_Block_LargeCreate.sql"));
var parsed = SqlDumpParser.ParseInsertBlock(sql,sqlCreate);
foreach (var kvp in parsed)
{
Console.WriteLine($"ID {kvp.Key}:");
//Get first 50 chars for all colums
foreach (var col in kvp.Value)
Console.WriteLine($" {col.Key} = {col.Value.Substring(0, Math.Min(80, col.Value.Length))}...");
}
Assert.AreEqual(4539, parsed.Last().Key);
}
[TestMethod]
public void ParseForeignKeys()
@ -68,4 +87,24 @@ public class SqlDumpParserTest
Assert.AreEqual("procedure", foreignKeys.First().Value.First().RefTable);
Assert.AreEqual("Id", foreignKeys.First().Value.First().RefColumn);
}
[TestMethod]
public void ParseCreateSqlToRows()
{
string slqCreate = "CREATE TABLE `componentprocedure` (\n `Id` int NOT NULL AUTO_INCREMENT,\n `CmpntName` varchar(255) NOT NULL,\n `Parameters` longtext NOT NULL,\n `Procedure_id` int NOT NULL,\n PRIMARY KEY (`Id`),\n KEY `IDX_F52C29EE1EC52E2B` (`Procedure_id`),\n CONSTRAINT `FKF52C29EE1EC52E2B` FOREIGN KEY (`Procedure_id`) REFERENCES `procedure` (`Id`)\n) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;";
string[] columns = SqlDumpParser.ParseCreateSqlToRows(slqCreate);
Assert.AreEqual(4, columns.Length);
}
[TestMethod]
public void ParseCreateSqlToRows2()
{
string slqCreate = "CREATE TABLE procedure ( Id int(11) NOT NULL AUTO_INCREMENT, ItemNr int(11) DEFAULT NULL, Name varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, Description varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, CreationUser varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, CreationTime datetime DEFAULT NULL, LastChgUser varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, LastChgTime datetime DEFAULT NULL, LongDescription varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, ProcedureState varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, Revision int(11) DEFAULT NULL, PredecessorId int(11) DEFAULT NULL, ObtainedByCopy tinyint(1) DEFAULT NULL, MetersKind varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, Watermeters varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, ProtocolTitle varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, Protected tinyint(1) NOT NULL DEFAULT '0', MustBeComplete tinyint(1) NOT NULL DEFAULT '0', ManualCtrlDisabled tinyint(1) NOT NULL DEFAULT '0', DataEntry varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, ResultsPrinter varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, ResultsWriter varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, EventTriggers varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, TransitionStart varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, TransitionEnd varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, AltProcName varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL, AltProcPeriod int(11) NOT NULL DEFAULT '0', PRIMARY KEY (Id) ) ENGINE=InnoDB AUTO_INCREMENT=4549 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;";
string[] columns = SqlDumpParser.ParseCreateSqlToRows(slqCreate);
Assert.AreEqual(27, columns.Length);
}
}

355
tbfDBBackup/compare_dumps/Parserer/SqlDumpFileParser.cs
View File

@ -5,79 +5,45 @@ namespace tbfDBBackup.compare_dumps.Parserer;
public class SqlDumpFileParser
{
public Dictionary<string, string> TableDefinitions { get;set; }
public Dictionary<string, List<string>> TableInserts { get;set; }
public Dictionary<string, string> TableDefinitions { get; private set; } = new();
public Dictionary<string, List<string>> TableInserts { get; private set; } = new();
public Dictionary<string, List<string>> TableForeignKeys { get; private set; } = new();
public Dictionary<string, List<ForeignKeyDef>> TableForeignKeysObj { get; private set; } = new();
public void SqlDumpParserFile(string filePath)
{
TableDefinitions = new Dictionary<string, string>();
TableInserts = new Dictionary<string, List<string>>();
TableForeignKeys = new Dictionary<string, List<string>>();
TableDefinitions.Clear();
TableInserts.Clear();
TableForeignKeys.Clear();
TableForeignKeysObj.Clear();
SqlSection currentSection = SqlSection.None;
StringBuilder blockBuilder = new StringBuilder();
string currentTable = null;
string? currentTable = null;
foreach (var line in File.ReadLines(filePath))
foreach (var rawLine in File.ReadLines(filePath))
{
// Normalize line
var trimmed = line.Trim();
if (trimmed.StartsWith("CREATE TABLE", StringComparison.OrdinalIgnoreCase))
var line = rawLine.Trim();
// --- Skip comments & MySQL directives
if (string.IsNullOrWhiteSpace(line) ||
line.StartsWith("--") ||
line.StartsWith("/*") ||
line.StartsWith("/*!"))
{
currentSection = SqlSection.CreateTable;
blockBuilder.Clear();
blockBuilder.AppendLine(line);
// Extract table name
var match = Regex.Match(trimmed, @"CREATE TABLE(?: IF NOT EXISTS)? `(\w+)`", RegexOptions.IgnoreCase);
if (match.Success)
currentTable = match.Groups[1].Value;
continue;
}
if (trimmed.StartsWith("INSERT INTO", StringComparison.OrdinalIgnoreCase))
{
currentSection = SqlSection.InsertInto;
blockBuilder.Clear();
blockBuilder.AppendLine(line);
var match = Regex.Match(trimmed, @"INSERT INTO `(\w+)`", RegexOptions.IgnoreCase);
if (match.Success)
currentTable = match.Groups[1].Value;
continue;
}
// --- ALTER TABLE (foreign keys, constraints, etc.) ---
if (trimmed.StartsWith("ALTER TABLE", StringComparison.OrdinalIgnoreCase))
{
currentSection = SqlSection.AlterTable;
blockBuilder.Clear();
blockBuilder.AppendLine(line);
var match = Regex.Match(trimmed, @"ALTER TABLE `(\w+)`", RegexOptions.IgnoreCase);
if (match.Success)
currentTable = match.Groups[1].Value;
continue;
}
// ------------------------
// Accumulate CREATE TABLE
// ------------------------
if (currentSection == SqlSection.CreateTable)
{
blockBuilder.AppendLine(line);
if (trimmed.EndsWith(";"))
if (line.EndsWith(";"))
{
// Save table schema
if (currentTable != null)
TableDefinitions[currentTable] = blockBuilder.ToString();
currentSection = SqlSection.None;
currentSection = StoreCurrentTableSchema(currentTable, blockBuilder);
}
}
// ------------------------
@ -86,17 +52,9 @@ public class SqlDumpFileParser
else if (currentSection == SqlSection.InsertInto)
{
blockBuilder.AppendLine(line);
if (trimmed.EndsWith(";"))
if (line.EndsWith(";"))
{
// Save INSERT block
if (currentTable != null)
{
if (!TableInserts.ContainsKey(currentTable))
TableInserts[currentTable] = new List<string>();
TableInserts[currentTable].Add(blockBuilder.ToString());
}
currentSection = SqlSection.None;
currentSection = StoreCurrentTableInsert(currentTable, blockBuilder);
}
}
// ------------------------
@ -105,18 +63,273 @@ public class SqlDumpFileParser
else if (currentSection == SqlSection.AlterTable)
{
blockBuilder.AppendLine(line);
if (trimmed.EndsWith(";"))
if (line.EndsWith(";"))
{
if (currentTable != null)
{
if (!TableForeignKeys.ContainsKey(currentTable))
TableForeignKeys[currentTable] = new List<string>();
TableForeignKeys[currentTable].Add(blockBuilder.ToString());
}
currentSection = SqlSection.None;
currentSection = StoreTableForeignKey(currentTable, blockBuilder);
}
}
// --- DROP TABLE: just skip
if (line.StartsWith("DROP TABLE", StringComparison.OrdinalIgnoreCase))
{
continue;
}
// --- LOCK TABLES: just skip
if (line.StartsWith("LOCK TABLES", StringComparison.OrdinalIgnoreCase))
{
continue;
}
// --- UNLOCK TABLES; just skip
if (line.StartsWith("UNLOCK TABLES", StringComparison.OrdinalIgnoreCase))
{
continue;
}
if (line.StartsWith("CREATE TABLE", StringComparison.OrdinalIgnoreCase))
{
if (currentSection != SqlSection.None)
{
Console.WriteLine($"Warning: CREATE TABLE found in wrong block {currentSection}");
}
currentSection = SqlSection.CreateTable;
blockBuilder.Clear();
blockBuilder.AppendLine(line);
// Extract table name
var match = Regex.Match(line, @"CREATE TABLE(?: IF NOT EXISTS)? `(\w+)`", RegexOptions.IgnoreCase);
if (match.Success)
currentTable = match.Groups[1].Value;
if (match.Success && line.EndsWith(";"))
{
currentSection = StoreCurrentTableSchema(currentTable, blockBuilder);
}
continue;
}
if (line.StartsWith("INSERT INTO", StringComparison.OrdinalIgnoreCase))
{
if (currentSection != SqlSection.None)
{
Console.WriteLine($"Warning: INSERT INTO found in wrong block {currentSection}");
}
string fistPartLine = line.Substring(0,((line.Length > 200) ? 200 : (line.Length-1)));
// Robust table name extraction:
// Matches:
// INSERT INTO table ...
// INSERT INTO `table` ...
// INSERT INTO db.table ...
// INSERT INTO `db`.`table` ...
var match = Regex.Match(fistPartLine, @"^\s*INSERT\s+INTO\s+(?:`[^`]+`\.|[^`\s]+\.)?`?([^\s`(]+)`?",
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
currentTable = match.Success ? match.Groups[1].Value : null;
// Fast path: single-line INSERT, avoid StringBuilder entirely
if (line.Length > 0 && line[^1] == ';')
{
currentSection = StoreCurrentTableInsert(currentTable, line);
continue;
}
// Multi-line INSERT accumulation
currentSection = SqlSection.InsertInto;
blockBuilder.Clear();
blockBuilder.AppendLine(line);
continue;
}
// --- ALTER TABLE (foreign keys, constraints, etc.) ---
if (line.StartsWith("ALTER TABLE", StringComparison.OrdinalIgnoreCase))
{
if (currentSection != SqlSection.None)
{
Console.WriteLine($"Warning: ALTER TABLE found in wrong block {currentSection}");
}
currentSection = SqlSection.AlterTable;
blockBuilder.Clear();
blockBuilder.AppendLine(line);
var match = Regex.Match(line, @"ALTER TABLE `(\w+)`", RegexOptions.IgnoreCase);
if (match.Success)
currentTable = match.Groups[1].Value;
if (match.Success && line.EndsWith(";"))
{
currentSection = StoreTableForeignKey(currentTable, blockBuilder);
}
continue;
}
}
RunExtractForeignKeysFromCreateTableList();
}
private SqlSection StoreCurrentTableSchema(string? currentTable, StringBuilder blockBuilder)
{
SqlSection currentSection;
// Save table schema
if (currentTable != null)
{
TableDefinitions[currentTable] = blockBuilder.ToString();
}
currentSection = SqlSection.None;
return currentSection;
}
private SqlSection StoreCurrentTableInsert(string? currentTable, StringBuilder blockBuilder)
{
// Save INSERT block
if (currentTable != null)
{
if (!TableInserts.ContainsKey(currentTable))
TableInserts[currentTable] = new List<string>();
TableInserts[currentTable].Add(blockBuilder.ToString());
}
return SqlSection.None;
}
// Fast path overload for single-line INSERTs (no StringBuilder allocation)
private SqlSection StoreCurrentTableInsert(string? currentTable, string line)
{
if (currentTable != null)
{
if (!TableInserts.ContainsKey(currentTable))
TableInserts[currentTable] = new List<string>();
TableInserts[currentTable].Add(line);
}
return SqlSection.None;
}
private SqlSection StoreTableForeignKey(string? currentTable, StringBuilder blockBuilder)
{
SqlSection currentSection;
if (currentTable != null)
{
if (!TableForeignKeys.ContainsKey(currentTable))
TableForeignKeys[currentTable] = new List<string>();
TableForeignKeys[currentTable].Add(blockBuilder.ToString());
}
currentSection = SqlSection.None;
return currentSection;
}
public void RunExtractForeignKeysFromCreateTableList()
{
foreach (var parserTableDefinition in TableDefinitions)
{
List<ForeignKeyDef> foreignKeyDefs = ExtractForeignKeysFromCreateTable(parserTableDefinition.Value);
foreignKeyDefs.ForEach(FK =>
{
if (!TableForeignKeysObj.TryGetValue(parserTableDefinition.Key, out List<ForeignKeyDef>? foreignKeyConstraints))
{
foreignKeyConstraints = new List<ForeignKeyDef>();
TableForeignKeysObj[parserTableDefinition.Key] = foreignKeyConstraints;
}
foreignKeyConstraints.Add(FK);
});
}
}
public static List<ForeignKeyDef> ExtractForeignKeysFromCreateTable(string createTableSql)
{
var result = new List<ForeignKeyDef>();
// Regex to capture foreign key constraints inside CREATE TABLE
var regex = new Regex(
@"CONSTRAINT\s+`(?<constraint>\w+)`\s+FOREIGN\s+KEY\s*\(`(?<column>\w+)`\)\s+REFERENCES\s+`(?<refTable>\w+)`\s*\(`(?<refColumn>\w+)`\)",
RegexOptions.IgnoreCase | RegexOptions.Multiline);
foreach (Match match in regex.Matches(createTableSql))
{
result.Add(new ForeignKeyDef
{
Name = match.Groups["constraint"].Value,
Column = match.Groups["column"].Value,
RefTable = match.Groups["refTable"].Value,
RefColumn = match.Groups["refColumn"].Value
});
}
return result;
}
private List<ForeignKeyDef> ExtractForeignKeysFromForeignKeysByTableName(string TableName)
{
bool tryGetForeignerColums =
TableForeignKeys.TryGetValue(TableName, out List<string>? oldForeignerTableNames);
List<ForeignKeyDef> foreignKeys = new List<ForeignKeyDef>();
if (tryGetForeignerColums)
{
oldForeignerTableNames.ForEach(ForeignKeyConstraintRow =>
{
Dictionary<string, List<ForeignKeyDef>> foreignerList =
SqlDumpParser.ParseForeignKeys(ForeignKeyConstraintRow);
if (foreignerList.Count == 1) // we expecting only one foreign key definition per table
{
var keyValuePair = foreignerList.First().Value;
foreach (var combination in keyValuePair)
{
foreignKeys.Add(combination);
}
}
}
);
}
return foreignKeys;
}
public List<ForeignKeyDef> GetAllForeignKeysByTableName(string TableName)
{
List<ForeignKeyDef> allForeignKeys = new List<ForeignKeyDef>();
List<ForeignKeyDef> keysByTableNameFromForineKeys = ExtractForeignKeysFromForeignKeysByTableName(TableName);
foreach (var keyValuePair in TableForeignKeysObj)
{
if (keyValuePair.Key == TableName)
{
allForeignKeys.AddRange(keyValuePair.Value);
}
}
if (keysByTableNameFromForineKeys.Count == 0)
return allForeignKeys;
if (allForeignKeys.Count == 0)
return keysByTableNameFromForineKeys;
//TODO combine foreign keys from table and foreign keys from foreign keys
foreach (ForeignKeyDef keyDef in keysByTableNameFromForineKeys)
{
//check if key already exist in allForeignKeys
bool bKeyExisted = false;
foreach (ForeignKeyDef keyExisted in allForeignKeys)
{
if (keyExisted.RefColumn == keyDef.RefColumn && keyExisted.RefTable == keyDef.RefTable)
{
bKeyExisted = true;
}
}
if (!bKeyExisted)
allForeignKeys.Add(keyDef);
}
return allForeignKeys;
}
}

260
tbfDBBackup/compare_dumps/Parserer/SqlDumpParser.cs
View File

@ -7,24 +7,59 @@ namespace tbfDBBackup.compare_dumps;
public class SqlDumpParser
{
public static Dictionary<int, Dictionary<string, string>> ParseInsertBlock(string sql)
private static readonly Regex InsertWithColsRegex = new Regex(
@"INSERT INTO\s+`(?<table>\w+)`\s*\((?<columns>[^)]+)\)\s*VALUES\s*(?<values>.+);",
RegexOptions.Singleline | RegexOptions.IgnoreCase);
private static readonly Regex InsertNoColsRegex = new Regex(
@"INSERT\s+INTO\s+`(?<table>\w+)`\s*VALUES\s*(?<values>.+?);",
RegexOptions.Singleline | RegexOptions.IgnoreCase);
// Regex for ALTER TABLE ... ADD CONSTRAINT ... FOREIGN KEY ...
private static readonly Regex alterTableForeignKeyRegex = new Regex(
@"ALTER\s+TABLE\s+`(?<table>\w+)`\s+ADD\s+CONSTRAINT\s+`(?<constraint>\w+)`\s+FOREIGN\s+KEY\s*\(`(?<column>\w+)`\)\s+REFERENCES\s+`(?<refTable>\w+)`\s*\(`(?<refColumn>\w+)`\)",
RegexOptions.IgnoreCase);
public static Dictionary<int, Dictionary<string, string>> ParseInsertBlock(string sql, string? createTableSql = null)
{
var result = new Dictionary<int, Dictionary<string, string>>();
var fullInsertRegex = new Regex(
@"INSERT INTO\s+`(?<table>\w+)`\s*\((?<columns>[^)]+)\)\s*VALUES\s*(?<values>.+);",
RegexOptions.Singleline | RegexOptions.IgnoreCase);
var match = fullInsertRegex.Match(sql);
Match match = InsertWithColsRegex.Match(sql);
Match matchNoCols = null;
if (!match.Success)
throw new Exception("INSERT block not matched.");
{
matchNoCols = InsertNoColsRegex.Match(sql);
if (!matchNoCols.Success)
{
throw new Exception("INSERT block not matched.");
}
}
var columns = match.Groups["columns"].Value
.Split(',')
.Select(c => c.Trim().Trim('`', ' '))
.ToArray();
string[] columns;
string valuesBlock = "";
if (match.Success)
{
columns = match.Groups["columns"].Value
.Split(',')
.Select(c => c.Trim().Trim('`', ' '))
.ToArray();
valuesBlock = match.Groups["values"].Value;
}
else
{
columns = ParseCreateSqlToRows(createTableSql);
if(matchNoCols != null)
valuesBlock = matchNoCols.Groups["values"].Value;
}
var valuesBlock = match.Groups["values"].Value;
// Long block - group not working with lagre blocks
if (valuesBlock.Length > 250)
{
string blockStart = valuesBlock.Substring(0, 25);
int startIndex = sql.IndexOf(blockStart);
valuesBlock = sql.Substring(startIndex, sql.Length-startIndex);
}
// Match each value row: (1, 'WM1', '...', 1)
var rowRegex = new Regex(@"\(([^()]*(?:\([^)]*\)[^()]*)*)\)", RegexOptions.Singleline);
@ -49,6 +84,200 @@ public class SqlDumpParser
return result;
}
public static string[] ParseCreateSqlToRows(string? createTableSql)
{
if (string.IsNullOrWhiteSpace(createTableSql))
return Array.Empty<string>();
// Find the CREATE TABLE header and the index of the opening '(' that starts the body
var headerRegex = new Regex(
@"CREATE\s+TABLE(?:\s+IF\s+NOT\s+EXISTS)?\s+(`[^`]+`|\w+)\s*\(",
RegexOptions.IgnoreCase | RegexOptions.Singleline);
var headerMatch = headerRegex.Match(createTableSql);
int openIndex = -1;
if (headerMatch.Success)
{
// position of '(' (headerMatch includes the '(' at the end)
openIndex = headerMatch.Index + headerMatch.Length - 1;
if (openIndex < 0 || openIndex >= createTableSql.Length || createTableSql[openIndex] != '(')
openIndex = createTableSql.IndexOf('(', headerMatch.Index + headerMatch.Length - 1);
}
else
{
// fallback: first '('
openIndex = createTableSql.IndexOf('(');
}
if (openIndex < 0)
return Array.Empty<string>();
// Find matching closing ')' for that opening '(' respecting quotes/escapes
int depth = 1;
bool inString = false;
char stringChar = '\0';
bool escape = false;
int closeIndex = -1;
for (int i = openIndex + 1; i < createTableSql.Length; i++)
{
char c = createTableSql[i];
if (inString)
{
if (escape)
{
escape = false;
}
else if (c == '\\')
{
escape = true;
}
else if (c == stringChar)
{
inString = false;
stringChar = '\0';
}
continue;
}
if (c == '\'' || c == '"')
{
inString = true;
stringChar = c;
continue;
}
if (c == '(') depth++;
else if (c == ')')
{
depth--;
if (depth == 0)
{
closeIndex = i;
break;
}
}
}
if (closeIndex < 0)
return Array.Empty<string>();
// Extract body between parentheses
string body = createTableSql.Substring(openIndex + 1, closeIndex - openIndex - 1);
// Split body into top-level comma separated segments (respecting parentheses and quotes)
var segments = new List<string>();
var sb = new StringBuilder();
depth = 0;
inString = false;
stringChar = '\0';
escape = false;
for (int i = 0; i < body.Length; i++)
{
char ch = body[i];
if (inString)
{
sb.Append(ch);
if (escape)
{
escape = false;
}
else if (ch == '\\')
{
escape = true;
}
else if (ch == stringChar)
{
inString = false;
stringChar = '\0';
}
continue;
}
if (ch == '\'' || ch == '"')
{
inString = true;
stringChar = ch;
sb.Append(ch);
continue;
}
if (ch == '(')
{
depth++;
sb.Append(ch);
continue;
}
if (ch == ')')
{
depth--;
sb.Append(ch);
continue;
}
// top-level comma is a segment separator
if (ch == ',' && depth == 0)
{
segments.Add(sb.ToString());
sb.Clear();
continue;
}
sb.Append(ch);
}
// last segment
if (sb.Length > 0)
segments.Add(sb.ToString());
// From segments pick only column definitions (not constraints/indexes)
var columns = new List<string>();
foreach (var seg in segments)
{
var s = seg.Trim();
if (string.IsNullOrEmpty(s))
continue;
// Skip constraints/indexes / primary keys / unique / key / constraint / foreign key / check / fulltext / spatial etc.
string up = s.Length > 30 ? s.Substring(0, 30).ToUpperInvariant() : s.ToUpperInvariant();
if (up.StartsWith("PRIMARY ") ||
up.StartsWith("KEY ") ||
up.StartsWith("UNIQUE") ||
up.StartsWith("CONSTRAINT") ||
up.StartsWith("FOREIGN ") ||
up.StartsWith("INDEX") ||
up.StartsWith("FULLTEXT") ||
up.StartsWith("SPATIAL") ||
up.StartsWith("CHECK"))
{
continue;
}
// Column names can be backticked or plain identifiers
var mBack = Regex.Match(s, @"^\s*`(?<name>[^`]+)`\s+", RegexOptions.Singleline);
if (mBack.Success)
{
columns.Add(mBack.Groups["name"].Value);
continue;
}
var mPlain = Regex.Match(s, @"^\s*(?<name>[A-Za-z_][A-Za-z0-9_]*)\s+", RegexOptions.Singleline);
if (mPlain.Success)
{
columns.Add(mPlain.Groups["name"].Value);
continue;
}
// otherwise ignore segment
}
return columns.ToArray();
}
private static List<string> ParseCsvRow(string row)
{
var values = new List<string>();
@ -103,14 +332,7 @@ public class SqlDumpParser
return values;
}
// Regex for ALTER TABLE ... ADD CONSTRAINT ... FOREIGN KEY ...
private static readonly Regex alterTableForeignKeyRegex = new Regex(
@"ALTER\s+TABLE\s+`(?<table>\w+)`\s+ADD\s+CONSTRAINT\s+`(?<constraint>\w+)`\s+FOREIGN\s+KEY\s*\(`(?<column>\w+)`\)\s+REFERENCES\s+`(?<refTable>\w+)`\s*\(`(?<refColumn>\w+)`\)",
RegexOptions.IgnoreCase);
public static Dictionary<string, List<ForeignKeyDef>> ParseForeignKeys(string line)
{
var tableForeignKeys = new Dictionary<string, List<ForeignKeyDef>>();

26
tbfDBBackup/compare_dumps/joinValues/Joiner.cs
View File

@ -122,6 +122,7 @@ public class Joiner
{
bool tryGetInsertValue =
oldDumpParser.TableInserts.TryGetValue(table.TableName, out List<string>? oldRowsInserts);
oldDumpParser.TableDefinitions.TryGetValue(table.TableName, out string? oldRowCreate);
StringBuilder builder = new StringBuilder();
@ -134,26 +135,8 @@ public class Joiner
if (col.IsPrimaryColumn) { primaryColumns.Add(col?.OldColumn); }
});
//get foreigner columns
bool tryGetForeignerColums =
oldDumpParser.TableForeignKeys.TryGetValue(table.TableName, out List<string>? oldForeignerTableNames);
List<ForeignKeyDef> foreignKeys = new List<ForeignKeyDef>();
if (tryGetForeignerColums)
{
oldForeignerTableNames.ForEach(ForeignKeyConstraintRow =>
{
Dictionary<string, List<ForeignKeyDef>> foreignerList =
SqlDumpParser.ParseForeignKeys(ForeignKeyConstraintRow);
if (foreignerList.Count == 1) // we expecting only one foreign key definition per table
{
var keyValuePair = foreignerList.First().Value;
foreach (var combination in keyValuePair)
{
foreignKeys.Add(combination);
}
}
}
);
}
List<ForeignKeyDef> foreignKeys = oldDumpParser.GetAllForeignKeysByTableName(table.TableName);
//get max id from table - for insert
long iMaxId = GetMaxId(table.TableName, primaryColumns.First());
@ -165,7 +148,7 @@ public class Joiner
//collect data by columns to insert command
oldRowsInserts.ForEach(colapsedInsertRow =>
{
Dictionary<int, Dictionary<string, string>> insertBlock = SqlDumpParser.ParseInsertBlock(colapsedInsertRow);
Dictionary<int, Dictionary<string, string>> insertBlock = SqlDumpParser.ParseInsertBlock(colapsedInsertRow, oldRowCreate);
foreach (KeyValuePair<int, Dictionary<string, string>> keyValuePair in insertBlock)
{
@ -297,6 +280,7 @@ public class Joiner
if (dbConfiguration.UpgradeDatabase != null)
foreach (Upgrade upgrade in dbConfiguration.UpgradeDatabase)
{
string databaseName = upgrade.DatabaseName;

29
tbfDBBackup/config/configuration.json
View File

@ -6,41 +6,18 @@
{
"Name": "Settings DB",
"Connection": "SERVER=localhost; DATABASE={DatabaseName}; UID=root; PASSWORD=; CHARSET=utf8;",
"DatabaseName": "wrcswindon298",
"DatabaseName": "testupgrade",
"Enabled": true
}
],
"UpdateString": [
{
"Name": "Alter table Water Meter",
"Query": "INSERT INTO \u0060group\u0060 (Name, AccessFlags) VALUES (\u0027MetrologicalAuthority\u0027, 0);\nINSERT INTO \u0060group\u0060 (Name, AccessFlags) VALUES (\u0027WaterMeterAuthority\u0027, 0);",
"DatabaseName": "wrcswindon298",
"Enabled": false,
"IsOneTimeOpaque": true
},
{
"Name": "Alter table Water Meter",
"Query": "INSERT INTO usersgroups (Group_id, User_id) VALUES (8, 1);\nINSERT INTO usersgroups (Group_id, User_id) VALUES (9, 1);",
"DatabaseName": "wrcswindon298",
"Enabled": false,
"IsOneTimeOpaque": true
}
],
"UpgradeDatabase": [
{
"Enabled": true,
"Name": "Upgrade by JSON",
"Query": "",
"JSONFileName": "",
"DatabaseName": "wrcswindon298",
"IsOneTimeOpaque": true
},
{
"Enabled": true,
"Name": "Alter table Water Meter",
"Query": "INSERT INTO usersgroups (Group_id, User_id) VALUES (8, 1);\nINSERT INTO usersgroups (Group_id, User_id) VALUES (9, 1);",
"JSONFileName": null,
"DatabaseName": "wrcswindon298",
"DatabaseName": "testupgrade",
"IsOneTimeOpaque": true
}
]

224
tbfDBBackup/config/schema-mapping.json
View File

@ -1,97 +1,5 @@
{
"Tables": [
{
"TableName": "componentprocedure",
"ColumnMappings": [
{
"OldColumn": "Id",
"NewColumn": "Id",
"Status": "matched",
"ColumnDef": {
"Name": "Id",
"Type": "int(11)",
"Nullable": "NO"
}
},
{
"OldColumn": "CmpntName",
"NewColumn": "CmpntName",
"Status": "matched",
"ColumnDef": {
"Name": "CmpntName",
"Type": "varchar(255)",
"Nullable": "YES"
}
},
{
"OldColumn": "Parameters",
"NewColumn": "Parameters",
"Status": "matched",
"ColumnDef": {
"Name": "Parameters",
"Type": "varchar(8000)",
"Nullable": "YES"
}
},
{
"OldColumn": "Procedure_id",
"NewColumn": "Procedure_id",
"Status": "matched",
"ColumnDef": {
"Name": "Procedure_id",
"Type": "int(11)",
"Nullable": "YES"
}
}
],
"IsNewTable": false
},
{
"TableName": "componenttest",
"ColumnMappings": [
{
"OldColumn": "Id",
"NewColumn": "Id",
"Status": "matched",
"ColumnDef": {
"Name": "Id",
"Type": "int(11)",
"Nullable": "NO"
}
},
{
"OldColumn": "CmpntName",
"NewColumn": "CmpntName",
"Status": "matched",
"ColumnDef": {
"Name": "CmpntName",
"Type": "varchar(255)",
"Nullable": "YES"
}
},
{
"OldColumn": "Parameters",
"NewColumn": "Parameters",
"Status": "matched",
"ColumnDef": {
"Name": "Parameters",
"Type": "varchar(8000)",
"Nullable": "YES"
}
},
{
"OldColumn": "Test_id",
"NewColumn": "Test_id",
"Status": "matched",
"ColumnDef": {
"Name": "Test_id",
"Type": "int(11)",
"Nullable": "YES"
}
}
],
"IsNewTable": false
},
{
"TableName": "procedure",
"ColumnMappings": [
@ -437,7 +345,7 @@
"Type": "double",
"Nullable": "YES"
}
]
]
},
{
"OldColumn": "Qto",
@ -446,9 +354,9 @@
"Note": "",
"ColumnDefList": [
{
"Name": "Qto",
"Type": "float",
"Nullable": "YES"
"Name": "Qto",
"Type": "float",
"Nullable": "YES"
},
{
"Name": "Qto",
@ -463,16 +371,16 @@
"Note": "",
"ColumnDefList": [
{
"Name": "Volume",
"Type": "float",
"Nullable": "YES"
},
"Name": "Volume",
"Type": "float",
"Nullable": "YES"
},
{
"Name": "Volume",
"Type": "double",
"Nullable": "YES"
}
]
]
},
{
"OldColumn": "TstTime",
@ -481,16 +389,16 @@
"Note": "Could be part of JSON or renamed",
"ColumnDefList": [
{
"Name": "TstTime",
"Type": "float",
"Nullable": "YES"
},
"Name": "TstTime",
"Type": "float",
"Nullable": "YES"
},
{
"Name": "TestTime",
"Type": "double",
"Nullable": "YES"
}
]
]
},
{
"OldColumn": "Repeats",
@ -590,17 +498,17 @@
"Note": "`MassMethod` tinyint(3) unsigned DEFAULT NULL,",
"ColumnDefList": [
{
"Name": "MassMethod",
"Type": "tinyint(1)",
"Nullable": "YES"
},
"Name": "MassMethod",
"Type": "tinyint(1)",
"Nullable": "YES"
},
{
"Name": "MassMethod",
"Type": "tinyint(3)",
"Nullable": "YES"
}
]
]
},
{
"OldColumn": "TimeBeforeFlow",
@ -805,6 +713,98 @@
}
],
"IsNewTable": false
},
{
"TableName": "componentprocedure",
"ColumnMappings": [
{
"OldColumn": "Id",
"NewColumn": "Id",
"Status": "matched",
"ColumnDef": {
"Name": "Id",
"Type": "int(11)",
"Nullable": "NO"
}
},
{
"OldColumn": "CmpntName",
"NewColumn": "CmpntName",
"Status": "matched",
"ColumnDef": {
"Name": "CmpntName",
"Type": "varchar(255)",
"Nullable": "YES"
}
},
{
"OldColumn": "Parameters",
"NewColumn": "Parameters",
"Status": "matched",
"ColumnDef": {
"Name": "Parameters",
"Type": "varchar(8000)",
"Nullable": "YES"
}
},
{
"OldColumn": "Procedure_id",
"NewColumn": "Procedure_id",
"Status": "matched",
"ColumnDef": {
"Name": "Procedure_id",
"Type": "int(11)",
"Nullable": "YES"
}
}
],
"IsNewTable": false
},
{
"TableName": "componenttest",
"ColumnMappings": [
{
"OldColumn": "Id",
"NewColumn": "Id",
"Status": "matched",
"ColumnDef": {
"Name": "Id",
"Type": "int(11)",
"Nullable": "NO"
}
},
{
"OldColumn": "CmpntName",
"NewColumn": "CmpntName",
"Status": "matched",
"ColumnDef": {
"Name": "CmpntName",
"Type": "varchar(255)",
"Nullable": "YES"
}
},
{
"OldColumn": "Parameters",
"NewColumn": "Parameters",
"Status": "matched",
"ColumnDef": {
"Name": "Parameters",
"Type": "varchar(8000)",
"Nullable": "YES"
}
},
{
"OldColumn": "Test_id",
"NewColumn": "Test_id",
"Status": "matched",
"ColumnDef": {
"Name": "Test_id",
"Type": "int(11)",
"Nullable": "YES"
}
}
],
"IsNewTable": false
}
]
}

27
tbfDBBackup/joinConfig/configuration.json
View File

@ -6,40 +6,17 @@
{
"Name": "Settings DB",
"Connection": "SERVER=localhost; DATABASE={DatabaseName}; UID=root; PASSWORD=; CHARSET=utf8;",
"DatabaseName": "wrcswindon298",
"DatabaseName": "testupgrade",
"Enabled": true
}
],
"UpdateString": [
{
"Enabled": true,
"Name": "Alter table Water Meter",
"Query": "INSERT INTO `group` (Name, AccessFlags) VALUES ('MetrologicalAuthority', 0);\nINSERT INTO `group` (Name, AccessFlags) VALUES ('WaterMeterAuthority', 0);",
"DatabaseName": "wrcswindon298",
"IsOneTimeOpaque": true
},
{
"Enabled": true,
"Name": "Alter table Water Meter",
"Query": "INSERT INTO usersgroups (Group_id, User_id) VALUES (8, 1);\nINSERT INTO usersgroups (Group_id, User_id) VALUES (9, 1);",
"DatabaseName": "wrcswindon298",
"IsOneTimeOpaque": true
}
],
"UpgradeDatabase": [
{
"Enabled": true,
"Name": "Upgrade by JSON",
"Query": "",
"JSONFileName": "",
"DatabaseName": "wrcswindon298",
"IsOneTimeOpaque": true
},
{
"Enabled": true,
"Name": "Alter table Water Meter",
"Query": "INSERT INTO usersgroups (Group_id, User_id) VALUES (8, 1);\nINSERT INTO usersgroups (Group_id, User_id) VALUES (9, 1);",
"DatabaseName": "wrcswindon298",
"DatabaseName": "testupgrade",
"IsOneTimeOpaque": true
}
]