<Fix>: Batch XLSX result - long exports, increase revision to 3.9.3147.118

Cause:

- ResultsWriter exported each meter/test result as a separate write operation, repeatedly loading and saving the entire XLSX workbook.
- Exporting 20 meters with 12 tests took approximately 9 minutes in the supplied production log.

Solution:

1. Added an explicit batch write scope
   - Load the XLSX workbook once, buffer rows and save once after successful completion.
   - Preserve standalone SetData behavior, existing CSV exports and both result row views.

2. Handle unsuccessful exports safely
   - Discard buffered XLSX changes on failure or cancellation.
   - Publish through a temporary file to protect the existing workbook.
   - Report export success only after saving completes; preserve the existing RsltsSent behavior.

3. Added regression coverage
   - Verify 240-row export, subsequent appends, standalone writes, cancellation, invalid rows, locked destinations and empty exports.
   - All 16 targeted tests passed. A small test workbook export of 20 meters x 12 tests took 446 ms.

4. Increased revision
   - Updated TBF AssemblyVersion and AssemblyFileVersion from 3.9.3147.117 to 3.9.3147.118.
This commit is contained in:
2026-10-09 10:09:42 +02:00
parent 3202da1c86
commit 3e1240c0ea
5 changed files with 284 additions and 31 deletions
+2 -2
View File
@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// Build Number
// Revision
//
[assembly: AssemblyVersion("3.9.3147.117")]
[assembly: AssemblyFileVersion("3.9.3147.117")]
[assembly: AssemblyVersion("3.9.3147.118")]
[assembly: AssemblyFileVersion("3.9.3147.118")]
@@ -9,6 +9,7 @@ using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading;
using TBF.Rig.Generic;
using TBF.Rig.Output.DataStorage.UniDataStorageWriter;
using TBF.Rig.Output.DataStorage.UniDataStorageWriter.Formatters;
@@ -40,6 +41,7 @@ namespace TBF.Rig.Output.DB.ResultsWriter
private Batch batch;
private bool opCompleted;
private bool anyError;
private CancellationTokenSource exportCancellation = new CancellationTokenSource();
private enum OpState
{
@@ -139,6 +141,8 @@ namespace TBF.Rig.Output.DB.ResultsWriter
public void Start()
{
exportCancellation.Dispose();
exportCancellation = new CancellationTokenSource();
if (currentOpState == OpState.WriteResultsScheduled)
{
currentOpState = OpState.WriteResultsRunning;
@@ -176,7 +180,7 @@ namespace TBF.Rig.Output.DB.ResultsWriter
try
{
WriteBatchResults(batch);
WriteBatchResults(batch, exportCancellation.Token);
}
catch (Exception exc)
{
@@ -190,6 +194,7 @@ namespace TBF.Rig.Output.DB.ResultsWriter
public void Stop()
{
exportCancellation.Cancel();
currentOpState = OpState.None;
}
@@ -205,6 +210,11 @@ namespace TBF.Rig.Output.DB.ResultsWriter
/// regular meter test is written with its concrete test ID context.
/// </remarks>
public void WriteBatchResults(Batch batch)
{
WriteBatchResults(batch, CancellationToken.None);
}
public void WriteBatchResults(Batch batch, CancellationToken cancellationToken)
{
if (batch == null)
throw new ArgumentNullException(nameof(batch));
@@ -234,20 +244,19 @@ namespace TBF.Rig.Output.DB.ResultsWriter
ValidateRowView();
foreach (WaterMeter wm in batch.WaterMeters)
using (var export = dataStorageWriter.BeginBatch())
{
if (wm == null || wm.Disabled)
foreach (WaterMeter wm in batch.WaterMeters)
{
continue;
}
cancellationToken.ThrowIfCancellationRequested();
if (wm == null || wm.Disabled) continue;
//
// Insert mode needs the same test context as the legacy TBF
// result writers. One row is generated for every published
// regular meter test and all selected items are evaluated
// with the actual mtr.Name() as test ID.
//
WriteInsertResults(wm, storageCfg);
// Each published test retains its concrete test ID context.
WriteInsertResults(wm, storageCfg, export, cancellationToken);
}
cancellationToken.ThrowIfCancellationRequested();
export.Complete();
log.Warn("ResultsWriter export completed and saved.");
}
}
@@ -434,11 +443,11 @@ namespace TBF.Rig.Output.DB.ResultsWriter
/// <summary>
/// Writes one row per meter, or one per published regular test in legacy mode.
/// </summary>
private void WriteInsertResults(WaterMeter wm, WriterCfg storageCfg)
private void WriteInsertResults(WaterMeter wm, WriterCfg storageCfg, Writer.BatchWriteSession export, CancellationToken cancellationToken)
{
if (resultsWriterCfg.RowView == ResultsRowView.PerMeter)
{
WriteInsertRequest(wm, string.Empty, BuildInsertRequest(wm, string.Empty), storageCfg);
WriteInsertRequest(wm, string.Empty, BuildInsertRequest(wm, string.Empty), storageCfg, export);
return;
}
@@ -453,18 +462,19 @@ namespace TBF.Rig.Output.DB.ResultsWriter
{
DataWriteRequest fallbackRequest = BuildInsertRequest(wm, string.Empty);
WriteInsertRequest(wm, string.Empty, fallbackRequest, storageCfg);
WriteInsertRequest(wm, string.Empty, fallbackRequest, storageCfg, export);
return;
}
foreach (MeterTestRslt mtr in publishedTests)
{
cancellationToken.ThrowIfCancellationRequested();
string testId = mtr.Name() ?? string.Empty;
DataWriteRequest request = BuildInsertRequest(wm, testId);
WriteInsertRequest(wm, testId, request, storageCfg);
WriteInsertRequest(wm, testId, request, storageCfg, export);
}
}
@@ -504,7 +514,7 @@ namespace TBF.Rig.Output.DB.ResultsWriter
/// <summary>
/// Executes one prepared Insert request.
/// </summary>
private void WriteInsertRequest(WaterMeter wm, string testId, DataWriteRequest request, WriterCfg storageCfg)
private void WriteInsertRequest(WaterMeter wm, string testId, DataWriteRequest request, WriterCfg storageCfg, Writer.BatchWriteSession export)
{
if (request == null || request.InsertItems.Count == 0)
{
@@ -513,7 +523,7 @@ namespace TBF.Rig.Output.DB.ResultsWriter
return;
}
WriterDiagnosticResult result = dataStorageWriter.SetData(request);
WriterDiagnosticResult result = export.SetData(request);
if (!result.Success)
{
@@ -521,7 +531,7 @@ namespace TBF.Rig.Output.DB.ResultsWriter
}
log.WarnFormat(
"ResultsWriter wrote WM position {0}, TestID='{1}'. Target={2}, Technology={3}, Mode={4}.",
"ResultsWriter accepted WM position {0}, TestID='{1}'. Target={2}, Technology={3}, Mode={4}.",
wm.WMPosition,
testId,
storageCfg.DataStorageType,
@@ -129,6 +129,61 @@ namespace TBF.Rig.Output.DataStorage.UniDataStorageWriter
}
}
/// <summary>Creates an export scope. Dispose without Complete discards buffered XLSX rows.</summary>
public BatchWriteSession BeginBatch()
{
return new BatchWriteSession(this);
}
public sealed class BatchWriteSession : IDisposable
{
private readonly Writer owner;
private readonly XlsxWriter xlsx;
private bool failed;
private bool finished;
internal BatchWriteSession(Writer owner)
{
this.owner = owner;
xlsx = owner.CreateWriter() as XlsxWriter;
if (xlsx != null) xlsx.BeginBatch();
}
public WriterDiagnosticResult SetData(DataWriteRequest request)
{
if (finished || failed) throw new InvalidOperationException("The export scope is closed or failed.");
try
{
WriterDiagnosticResult result;
if (xlsx == null) result = owner.SetData(request);
else
{
if (request == null) throw new ArgumentNullException(nameof(request));
result = owner.ValidateCapabilities(xlsx, request);
if (result.Success) result = xlsx.WriteData(request);
}
failed = !result.Success;
return result;
}
catch { failed = true; throw; }
}
/// <summary>Only successful completion confirms that buffered rows reached the file.</summary>
public void Complete()
{
if (finished || failed) throw new InvalidOperationException("Cannot complete a closed or failed export.");
try { if (xlsx != null) xlsx.CompleteBatch(); }
finally { Dispose(); }
}
public void Dispose()
{
if (finished) return;
finished = true;
if (xlsx != null) xlsx.AbortBatch();
}
}
public WriterDiagnosticResult SetData(DataWriteRequest request)
{
try
@@ -231,4 +286,4 @@ namespace TBF.Rig.Output.DataStorage.UniDataStorageWriter
};
}
}
}
}
@@ -29,6 +29,56 @@ namespace TBF.Rig.Output.DataStorage.UniDataStorageWriter.Writers
private const string DefaultWorksheetName = "Data";
private readonly WriterCfg cfg;
private bool batchMode;
private XLWorkbook batchWorkbook;
private string batchPath;
private bool batchFileExisted;
internal void BeginBatch()
{
if (batchMode) throw new InvalidOperationException("An XLSX batch is already active.");
batchMode = true;
batchPath = cfg.DataSource;
}
internal void CompleteBatch()
{
if (batchWorkbook == null) return; // Empty export does not touch the destination.
string path = Path.GetFullPath(batchPath);
string temporary = Path.Combine(Path.GetDirectoryName(path), Guid.NewGuid() + ".xlsx");
try
{
batchWorkbook.SaveAs(temporary);
batchWorkbook.Dispose();
batchWorkbook = null;
// Publish only after serialization succeeds. A locked destination fails the export.
if (batchFileExisted) File.Replace(temporary, path, null);
else File.Move(temporary, path);
}
finally
{
if (File.Exists(temporary)) File.Delete(temporary);
}
}
internal void AbortBatch()
{
if (batchWorkbook != null) batchWorkbook.Dispose();
batchWorkbook = null;
batchMode = false;
}
private XLWorkbook GetWriteWorkbook()
{
if (!batchMode) return OpenWorkbook();
if (batchWorkbook == null)
{
batchFileExisted = File.Exists(batchPath);
batchWorkbook = batchFileExisted ? new XLWorkbook(batchPath) : new XLWorkbook();
if (!batchFileExisted) batchWorkbook.Worksheets.Add(DefaultWorksheetName);
}
return batchWorkbook;
}
/// <summary>
/// Initializes a new instance of the <see cref="XlsxWriter"/> class.
@@ -161,12 +211,17 @@ namespace TBF.Rig.Output.DataStorage.UniDataStorageWriter.Writers
if (request == null)
throw new ArgumentNullException(nameof(request));
WriterDiagnosticResult sourceTest = TestSource(false);
WriterDiagnosticResult sourceTest = batchMode ? ValidateDataSource() : TestSource(false);
if (!sourceTest.Success)
return sourceTest;
try
{
if (batchMode)
{
if (cfg.DataSource != batchPath) throw new InvalidOperationException("XLSX destination changed during export.");
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(batchPath)));
}
switch (request.Mode)
{
case WriteMode.Insert:
@@ -248,7 +303,8 @@ namespace TBF.Rig.Output.DataStorage.UniDataStorageWriter.Writers
string.Join(", ", duplicateColumns));
}
using (XLWorkbook workbook = OpenWorkbook())
XLWorkbook workbook = GetWriteWorkbook();
try
{
string template = cfg.GetTemplate(request.Mode);
string worksheetName = ParseWorksheetName(template);
@@ -287,7 +343,7 @@ namespace TBF.Rig.Output.DataStorage.UniDataStorageWriter.Writers
item.Value);
}
workbook.Save();
if (!batchMode) workbook.Save();
string diagnostic = BuildInsertDiagnostic(
worksheet.Name,
@@ -297,10 +353,11 @@ namespace TBF.Rig.Output.DataStorage.UniDataStorageWriter.Writers
return new WriterDiagnosticResult
{
Success = true,
Message = "XLSX insert OK. Written 1 row.",
Message = batchMode ? "XLSX row buffered; completion required." : "XLSX insert OK. Written 1 row.",
ExecutedTemplate = diagnostic
};
}
finally { if (!batchMode) workbook.Dispose(); }
}
/// <summary>
@@ -329,7 +386,8 @@ namespace TBF.Rig.Output.DataStorage.UniDataStorageWriter.Writers
if (request.UpdateItems.Any(i => i == null))
return Fail("Update items contain a null item.");
using (XLWorkbook workbook = OpenWorkbook())
XLWorkbook workbook = GetWriteWorkbook();
try
{
string template = cfg.GetTemplate(request.Mode);
string worksheetName = ParseWorksheetName(template);
@@ -377,7 +435,7 @@ namespace TBF.Rig.Output.DataStorage.UniDataStorageWriter.Writers
updatedRows));
}
workbook.Save();
if (!batchMode) workbook.Save();
return new WriterDiagnosticResult
{
@@ -388,6 +446,7 @@ namespace TBF.Rig.Output.DataStorage.UniDataStorageWriter.Writers
diagnostics.ToString().TrimEnd()
};
}
finally { if (!batchMode) workbook.Dispose(); }
}
/// <summary>
+132 -3
View File
@@ -20,6 +20,113 @@ namespace TBFTests
[TestClass]
public class ResultsWriterRowViewTests
{
static Writer Storage(string path)
{
return new Writer(new WriterCfg("Storage", new StorageFactory()) {
DataStorageType = Types.StorageTypes.LocalFile, TechnologyType = ".xlsx",
DataSource = path, WriteMode = WriteMode.Insert, QueryTemplate = "Sheet=Data" });
}
static DataWriteRequest Row(string value)
{
var request = new DataWriteRequest();
request.InsertItems.Add(new InsertWriteItem { ColumnName = "Serial", Value = value });
return request;
}
static int XlsxRows(string path)
{
using (var archive = ZipFile.OpenRead(path))
using (var stream = archive.GetEntry("xl/worksheets/sheet1.xml").Open())
{
XNamespace ns = "http://schemas.openxmlformats.org/spreadsheetml/2006/main";
return XDocument.Load(stream).Descendants(ns + "row").Count();
}
}
[TestMethod]
public void XlsxBatch_DefersAll240RowsUntilCompletion_AndStandaloneStillAppends()
{
string path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".xlsx");
try
{
var writer = Storage(path);
Assert.IsTrue(writer.SetData(Row("original")).Success);
byte[] original = File.ReadAllBytes(path);
using (var export = writer.BeginBatch())
{
for (int i = 0; i < 240; i++) Assert.IsTrue(export.SetData(Row("SN" + i)).Success);
CollectionAssert.AreEqual(original, File.ReadAllBytes(path));
export.Complete();
}
Assert.AreEqual(242, XlsxRows(path));
Assert.IsTrue(writer.SetData(Row("standalone")).Success);
Assert.AreEqual(243, XlsxRows(path));
}
finally { if (File.Exists(path)) File.Delete(path); }
}
[TestMethod]
public void XlsxBatch_AbortAndInvalidRowPreserveOriginal_AndNextExportWorks()
{
string path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".xlsx");
try
{
var writer = Storage(path);
Assert.IsTrue(writer.SetData(Row("original")).Success);
byte[] original = File.ReadAllBytes(path);
using (var export = writer.BeginBatch()) { Assert.IsTrue(export.SetData(Row("discard")).Success); }
CollectionAssert.AreEqual(original, File.ReadAllBytes(path));
using (var export = writer.BeginBatch())
{
Assert.IsTrue(export.SetData(Row("discard")).Success);
var invalid = Row("invalid");
invalid.InsertItems[0].ColumnName = "";
Assert.IsFalse(export.SetData(invalid).Success);
Assert.ThrowsException<InvalidOperationException>(() => export.Complete());
}
CollectionAssert.AreEqual(original, File.ReadAllBytes(path));
using (var export = writer.BeginBatch()) { export.SetData(Row("saved")); export.Complete(); }
Assert.AreEqual(3, XlsxRows(path));
}
finally { if (File.Exists(path)) File.Delete(path); }
}
[TestMethod]
public void XlsxBatch_LockedDestinationFailsCompletionWithoutChangingFile()
{
string path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".xlsx");
try
{
var writer = Storage(path);
writer.SetData(Row("original"));
byte[] original = File.ReadAllBytes(path);
using (var export = writer.BeginBatch())
{
Assert.IsTrue(export.SetData(Row("discard")).Success);
using (var locked = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read))
Assert.ThrowsException<IOException>(() => export.Complete());
}
CollectionAssert.AreEqual(original, File.ReadAllBytes(path));
}
finally { if (File.Exists(path)) File.Delete(path); }
}
[TestMethod]
public void XlsxBatch_EmptyAndAbortedNewExportDoNotCreateFile()
{
string path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".xlsx");
try
{
var writer = Storage(path);
using (var export = writer.BeginBatch()) { export.Complete(); }
Assert.IsFalse(File.Exists(path));
using (var export = writer.BeginBatch()) { Assert.IsTrue(export.SetData(Row("discard")).Success); }
Assert.IsFalse(File.Exists(path));
}
finally { if (File.Exists(path)) File.Delete(path); }
}
static WMeterRsltItemSpec Item(ItemID id, string caption)
{
var item = WMeterRsltItemSpec.AllItems.Single(x => x.Uid == (int)id).Clone();
@@ -41,12 +148,12 @@ namespace TBFTests
return cfg;
}
static Batch Batch(int testCount)
static Batch Batch(int testCount, int enabledMeters = 10)
{
var batch = new Batch();
for (int i = 0; i < 11; i++)
for (int i = 0; i <= enabledMeters; i++)
{
var meter = new WaterMeter { Batch = batch, SerialNr = "SN" + i, WMPosition = i + 1, Passed = true, Disabled = i == 10 };
var meter = new WaterMeter { Batch = batch, SerialNr = "SN" + i, WMPosition = i + 1, Passed = true, Disabled = i == enabledMeters };
batch.WaterMeters.Add(meter);
for (int t = 0; t < testCount; t++)
meter.MeterTestRslts.Add(new MeterTestRslt { WaterMeter = meter, Passed = false,
@@ -63,6 +170,28 @@ namespace TBFTests
return new ResultsWriter(cfg, new List<IComponent> { new Writer(storage) });
}
[TestMethod]
public void Export_TwentyMetersTwelveTests_AppendsAllRowsAndHonorsCancellation()
{
string path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".xlsx");
try
{
var writer = Writer(Config(ResultsRowView.PerTest), path, ".xlsx");
var batch = Batch(12, 20);
var watch = System.Diagnostics.Stopwatch.StartNew();
writer.WriteBatchResults(batch);
Console.WriteLine("20 meters x 12 tests: {0} ms", watch.ElapsedMilliseconds);
Assert.AreEqual(241, XlsxRows(path));
byte[] original = File.ReadAllBytes(path);
Assert.ThrowsException<OperationCanceledException>(() =>
writer.WriteBatchResults(batch, new System.Threading.CancellationToken(true)));
CollectionAssert.AreEqual(original, File.ReadAllBytes(path));
writer.WriteBatchResults(batch);
Assert.AreEqual(481, XlsxRows(path));
}
finally { if (File.Exists(path)) File.Delete(path); }
}
[DataTestMethod]
[DataRow(ResultsRowView.PerTest, ".csv", 4, 40)]
[DataRow(ResultsRowView.PerMeter, ".csv", 4, 10)]