Add Tamil NER routing and WPF test harness for POC validation.

Introduce dual-script ONNX NER routing (English/Tamil/mixed), Tamil console samples and integration tests, model download scripts, and a resizable WPF MVVM harness with click-to-load prompts, batch validation, and runtime-adjustable detection panels.
This commit is contained in:
Bilal Nazer Ali
2026-07-07 17:12:38 +05:30
parent a707c6c9cf
commit cf8f5a7232
71 changed files with 4494 additions and 356 deletions

View File

@@ -6,12 +6,12 @@ using PiiRedaction.Infrastructure.Onnx;
namespace PiiRedaction.Tests.Shared;
/// <summary>
/// Reuses a single <see cref="OnnxNerModelRunner"/> per fixture for performance.
/// Reuses a single <see cref="EnglishOnnxNerRunner"/> per fixture for performance.
/// Skips all tests in the class when the ONNX model is missing or cannot be loaded.
/// </summary>
public abstract class RealNerModelFixture
{
protected OnnxNerModelRunner Runner { get; private set; } = null!;
protected EnglishOnnxNerRunner Runner { get; private set; } = null!;
protected string ModelPath { get; private set; } = null!;
@@ -24,8 +24,12 @@ public abstract class RealNerModelFixture
Assert.Ignore(RealNerModelPaths.ModelMissingMessage);
}
var options = Options.Create(new PiiRedactionOptions { OnnxModelPath = ModelPath });
Runner = new OnnxNerModelRunner(options, NullLogger<OnnxNerModelRunner>.Instance);
var options = Options.Create(new PiiRedactionOptions
{
EnglishOnnxModelPath = ModelPath,
OnnxModelPath = ModelPath
});
Runner = new EnglishOnnxNerRunner(options, NullLogger<EnglishOnnxNerRunner>.Instance);
if (!Runner.IsModelAvailable)
{

View File

@@ -7,16 +7,23 @@ public static class RealNerModelPaths
public static string ResolveRepoModelPath()
{
var directory = new DirectoryInfo(AppContext.BaseDirectory);
while (directory is not null)
foreach (var relativePath in new[]
{
Path.Combine("models", "en", "ner-model.onnx"),
Path.Combine("models", "ner-model.onnx")
})
{
var candidate = Path.Combine(directory.FullName, "models", "ner-model.onnx");
if (File.Exists(candidate))
var directory = new DirectoryInfo(AppContext.BaseDirectory);
while (directory is not null)
{
return candidate;
}
var candidate = Path.Combine(directory.FullName, relativePath);
if (File.Exists(candidate))
{
return candidate;
}
directory = directory.Parent;
directory = directory.Parent;
}
}
return Path.Combine(Environment.CurrentDirectory, "models", "ner-model.onnx");

View File

@@ -0,0 +1,69 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using PiiRedaction.Core.Configuration;
using PiiRedaction.Infrastructure.Onnx;
namespace PiiRedaction.Tests.Shared;
/// <summary>
/// Loads English and Tamil ONNX runners and exposes a <see cref="RoutingOnnxNerModelRunner"/>.
/// Skips when either model is missing or cannot be loaded.
/// </summary>
public abstract class RealRoutingNerModelFixture
{
protected RoutingOnnxNerModelRunner Runner { get; private set; } = null!;
protected EnglishOnnxNerRunner EnglishRunner { get; private set; } = null!;
protected TamilOnnxNerRunner TamilRunner { get; private set; } = null!;
[OneTimeSetUp]
public void OneTimeSetUpRoutingModels()
{
var englishPath = RealNerModelPaths.ResolveRepoModelPath();
if (!File.Exists(englishPath))
{
Assert.Ignore(RealNerModelPaths.ModelMissingMessage);
}
var tamilPath = RealTamilNerModelPaths.ResolveRepoModelPath();
if (!File.Exists(tamilPath))
{
Assert.Ignore(RealTamilNerModelPaths.ModelMissingMessage);
}
var options = Options.Create(new PiiRedactionOptions
{
EnglishOnnxModelPath = englishPath,
OnnxModelPath = englishPath,
TamilOnnxModelPath = tamilPath,
EnableTamilNer = true
});
EnglishRunner = new EnglishOnnxNerRunner(options, NullLogger<EnglishOnnxNerRunner>.Instance);
TamilRunner = new TamilOnnxNerRunner(options, NullLogger<TamilOnnxNerRunner>.Instance);
if (!EnglishRunner.IsModelAvailable)
{
EnglishRunner.Dispose();
TamilRunner.Dispose();
Assert.Ignore(RealNerModelPaths.ModelMissingMessage);
}
if (!TamilRunner.IsModelAvailable)
{
EnglishRunner.Dispose();
TamilRunner.Dispose();
Assert.Ignore(RealTamilNerModelPaths.ModelMissingMessage);
}
Runner = new RoutingOnnxNerModelRunner(EnglishRunner, TamilRunner, options);
}
[OneTimeTearDown]
public void OneTimeTearDownRoutingModels()
{
EnglishRunner?.Dispose();
TamilRunner?.Dispose();
}
}

View File

@@ -0,0 +1,45 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using PiiRedaction.Core.Configuration;
using PiiRedaction.Infrastructure.Onnx;
namespace PiiRedaction.Tests.Shared;
/// <summary>
/// Reuses a single <see cref="TamilOnnxNerRunner"/> per fixture for performance.
/// Skips all tests in the class when the Tamil ONNX model is missing or cannot be loaded.
/// </summary>
public abstract class RealTamilNerModelFixture
{
protected TamilOnnxNerRunner Runner { get; private set; } = null!;
protected string ModelPath { get; private set; } = null!;
[OneTimeSetUp]
public void OneTimeSetUpTamilModel()
{
ModelPath = RealTamilNerModelPaths.ResolveRepoModelPath();
if (!File.Exists(ModelPath))
{
Assert.Ignore(RealTamilNerModelPaths.ModelMissingMessage);
}
var options = Options.Create(new PiiRedactionOptions
{
TamilOnnxModelPath = ModelPath
});
Runner = new TamilOnnxNerRunner(options, NullLogger<TamilOnnxNerRunner>.Instance);
if (!Runner.IsModelAvailable)
{
Runner.Dispose();
Assert.Ignore(RealTamilNerModelPaths.ModelMissingMessage);
}
}
[OneTimeTearDown]
public void OneTimeTearDownTamilModel()
{
Runner?.Dispose();
}
}

View File

@@ -0,0 +1,25 @@
namespace PiiRedaction.Tests.Shared;
public static class RealTamilNerModelPaths
{
public const string ModelMissingMessage =
"Tamil ONNX model not found. Run scripts/download-tamil-ner-model.ps1 from the repository root.";
public static string ResolveRepoModelPath()
{
const string relativePath = "models/ta/model.onnx";
var directory = new DirectoryInfo(AppContext.BaseDirectory);
while (directory is not null)
{
var candidate = Path.Combine(directory.FullName, relativePath);
if (File.Exists(candidate))
{
return candidate;
}
directory = directory.Parent;
}
return Path.Combine(Environment.CurrentDirectory, relativePath);
}
}