Add Tamil NER routing and WPF test harness for POC validation.

Introduce dual-script ONNX NER routing (English/Tamil/mixed), Tamil console samples and integration tests, model download scripts, and a resizable WPF MVVM harness with click-to-load prompts, batch validation, and runtime-adjustable detection panels.
2026-07-07 17:12:38 +05:30
parent a707c6c9cf
commit cf8f5a7232
71 changed files with 4494 additions and 356 deletions
--- a/tests/PiiRedaction.Core.Tests/Detection/ScriptRouterTests.cs
+++ b/tests/PiiRedaction.Core.Tests/Detection/ScriptRouterTests.cs
@@ -0,0 +1,54 @@
+using FluentAssertions;
+using PiiRedaction.Core.Detection;
+using PiiRedaction.Infrastructure.Onnx;
+
+namespace PiiRedaction.Core.Tests.Detection;
+
+[TestFixture]
+public sealed class ScriptRouterTests
+{
+    private readonly ScriptRouter _router = new();
+
+    [Test]
+    public void GetComposition_LatinOnly_ReturnsLatinOnly()
+    {
+        _router.GetComposition("Customer Ravi Kumar called about billing.")
+            .Should().Be(ScriptComposition.LatinOnly);
+    }
+
+    [Test]
+    public void GetComposition_TamilOnly_ReturnsTamilOnly()
+    {
+        _router.GetComposition("வாடிக்கையாளர் ராஜேஷ் தொலைபேசி 9876543210")
+            .Should().Be(ScriptComposition.TamilOnly);
+    }
+
+    [Test]
+    public void GetComposition_Mixed_ReturnsMixed()
+    {
+        _router.GetComposition("Rajesh மற்றும் Priya disputed the charge.")
+            .Should().Be(ScriptComposition.Mixed);
+    }
+
+    [Test]
+    public void GetComposition_MixedTamilEnglishSample_ReturnsMixed()
+    {
+        _router.GetComposition("வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge.")
+            .Should().Be(ScriptComposition.Mixed);
+    }
+
+    [Test]
+    public void GetComposition_NoLetters_ReturnsNoLetters()
+    {
+        _router.GetComposition("9876543210 12345")
+            .Should().Be(ScriptComposition.NoLetters);
+    }
+
+    [Test]
+    public void GetComposition_TamilBoundaryChars_AreClassifiedAsTamil()
+    {
+        _router.GetComposition("\u0B80").Should().Be(ScriptComposition.TamilOnly);
+        _router.GetComposition("\u0BFF").Should().Be(ScriptComposition.TamilOnly);
+        _router.GetComposition("A").Should().Be(ScriptComposition.LatinOnly);
+    }
+}
--- a/tests/PiiRedaction.Core.Tests/Integration/GoldenPromptTests.cs
+++ b/tests/PiiRedaction.Core.Tests/Integration/GoldenPromptTests.cs
@@ -8,6 +8,7 @@ namespace PiiRedaction.Core.Tests.Integration;
 public sealed class GoldenPromptTests
 {
    [TestCaseSource(typeof(PromptScenarioCatalog), nameof(PromptScenarioCatalog.AllScenarios))]
+    [TestCaseSource(typeof(TamilPromptScenarioCatalog), nameof(TamilPromptScenarioCatalog.AllScenarios))]
    public void Sanitize_PromptScenario_ProducesExpectedOutput(PromptScenario scenario)
    {
        var sanitizer = ProductionPipelineFactory.CreateForScenario(scenario);
--- a/tests/PiiRedaction.Core.Tests/Integration/RealTamilPipelineTests.cs
+++ b/tests/PiiRedaction.Core.Tests/Integration/RealTamilPipelineTests.cs
@@ -0,0 +1,106 @@
+using FluentAssertions;
+using PiiRedaction.Core.Abstractions;
+using PiiRedaction.Core.Models;
+using PiiRedaction.Core.Tests.TestSupport;
+using PiiRedaction.Tests.Shared;
+
+namespace PiiRedaction.Core.Tests.Integration;
+
+/// <summary>
+/// End-to-end pipeline proof using routed English + Tamil ONNX NER models.
+/// </summary>
+[TestFixture]
+[Category("TamilNer")]
+public sealed class RealTamilPipelineTests : RealRoutingNerModelFixture
+{
+    private IPromptSanitizer _sanitizer = null!;
+
+    [OneTimeSetUp]
+    public void OneTimeSetUpPipeline()
+    {
+        _sanitizer = ProductionPipelineFactory.CreateWithRoutingRealModels(EnglishRunner, TamilRunner);
+    }
+
+    [Test]
+    public void Sanitize_TamilCustomerNameOnly_RedactsPerson()
+    {
+        const string prompt =
+            "வாடிக்கையாளர் ராஜேஷ் குமார் சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்.";
+
+        var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+        result.SanitizedPrompt.Should().Contain("<PERSON_1>");
+        result.SanitizedPrompt.Should().NotContain("ராஜேஷ்");
+        result.DetectedEntities.Should().Contain(entity =>
+            entity.Type == PiiEntityType.Person && entity.Source == PiiDetectionSource.Ner);
+    }
+
+    [Test]
+    public void Sanitize_TamilWithPhonePan_RedactsPersonPhoneAndPan()
+    {
+        const string prompt = "வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210 PAN ABCDE1234F.";
+
+        var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+        result.SanitizedPrompt.Should().Contain("<PERSON_1>");
+        result.SanitizedPrompt.Should().Contain("<PHONE_1>");
+        result.SanitizedPrompt.Should().Contain("<PAN_1>");
+        result.SanitizedPrompt.Should().NotContainAny("ராஜேஷ்", "9876543210", "ABCDE1234F");
+    }
+
+    [Test]
+    public void Sanitize_TanglishCustomer_RedactsPersonAndPhone()
+    {
+        const string prompt = "Customer Senthil phone 9876543210 reported a failed UPI transfer.";
+
+        var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+        result.SanitizedPrompt.Should().Contain("<PERSON_1>");
+        result.SanitizedPrompt.Should().Contain("<PHONE_1>");
+        result.SanitizedPrompt.Should().NotContainAny("Senthil", "9876543210");
+    }
+
+    [Test]
+    public void Sanitize_MixedTamilEnglish_RedactsPersonAndPhone()
+    {
+        const string prompt = "வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge.";
+
+        var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+        result.SanitizedPrompt.Should().Contain("<PERSON_1>");
+        result.SanitizedPrompt.Should().Contain("<PHONE_1>");
+        result.SanitizedPrompt.Should().NotContainAny("Ravi Kumar", "9876543210");
+    }
+
+    [Test]
+    public void Sanitize_TamilFullFinancial_RedactsAllPiiTypes()
+    {
+        const string prompt =
+            "வாடிக்கையாளர் ராஜேஷ் குமார் மின்னஞ்சல் ravi.kumar@gmail.com தொலைபேசி 9876543210 LoanNumber LN-456789 PAN ABCDE1234F. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்.";
+
+        var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+        result.SanitizedPrompt.Should().Contain("<PERSON_1>");
+        result.SanitizedPrompt.Should().Contain("<EMAIL_1>");
+        result.SanitizedPrompt.Should().Contain("<PHONE_1>");
+        result.SanitizedPrompt.Should().Contain("<LOAN_NUMBER_1>");
+        result.SanitizedPrompt.Should().Contain("<PAN_1>");
+        result.SanitizedPrompt.Should().NotContainAny(
+            "ராஜேஷ்",
+            "ravi.kumar@gmail.com",
+            "9876543210",
+            "LN-456789",
+            "ABCDE1234F");
+    }
+
+    [Test]
+    public void Sanitize_CleanTamilQuestion_PassesThroughWithoutPersonRedaction()
+    {
+        const string prompt = "பணத்தை திரும்பப் பெறுவது எப்படி?";
+
+        var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+        result.SanitizedPrompt.Should().Be(prompt);
+        result.DetectedEntities.Should().BeEmpty();
+    }
+}
--- a/tests/PiiRedaction.Core.Tests/PiiRedaction.Core.Tests.csproj
+++ b/tests/PiiRedaction.Core.Tests/PiiRedaction.Core.Tests.csproj
@@ -25,6 +25,9 @@
  <ItemGroup>
    <Compile Include="..\TestSupport.Shared\RealNerModelPaths.cs" Link="TestSupport.Shared\RealNerModelPaths.cs" />
    <Compile Include="..\TestSupport.Shared\RealNerModelFixture.cs" Link="TestSupport.Shared\RealNerModelFixture.cs" />
+    <Compile Include="..\TestSupport.Shared\RealTamilNerModelPaths.cs" Link="TestSupport.Shared\RealTamilNerModelPaths.cs" />
+    <Compile Include="..\TestSupport.Shared\RealTamilNerModelFixture.cs" Link="TestSupport.Shared\RealTamilNerModelFixture.cs" />
+    <Compile Include="..\TestSupport.Shared\RealRoutingNerModelFixture.cs" Link="TestSupport.Shared\RealRoutingNerModelFixture.cs" />
  </ItemGroup>

  <ItemGroup>
--- a/tests/PiiRedaction.Core.Tests/TestSupport/ProductionPipelineFactory.cs
+++ b/tests/PiiRedaction.Core.Tests/TestSupport/ProductionPipelineFactory.cs
@@ -1,7 +1,10 @@
+using Microsoft.Extensions.Options;
 using PiiRedaction.Core.Abstractions;
+using PiiRedaction.Core.Configuration;
 using PiiRedaction.Core.Detection;
 using PiiRedaction.Core.Redaction;
 using PiiRedaction.Core.Sanitization;
+using PiiRedaction.Infrastructure.Onnx;

 namespace PiiRedaction.Core.Tests.TestSupport;

@@ -10,6 +13,15 @@ public static class ProductionPipelineFactory
    public static IPromptSanitizer CreateWithRealModel(IOnnxNerModelRunner runner) =>
        new PromptSanitizer(CreateCompositeDetector(runner), new PlaceholderPiiRedactor());

+    public static IPromptSanitizer CreateWithRoutingRealModels(
+        EnglishOnnxNerRunner englishRunner,
+        TamilOnnxNerRunner tamilRunner,
+        IOptions<PiiRedactionOptions>? options = null) =>
+        CreateWithRealModel(new RoutingOnnxNerModelRunner(
+            englishRunner,
+            tamilRunner,
+            options ?? Options.Create(new PiiRedactionOptions { EnableTamilNer = true })));
+
    public static IPiiDetector CreateCompositeDetector(IOnnxNerModelRunner runner) =>
        new CompositePiiDetector(
        [
--- a/tests/PiiRedaction.Core.Tests/TestSupport/TamilPromptScenarioCatalog.cs
+++ b/tests/PiiRedaction.Core.Tests/TestSupport/TamilPromptScenarioCatalog.cs
@@ -0,0 +1,63 @@
+using NUnit.Framework;
+using PiiRedaction.Core.Models;
+
+namespace PiiRedaction.Core.Tests.TestSupport;
+
+/// <summary>
+/// Golden end-to-end scenarios for Tamil script, Tanglish, and mixed-script prompts.
+/// Uses fake NER spans for person names; regex and domain rules run for real.
+/// </summary>
+public static class TamilPromptScenarioCatalog
+{
+    public static IEnumerable<TestCaseData> AllScenarios()
+    {
+        foreach (var scenario in BuildScenarios())
+        {
+            yield return new TestCaseData(scenario).SetName(scenario.Name);
+        }
+    }
+
+    private static IEnumerable<PromptScenario> BuildScenarios()
+    {
+        yield return TamilCustomerNameOnly();
+        yield return TamilWithPhonePan();
+        yield return TanglishCustomer();
+        yield return MixedTamilEnglish();
+        yield return TamilFullFinancial();
+    }
+
+    private static PromptScenario TamilCustomerNameOnly() => new(
+        "Tamil_CustomerNameOnly",
+        "வாடிக்கையாளர் ராஜேஷ் குமார் சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்.",
+        "வாடிக்கையாளர் <PERSON_1> சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்.",
+        [PiiEntityType.Person],
+        ["ராஜேஷ் குமார்"]);
+
+    private static PromptScenario TamilWithPhonePan() => new(
+        "Tamil_WithPhonePan",
+        "வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210 PAN ABCDE1234F.",
+        "வாடிக்கையாளர் <PERSON_1> தொலைபேசி <PHONE_1> PAN <PAN_1>.",
+        [PiiEntityType.Person, PiiEntityType.Phone, PiiEntityType.Pan],
+        ["ராஜேஷ் குமார்", "9876543210", "ABCDE1234F"]);
+
+    private static PromptScenario TanglishCustomer() => new(
+        "Tanglish_CustomerPhone",
+        "Customer Senthil phone 9876543210 reported a failed UPI transfer.",
+        "Customer <PERSON_1> phone <PHONE_1> reported a failed UPI transfer.",
+        [PiiEntityType.Person, PiiEntityType.Phone],
+        ["Senthil", "9876543210"]);
+
+    private static PromptScenario MixedTamilEnglish() => new(
+        "Mixed_TamilEnglish",
+        "வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge.",
+        "வாடிக்கையாளர் <PERSON_1> phone <PHONE_1> disputed the charge.",
+        [PiiEntityType.Person, PiiEntityType.Phone],
+        ["Ravi Kumar", "9876543210"]);
+
+    private static PromptScenario TamilFullFinancial() => new(
+        "Tamil_FullFinancial",
+        "வாடிக்கையாளர் ராஜேஷ் குமார் மின்னஞ்சல் ravi.kumar@gmail.com தொலைபேசி 9876543210 LoanNumber LN-456789 PAN ABCDE1234F. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்.",
+        "வாடிக்கையாளர் <PERSON_1> மின்னஞ்சல் <EMAIL_1> தொலைபேசி <PHONE_1> LoanNumber <LOAN_NUMBER_1> PAN <PAN_1>. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்.",
+        [PiiEntityType.Person, PiiEntityType.Email, PiiEntityType.Phone, PiiEntityType.LoanNumber, PiiEntityType.Pan],
+        ["ராஜேஷ் குமார்", "ravi.kumar@gmail.com", "9876543210", "LN-456789", "ABCDE1234F"]);
+}