Add Tamil NER routing and WPF test harness for POC validation.

Introduce dual-script ONNX NER routing (English/Tamil/mixed), Tamil console samples and integration tests, model download scripts, and a resizable WPF MVVM harness with click-to-load prompts, batch validation, and runtime-adjustable detection panels.
2026-07-07 17:12:38 +05:30
parent a707c6c9cf
commit cf8f5a7232
71 changed files with 4494 additions and 356 deletions
--- a/tests/PiiRedaction.Core.Tests/Detection/ScriptRouterTests.cs
+++ b/tests/PiiRedaction.Core.Tests/Detection/ScriptRouterTests.cs
@@ -0,0 +1,54 @@
+using FluentAssertions;
+using PiiRedaction.Core.Detection;
+using PiiRedaction.Infrastructure.Onnx;
+
+namespace PiiRedaction.Core.Tests.Detection;
+
+[TestFixture]
+public sealed class ScriptRouterTests
+{
+    private readonly ScriptRouter _router = new();
+
+    [Test]
+    public void GetComposition_LatinOnly_ReturnsLatinOnly()
+    {
+        _router.GetComposition("Customer Ravi Kumar called about billing.")
+            .Should().Be(ScriptComposition.LatinOnly);
+    }
+
+    [Test]
+    public void GetComposition_TamilOnly_ReturnsTamilOnly()
+    {
+        _router.GetComposition("வாடிக்கையாளர் ராஜேஷ் தொலைபேசி 9876543210")
+            .Should().Be(ScriptComposition.TamilOnly);
+    }
+
+    [Test]
+    public void GetComposition_Mixed_ReturnsMixed()
+    {
+        _router.GetComposition("Rajesh மற்றும் Priya disputed the charge.")
+            .Should().Be(ScriptComposition.Mixed);
+    }
+
+    [Test]
+    public void GetComposition_MixedTamilEnglishSample_ReturnsMixed()
+    {
+        _router.GetComposition("வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge.")
+            .Should().Be(ScriptComposition.Mixed);
+    }
+
+    [Test]
+    public void GetComposition_NoLetters_ReturnsNoLetters()
+    {
+        _router.GetComposition("9876543210 12345")
+            .Should().Be(ScriptComposition.NoLetters);
+    }
+
+    [Test]
+    public void GetComposition_TamilBoundaryChars_AreClassifiedAsTamil()
+    {
+        _router.GetComposition("\u0B80").Should().Be(ScriptComposition.TamilOnly);
+        _router.GetComposition("\u0BFF").Should().Be(ScriptComposition.TamilOnly);
+        _router.GetComposition("A").Should().Be(ScriptComposition.LatinOnly);
+    }
+}
--- a/tests/PiiRedaction.Core.Tests/Integration/GoldenPromptTests.cs
+++ b/tests/PiiRedaction.Core.Tests/Integration/GoldenPromptTests.cs
@@ -8,6 +8,7 @@ namespace PiiRedaction.Core.Tests.Integration;
 public sealed class GoldenPromptTests
 {
    [TestCaseSource(typeof(PromptScenarioCatalog), nameof(PromptScenarioCatalog.AllScenarios))]
+    [TestCaseSource(typeof(TamilPromptScenarioCatalog), nameof(TamilPromptScenarioCatalog.AllScenarios))]
    public void Sanitize_PromptScenario_ProducesExpectedOutput(PromptScenario scenario)
    {
        var sanitizer = ProductionPipelineFactory.CreateForScenario(scenario);
--- a/tests/PiiRedaction.Core.Tests/Integration/RealTamilPipelineTests.cs
+++ b/tests/PiiRedaction.Core.Tests/Integration/RealTamilPipelineTests.cs
@@ -0,0 +1,106 @@
+using FluentAssertions;
+using PiiRedaction.Core.Abstractions;
+using PiiRedaction.Core.Models;
+using PiiRedaction.Core.Tests.TestSupport;
+using PiiRedaction.Tests.Shared;
+
+namespace PiiRedaction.Core.Tests.Integration;
+
+/// <summary>
+/// End-to-end pipeline proof using routed English + Tamil ONNX NER models.
+/// </summary>
+[TestFixture]
+[Category("TamilNer")]
+public sealed class RealTamilPipelineTests : RealRoutingNerModelFixture
+{
+    private IPromptSanitizer _sanitizer = null!;
+
+    [OneTimeSetUp]
+    public void OneTimeSetUpPipeline()
+    {
+        _sanitizer = ProductionPipelineFactory.CreateWithRoutingRealModels(EnglishRunner, TamilRunner);
+    }
+
+    [Test]
+    public void Sanitize_TamilCustomerNameOnly_RedactsPerson()
+    {
+        const string prompt =
+            "வாடிக்கையாளர் ராஜேஷ் குமார் சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்.";
+
+        var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+        result.SanitizedPrompt.Should().Contain("<PERSON_1>");
+        result.SanitizedPrompt.Should().NotContain("ராஜேஷ்");
+        result.DetectedEntities.Should().Contain(entity =>
+            entity.Type == PiiEntityType.Person && entity.Source == PiiDetectionSource.Ner);
+    }
+
+    [Test]
+    public void Sanitize_TamilWithPhonePan_RedactsPersonPhoneAndPan()
+    {
+        const string prompt = "வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210 PAN ABCDE1234F.";
+
+        var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+        result.SanitizedPrompt.Should().Contain("<PERSON_1>");
+        result.SanitizedPrompt.Should().Contain("<PHONE_1>");
+        result.SanitizedPrompt.Should().Contain("<PAN_1>");
+        result.SanitizedPrompt.Should().NotContainAny("ராஜேஷ்", "9876543210", "ABCDE1234F");
+    }
+
+    [Test]
+    public void Sanitize_TanglishCustomer_RedactsPersonAndPhone()
+    {
+        const string prompt = "Customer Senthil phone 9876543210 reported a failed UPI transfer.";
+
+        var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+        result.SanitizedPrompt.Should().Contain("<PERSON_1>");
+        result.SanitizedPrompt.Should().Contain("<PHONE_1>");
+        result.SanitizedPrompt.Should().NotContainAny("Senthil", "9876543210");
+    }
+
+    [Test]
+    public void Sanitize_MixedTamilEnglish_RedactsPersonAndPhone()
+    {
+        const string prompt = "வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge.";
+
+        var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+        result.SanitizedPrompt.Should().Contain("<PERSON_1>");
+        result.SanitizedPrompt.Should().Contain("<PHONE_1>");
+        result.SanitizedPrompt.Should().NotContainAny("Ravi Kumar", "9876543210");
+    }
+
+    [Test]
+    public void Sanitize_TamilFullFinancial_RedactsAllPiiTypes()
+    {
+        const string prompt =
+            "வாடிக்கையாளர் ராஜேஷ் குமார் மின்னஞ்சல் ravi.kumar@gmail.com தொலைபேசி 9876543210 LoanNumber LN-456789 PAN ABCDE1234F. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்.";
+
+        var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+        result.SanitizedPrompt.Should().Contain("<PERSON_1>");
+        result.SanitizedPrompt.Should().Contain("<EMAIL_1>");
+        result.SanitizedPrompt.Should().Contain("<PHONE_1>");
+        result.SanitizedPrompt.Should().Contain("<LOAN_NUMBER_1>");
+        result.SanitizedPrompt.Should().Contain("<PAN_1>");
+        result.SanitizedPrompt.Should().NotContainAny(
+            "ராஜேஷ்",
+            "ravi.kumar@gmail.com",
+            "9876543210",
+            "LN-456789",
+            "ABCDE1234F");
+    }
+
+    [Test]
+    public void Sanitize_CleanTamilQuestion_PassesThroughWithoutPersonRedaction()
+    {
+        const string prompt = "பணத்தை திரும்பப் பெறுவது எப்படி?";
+
+        var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+        result.SanitizedPrompt.Should().Be(prompt);
+        result.DetectedEntities.Should().BeEmpty();
+    }
+}
--- a/tests/PiiRedaction.Core.Tests/PiiRedaction.Core.Tests.csproj
+++ b/tests/PiiRedaction.Core.Tests/PiiRedaction.Core.Tests.csproj
@@ -25,6 +25,9 @@
  <ItemGroup>
    <Compile Include="..\TestSupport.Shared\RealNerModelPaths.cs" Link="TestSupport.Shared\RealNerModelPaths.cs" />
    <Compile Include="..\TestSupport.Shared\RealNerModelFixture.cs" Link="TestSupport.Shared\RealNerModelFixture.cs" />
+    <Compile Include="..\TestSupport.Shared\RealTamilNerModelPaths.cs" Link="TestSupport.Shared\RealTamilNerModelPaths.cs" />
+    <Compile Include="..\TestSupport.Shared\RealTamilNerModelFixture.cs" Link="TestSupport.Shared\RealTamilNerModelFixture.cs" />
+    <Compile Include="..\TestSupport.Shared\RealRoutingNerModelFixture.cs" Link="TestSupport.Shared\RealRoutingNerModelFixture.cs" />
  </ItemGroup>

  <ItemGroup>
--- a/tests/PiiRedaction.Core.Tests/TestSupport/ProductionPipelineFactory.cs
+++ b/tests/PiiRedaction.Core.Tests/TestSupport/ProductionPipelineFactory.cs
@@ -1,7 +1,10 @@
+using Microsoft.Extensions.Options;
 using PiiRedaction.Core.Abstractions;
+using PiiRedaction.Core.Configuration;
 using PiiRedaction.Core.Detection;
 using PiiRedaction.Core.Redaction;
 using PiiRedaction.Core.Sanitization;
+using PiiRedaction.Infrastructure.Onnx;

 namespace PiiRedaction.Core.Tests.TestSupport;

@@ -10,6 +13,15 @@ public static class ProductionPipelineFactory
    public static IPromptSanitizer CreateWithRealModel(IOnnxNerModelRunner runner) =>
        new PromptSanitizer(CreateCompositeDetector(runner), new PlaceholderPiiRedactor());

+    public static IPromptSanitizer CreateWithRoutingRealModels(
+        EnglishOnnxNerRunner englishRunner,
+        TamilOnnxNerRunner tamilRunner,
+        IOptions<PiiRedactionOptions>? options = null) =>
+        CreateWithRealModel(new RoutingOnnxNerModelRunner(
+            englishRunner,
+            tamilRunner,
+            options ?? Options.Create(new PiiRedactionOptions { EnableTamilNer = true })));
+
    public static IPiiDetector CreateCompositeDetector(IOnnxNerModelRunner runner) =>
        new CompositePiiDetector(
        [
--- a/tests/PiiRedaction.Core.Tests/TestSupport/TamilPromptScenarioCatalog.cs
+++ b/tests/PiiRedaction.Core.Tests/TestSupport/TamilPromptScenarioCatalog.cs
@@ -0,0 +1,63 @@
+using NUnit.Framework;
+using PiiRedaction.Core.Models;
+
+namespace PiiRedaction.Core.Tests.TestSupport;
+
+/// <summary>
+/// Golden end-to-end scenarios for Tamil script, Tanglish, and mixed-script prompts.
+/// Uses fake NER spans for person names; regex and domain rules run for real.
+/// </summary>
+public static class TamilPromptScenarioCatalog
+{
+    public static IEnumerable<TestCaseData> AllScenarios()
+    {
+        foreach (var scenario in BuildScenarios())
+        {
+            yield return new TestCaseData(scenario).SetName(scenario.Name);
+        }
+    }
+
+    private static IEnumerable<PromptScenario> BuildScenarios()
+    {
+        yield return TamilCustomerNameOnly();
+        yield return TamilWithPhonePan();
+        yield return TanglishCustomer();
+        yield return MixedTamilEnglish();
+        yield return TamilFullFinancial();
+    }
+
+    private static PromptScenario TamilCustomerNameOnly() => new(
+        "Tamil_CustomerNameOnly",
+        "வாடிக்கையாளர் ராஜேஷ் குமார் சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்.",
+        "வாடிக்கையாளர் <PERSON_1> சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்.",
+        [PiiEntityType.Person],
+        ["ராஜேஷ் குமார்"]);
+
+    private static PromptScenario TamilWithPhonePan() => new(
+        "Tamil_WithPhonePan",
+        "வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210 PAN ABCDE1234F.",
+        "வாடிக்கையாளர் <PERSON_1> தொலைபேசி <PHONE_1> PAN <PAN_1>.",
+        [PiiEntityType.Person, PiiEntityType.Phone, PiiEntityType.Pan],
+        ["ராஜேஷ் குமார்", "9876543210", "ABCDE1234F"]);
+
+    private static PromptScenario TanglishCustomer() => new(
+        "Tanglish_CustomerPhone",
+        "Customer Senthil phone 9876543210 reported a failed UPI transfer.",
+        "Customer <PERSON_1> phone <PHONE_1> reported a failed UPI transfer.",
+        [PiiEntityType.Person, PiiEntityType.Phone],
+        ["Senthil", "9876543210"]);
+
+    private static PromptScenario MixedTamilEnglish() => new(
+        "Mixed_TamilEnglish",
+        "வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge.",
+        "வாடிக்கையாளர் <PERSON_1> phone <PHONE_1> disputed the charge.",
+        [PiiEntityType.Person, PiiEntityType.Phone],
+        ["Ravi Kumar", "9876543210"]);
+
+    private static PromptScenario TamilFullFinancial() => new(
+        "Tamil_FullFinancial",
+        "வாடிக்கையாளர் ராஜேஷ் குமார் மின்னஞ்சல் ravi.kumar@gmail.com தொலைபேசி 9876543210 LoanNumber LN-456789 PAN ABCDE1234F. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்.",
+        "வாடிக்கையாளர் <PERSON_1> மின்னஞ்சல் <EMAIL_1> தொலைபேசி <PHONE_1> LoanNumber <LOAN_NUMBER_1> PAN <PAN_1>. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்.",
+        [PiiEntityType.Person, PiiEntityType.Email, PiiEntityType.Phone, PiiEntityType.LoanNumber, PiiEntityType.Pan],
+        ["ராஜேஷ் குமார்", "ravi.kumar@gmail.com", "9876543210", "LN-456789", "ABCDE1234F"]);
+}
--- a/tests/PiiRedaction.Infrastructure.Tests/Onnx/NerLabelConfigTests.cs
+++ b/tests/PiiRedaction.Infrastructure.Tests/Onnx/NerLabelConfigTests.cs
@@ -0,0 +1,28 @@
+using FluentAssertions;
+using PiiRedaction.Core.Detection;
+using PiiRedaction.Core.Models;
+using PiiRedaction.Infrastructure.Onnx;
+
+namespace PiiRedaction.Infrastructure.Tests.Onnx;
+
+[TestFixture]
+public sealed class NerLabelConfigTests
+{
+    [TestCase("B-PER", true)]
+    [TestCase("I-PER", true)]
+    [TestCase("B-PERSON", true)]
+    [TestCase("B-ORG", false)]
+    public void English_IsPersonLabel_MatchesExpected(string label, bool expected)
+    {
+        NerLabelConfig.English.IsPersonLabel(label).Should().Be(expected);
+    }
+
+    [TestCase("B-person-politician", true)]
+    [TestCase("I-person-artist", true)]
+    [TestCase("B-location", false)]
+    [TestCase("O", false)]
+    public void Tamil_IsPersonLabel_MatchesFineGrainedTags(string label, bool expected)
+    {
+        NerLabelConfig.Tamil.IsPersonLabel(label).Should().Be(expected);
+    }
+}
--- a/tests/PiiRedaction.Infrastructure.Tests/Onnx/RealTamilNerModelRunnerTests.cs
+++ b/tests/PiiRedaction.Infrastructure.Tests/Onnx/RealTamilNerModelRunnerTests.cs
@@ -0,0 +1,70 @@
+using FluentAssertions;
+using PiiRedaction.Core.Models;
+using PiiRedaction.Infrastructure.Onnx;
+using PiiRedaction.Tests.Shared;
+
+namespace PiiRedaction.Infrastructure.Tests.Onnx;
+
+[TestFixture]
+[Category("TamilNer")]
+public sealed class RealTamilNerModelRunnerTests : RealTamilNerModelFixture
+{
+    [Test]
+    public void IsModelAvailable_LoadsOnnxAndTokenizer()
+    {
+        Runner.IsModelAvailable.Should().BeTrue();
+    }
+
+    [TestCase("வாடிக்கையாளர் ராஜேஷ் குமார் அழைத்தார்.", "ராஜேஷ்", "ராஜேஷ் குமார்")]
+    public void PredictEntities_TamilScript_DetectsPersonEntity(
+        string prompt,
+        string expectedNamePart,
+        string expectedValue)
+    {
+        var entities = Runner.PredictEntities(prompt);
+
+        entities.Should().Contain(entity =>
+            entity.Type == PiiEntityType.Person &&
+            entity.Source == PiiDetectionSource.Ner &&
+            entity.Value.Contains(expectedNamePart, StringComparison.Ordinal) &&
+            prompt.AsSpan(entity.StartIndex, entity.Length).ToString() == entity.Value);
+
+        entities.Should().Contain(entity => entity.Value == expectedValue);
+    }
+
+    [Test]
+    public void PredictEntities_TamilWithPhone_DetectsPersonAndLeavesPhoneToRegex()
+    {
+        const string prompt = "வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210";
+
+        var entities = Runner.PredictEntities(prompt);
+
+        entities.Should().Contain(entity =>
+            entity.Type == PiiEntityType.Person &&
+            entity.Source == PiiDetectionSource.Ner &&
+            entity.Value.Contains("ராஜேஷ்", StringComparison.Ordinal));
+        entities.Should().NotContain(entity => entity.Type == PiiEntityType.Phone);
+    }
+
+    [Test]
+    public void PredictEntities_CleanTamilQuestion_ReturnsNoEntities()
+    {
+        const string prompt = "பணத்தை திரும்பப் பெறுவது எப்படி?";
+
+        Runner.PredictEntities(prompt).Should().BeEmpty();
+    }
+
+    [TestCase("Customer Senthil phone 9876543210", "Senthil")]
+    public void PredictEntities_TanglishLatinScript_DoesNotInvokeTamilRunner(
+        string prompt,
+        string expectedNamePart)
+    {
+        // TamilOnnxNerRunner is script-scoped; Tanglish is handled by English routing in pipeline tests.
+        // Direct Tamil runner on Latin-only text should not emit person spans.
+        var entities = Runner.PredictEntities(prompt);
+
+        entities.Should().NotContain(entity =>
+            entity.Type == PiiEntityType.Person &&
+            entity.Value.Contains(expectedNamePart, StringComparison.OrdinalIgnoreCase));
+    }
+}
--- a/tests/PiiRedaction.Infrastructure.Tests/Onnx/RoutingOnnxNerModelRunnerTests.cs
+++ b/tests/PiiRedaction.Infrastructure.Tests/Onnx/RoutingOnnxNerModelRunnerTests.cs
@@ -0,0 +1,118 @@
+using FluentAssertions;
+using PiiRedaction.Core.Detection;
+using PiiRedaction.Core.Models;
+using PiiRedaction.Infrastructure.Onnx;
+
+namespace PiiRedaction.Infrastructure.Tests.Onnx;
+
+[TestFixture]
+public sealed class RoutingOnnxNerModelRunnerTests
+{
+    [Test]
+    public void PredictEntities_LatinOnly_UsesEnglishRunnerOnly()
+    {
+        var english = new FakeLanguageNerRunner("Ravi Kumar");
+        var tamil = new FakeLanguageNerRunner("தமிழ் பெயர்");
+        var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
+
+        var entities = router.PredictEntities("Customer Ravi Kumar called.");
+
+        entities.Should().ContainSingle(entity => entity.Value == "Ravi Kumar");
+        english.CallCount.Should().Be(1);
+        tamil.CallCount.Should().Be(0);
+    }
+
+    [Test]
+    public void PredictEntities_TamilOnly_UsesTamilRunnerOnly()
+    {
+        var english = new FakeLanguageNerRunner("Ravi Kumar");
+        var tamil = new FakeLanguageNerRunner("ராஜேஷ்");
+        var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
+
+        var entities = router.PredictEntities("வாடிக்கையாளர் ராஜேஷ்");
+
+        entities.Should().ContainSingle(entity => entity.Value == "ராஜேஷ்");
+        english.CallCount.Should().Be(0);
+        tamil.CallCount.Should().Be(1);
+    }
+
+    [Test]
+    public void PredictEntities_Mixed_InvokesBothRunners()
+    {
+        var english = new FakeLanguageNerRunner("EnglishName");
+        var tamil = new FakeLanguageNerRunner("தமிழ்");
+        var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
+
+        router.PredictEntities("Rajesh மற்றும் Priya");
+
+        english.CallCount.Should().Be(1);
+        tamil.CallCount.Should().Be(1);
+    }
+
+    [Test]
+    public void PredictEntities_NoLetters_InvokesNeither()
+    {
+        var english = new FakeLanguageNerRunner("ignored");
+        var tamil = new FakeLanguageNerRunner("ignored");
+        var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
+
+        router.PredictEntities("9876543210").Should().BeEmpty();
+        english.CallCount.Should().Be(0);
+        tamil.CallCount.Should().Be(0);
+    }
+
+    [Test]
+    public void PredictEntities_TamilDisabled_SkipsTamilRunnerForMixedText()
+    {
+        var english = new FakeLanguageNerRunner("EnglishName");
+        var tamil = new FakeLanguageNerRunner("தமிழ்");
+        var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: false);
+
+        router.PredictEntities("Rajesh மற்றும் Priya");
+
+        english.CallCount.Should().Be(1);
+        tamil.CallCount.Should().Be(0);
+    }
+
+    [Test]
+    public void MergePersonSpans_PrefersLongerOverlappingSpan()
+    {
+        var entities = new[]
+        {
+            CreatePerson("Raj", 0, 3),
+            CreatePerson("Rajesh", 0, 6)
+        };
+
+        var merged = RoutingOnnxNerModelRunner.MergePersonSpans(entities);
+
+        merged.Should().ContainSingle(entity => entity.Value == "Rajesh");
+    }
+
+    private static PiiEntity CreatePerson(string value, int start, int length) =>
+        new(PiiEntityType.Person, value, start, length, PiiDetectionSource.Ner);
+
+    private sealed class FakeLanguageNerRunner : IOnnxNerModelRunner
+    {
+        private readonly string _personValue;
+
+        public FakeLanguageNerRunner(string personValue) => _personValue = personValue;
+
+        public int CallCount { get; private set; }
+
+        public bool IsModelAvailable => true;
+
+        public IReadOnlyList<PiiEntity> PredictEntities(string text)
+        {
+            CallCount++;
+            return
+            [
+                new PiiEntity(
+                    PiiEntityType.Person,
+                    _personValue,
+                    0,
+                    _personValue.Length,
+                    PiiDetectionSource.Ner)
+            ];
+        }
+    }
+}
--- a/tests/PiiRedaction.Infrastructure.Tests/Onnx/TokenClassifierEncoderFactoryTests.cs
+++ b/tests/PiiRedaction.Infrastructure.Tests/Onnx/TokenClassifierEncoderFactoryTests.cs
@@ -0,0 +1,43 @@
+using FluentAssertions;
+using Microsoft.Extensions.Logging.Abstractions;
+using PiiRedaction.Infrastructure.Onnx;
+
+namespace PiiRedaction.Infrastructure.Tests.Onnx;
+
+[TestFixture]
+public sealed class TokenClassifierEncoderFactoryTests
+{
+    [Test]
+    public void Create_PrefersWordPieceWhenVocabExists()
+    {
+        var modelDirectory = ResolveTamilModelDirectory();
+        if (!File.Exists(Path.Combine(modelDirectory, "vocab.txt")))
+        {
+            Assert.Ignore("Tamil vocab.txt not found.");
+        }
+
+        var encoder = TokenClassifierEncoderFactory.Create(
+            modelDirectory,
+            NullLogger.Instance);
+
+        encoder.Should().BeOfType<BertWordPieceEncoder>();
+        encoder.IsAvailable.Should().BeTrue();
+    }
+
+    private static string ResolveTamilModelDirectory()
+    {
+        var directory = new DirectoryInfo(AppContext.BaseDirectory);
+        while (directory is not null)
+        {
+            var candidate = Path.Combine(directory.FullName, "models", "ta");
+            if (Directory.Exists(candidate))
+            {
+                return candidate;
+            }
+
+            directory = directory.Parent;
+        }
+
+        return Path.Combine(Environment.CurrentDirectory, "models", "ta");
+    }
+}
--- a/tests/PiiRedaction.Infrastructure.Tests/PiiRedaction.Infrastructure.Tests.csproj
+++ b/tests/PiiRedaction.Infrastructure.Tests/PiiRedaction.Infrastructure.Tests.csproj
@@ -24,6 +24,8 @@
  <ItemGroup>
    <Compile Include="..\TestSupport.Shared\RealNerModelPaths.cs" Link="TestSupport.Shared\RealNerModelPaths.cs" />
    <Compile Include="..\TestSupport.Shared\RealNerModelFixture.cs" Link="TestSupport.Shared\RealNerModelFixture.cs" />
+    <Compile Include="..\TestSupport.Shared\RealTamilNerModelPaths.cs" Link="TestSupport.Shared\RealTamilNerModelPaths.cs" />
+    <Compile Include="..\TestSupport.Shared\RealTamilNerModelFixture.cs" Link="TestSupport.Shared\RealTamilNerModelFixture.cs" />
  </ItemGroup>

  <ItemGroup>
--- a/tests/TestSupport.Shared/RealNerModelFixture.cs
+++ b/tests/TestSupport.Shared/RealNerModelFixture.cs
@@ -6,12 +6,12 @@ using PiiRedaction.Infrastructure.Onnx;
 namespace PiiRedaction.Tests.Shared;

 /// <summary>
-/// Reuses a single <see cref="OnnxNerModelRunner"/> per fixture for performance.
+/// Reuses a single <see cref="EnglishOnnxNerRunner"/> per fixture for performance.
 /// Skips all tests in the class when the ONNX model is missing or cannot be loaded.
 /// </summary>
 public abstract class RealNerModelFixture
 {
-    protected OnnxNerModelRunner Runner { get; private set; } = null!;
+    protected EnglishOnnxNerRunner Runner { get; private set; } = null!;

    protected string ModelPath { get; private set; } = null!;

@@ -24,8 +24,12 @@ public abstract class RealNerModelFixture
            Assert.Ignore(RealNerModelPaths.ModelMissingMessage);
        }

-        var options = Options.Create(new PiiRedactionOptions { OnnxModelPath = ModelPath });
-        Runner = new OnnxNerModelRunner(options, NullLogger<OnnxNerModelRunner>.Instance);
+        var options = Options.Create(new PiiRedactionOptions
+        {
+            EnglishOnnxModelPath = ModelPath,
+            OnnxModelPath = ModelPath
+        });
+        Runner = new EnglishOnnxNerRunner(options, NullLogger<EnglishOnnxNerRunner>.Instance);

        if (!Runner.IsModelAvailable)
        {
--- a/tests/TestSupport.Shared/RealNerModelPaths.cs
+++ b/tests/TestSupport.Shared/RealNerModelPaths.cs
@@ -7,16 +7,23 @@ public static class RealNerModelPaths

    public static string ResolveRepoModelPath()
    {
-        var directory = new DirectoryInfo(AppContext.BaseDirectory);
-        while (directory is not null)
+        foreach (var relativePath in new[]
+                 {
+                     Path.Combine("models", "en", "ner-model.onnx"),
+                     Path.Combine("models", "ner-model.onnx")
+                 })
        {
-            var candidate = Path.Combine(directory.FullName, "models", "ner-model.onnx");
-            if (File.Exists(candidate))
+            var directory = new DirectoryInfo(AppContext.BaseDirectory);
+            while (directory is not null)
            {
-                return candidate;
-            }
+                var candidate = Path.Combine(directory.FullName, relativePath);
+                if (File.Exists(candidate))
+                {
+                    return candidate;
+                }

-            directory = directory.Parent;
+                directory = directory.Parent;
+            }
        }

        return Path.Combine(Environment.CurrentDirectory, "models", "ner-model.onnx");
--- a/tests/TestSupport.Shared/RealRoutingNerModelFixture.cs
+++ b/tests/TestSupport.Shared/RealRoutingNerModelFixture.cs
@@ -0,0 +1,69 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using PiiRedaction.Core.Configuration;
+using PiiRedaction.Infrastructure.Onnx;
+
+namespace PiiRedaction.Tests.Shared;
+
+/// <summary>
+/// Loads English and Tamil ONNX runners and exposes a <see cref="RoutingOnnxNerModelRunner"/>.
+/// Skips when either model is missing or cannot be loaded.
+/// </summary>
+public abstract class RealRoutingNerModelFixture
+{
+    protected RoutingOnnxNerModelRunner Runner { get; private set; } = null!;
+
+    protected EnglishOnnxNerRunner EnglishRunner { get; private set; } = null!;
+
+    protected TamilOnnxNerRunner TamilRunner { get; private set; } = null!;
+
+    [OneTimeSetUp]
+    public void OneTimeSetUpRoutingModels()
+    {
+        var englishPath = RealNerModelPaths.ResolveRepoModelPath();
+        if (!File.Exists(englishPath))
+        {
+            Assert.Ignore(RealNerModelPaths.ModelMissingMessage);
+        }
+
+        var tamilPath = RealTamilNerModelPaths.ResolveRepoModelPath();
+        if (!File.Exists(tamilPath))
+        {
+            Assert.Ignore(RealTamilNerModelPaths.ModelMissingMessage);
+        }
+
+        var options = Options.Create(new PiiRedactionOptions
+        {
+            EnglishOnnxModelPath = englishPath,
+            OnnxModelPath = englishPath,
+            TamilOnnxModelPath = tamilPath,
+            EnableTamilNer = true
+        });
+
+        EnglishRunner = new EnglishOnnxNerRunner(options, NullLogger<EnglishOnnxNerRunner>.Instance);
+        TamilRunner = new TamilOnnxNerRunner(options, NullLogger<TamilOnnxNerRunner>.Instance);
+
+        if (!EnglishRunner.IsModelAvailable)
+        {
+            EnglishRunner.Dispose();
+            TamilRunner.Dispose();
+            Assert.Ignore(RealNerModelPaths.ModelMissingMessage);
+        }
+
+        if (!TamilRunner.IsModelAvailable)
+        {
+            EnglishRunner.Dispose();
+            TamilRunner.Dispose();
+            Assert.Ignore(RealTamilNerModelPaths.ModelMissingMessage);
+        }
+
+        Runner = new RoutingOnnxNerModelRunner(EnglishRunner, TamilRunner, options);
+    }
+
+    [OneTimeTearDown]
+    public void OneTimeTearDownRoutingModels()
+    {
+        EnglishRunner?.Dispose();
+        TamilRunner?.Dispose();
+    }
+}
--- a/tests/TestSupport.Shared/RealTamilNerModelFixture.cs
+++ b/tests/TestSupport.Shared/RealTamilNerModelFixture.cs
@@ -0,0 +1,45 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using PiiRedaction.Core.Configuration;
+using PiiRedaction.Infrastructure.Onnx;
+
+namespace PiiRedaction.Tests.Shared;
+
+/// <summary>
+/// Reuses a single <see cref="TamilOnnxNerRunner"/> per fixture for performance.
+/// Skips all tests in the class when the Tamil ONNX model is missing or cannot be loaded.
+/// </summary>
+public abstract class RealTamilNerModelFixture
+{
+    protected TamilOnnxNerRunner Runner { get; private set; } = null!;
+
+    protected string ModelPath { get; private set; } = null!;
+
+    [OneTimeSetUp]
+    public void OneTimeSetUpTamilModel()
+    {
+        ModelPath = RealTamilNerModelPaths.ResolveRepoModelPath();
+        if (!File.Exists(ModelPath))
+        {
+            Assert.Ignore(RealTamilNerModelPaths.ModelMissingMessage);
+        }
+
+        var options = Options.Create(new PiiRedactionOptions
+        {
+            TamilOnnxModelPath = ModelPath
+        });
+        Runner = new TamilOnnxNerRunner(options, NullLogger<TamilOnnxNerRunner>.Instance);
+
+        if (!Runner.IsModelAvailable)
+        {
+            Runner.Dispose();
+            Assert.Ignore(RealTamilNerModelPaths.ModelMissingMessage);
+        }
+    }
+
+    [OneTimeTearDown]
+    public void OneTimeTearDownTamilModel()
+    {
+        Runner?.Dispose();
+    }
+}
--- a/tests/TestSupport.Shared/RealTamilNerModelPaths.cs
+++ b/tests/TestSupport.Shared/RealTamilNerModelPaths.cs
@@ -0,0 +1,25 @@
+namespace PiiRedaction.Tests.Shared;
+
+public static class RealTamilNerModelPaths
+{
+    public const string ModelMissingMessage =
+        "Tamil ONNX model not found. Run scripts/download-tamil-ner-model.ps1 from the repository root.";
+
+    public static string ResolveRepoModelPath()
+    {
+        const string relativePath = "models/ta/model.onnx";
+        var directory = new DirectoryInfo(AppContext.BaseDirectory);
+        while (directory is not null)
+        {
+            var candidate = Path.Combine(directory.FullName, relativePath);
+            if (File.Exists(candidate))
+            {
+                return candidate;
+            }
+
+            directory = directory.Parent;
+        }
+
+        return Path.Combine(Environment.CurrentDirectory, relativePath);
+    }
+}