Add Tamil NER routing and WPF test harness for POC validation.
Introduce dual-script ONNX NER routing (English/Tamil/mixed), Tamil console samples and integration tests, model download scripts, and a resizable WPF MVVM harness with click-to-load prompts, batch validation, and runtime-adjustable detection panels.
This commit is contained in:
Bilal Nazer Ali
@@ -0,0 +1,28 @@
|
||||
using FluentAssertions;
|
||||
using PiiRedaction.Core.Detection;
|
||||
using PiiRedaction.Core.Models;
|
||||
using PiiRedaction.Infrastructure.Onnx;
|
||||
|
||||
namespace PiiRedaction.Infrastructure.Tests.Onnx;
|
||||
|
||||
[TestFixture]
|
||||
public sealed class NerLabelConfigTests
|
||||
{
|
||||
[TestCase("B-PER", true)]
|
||||
[TestCase("I-PER", true)]
|
||||
[TestCase("B-PERSON", true)]
|
||||
[TestCase("B-ORG", false)]
|
||||
public void English_IsPersonLabel_MatchesExpected(string label, bool expected)
|
||||
{
|
||||
NerLabelConfig.English.IsPersonLabel(label).Should().Be(expected);
|
||||
}
|
||||
|
||||
[TestCase("B-person-politician", true)]
|
||||
[TestCase("I-person-artist", true)]
|
||||
[TestCase("B-location", false)]
|
||||
[TestCase("O", false)]
|
||||
public void Tamil_IsPersonLabel_MatchesFineGrainedTags(string label, bool expected)
|
||||
{
|
||||
NerLabelConfig.Tamil.IsPersonLabel(label).Should().Be(expected);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
using FluentAssertions;
|
||||
using PiiRedaction.Core.Models;
|
||||
using PiiRedaction.Infrastructure.Onnx;
|
||||
using PiiRedaction.Tests.Shared;
|
||||
|
||||
namespace PiiRedaction.Infrastructure.Tests.Onnx;
|
||||
|
||||
[TestFixture]
|
||||
[Category("TamilNer")]
|
||||
public sealed class RealTamilNerModelRunnerTests : RealTamilNerModelFixture
|
||||
{
|
||||
[Test]
|
||||
public void IsModelAvailable_LoadsOnnxAndTokenizer()
|
||||
{
|
||||
Runner.IsModelAvailable.Should().BeTrue();
|
||||
}
|
||||
|
||||
[TestCase("வாடிக்கையாளர் ராஜேஷ் குமார் அழைத்தார்.", "ராஜேஷ்", "ராஜேஷ் குமார்")]
|
||||
public void PredictEntities_TamilScript_DetectsPersonEntity(
|
||||
string prompt,
|
||||
string expectedNamePart,
|
||||
string expectedValue)
|
||||
{
|
||||
var entities = Runner.PredictEntities(prompt);
|
||||
|
||||
entities.Should().Contain(entity =>
|
||||
entity.Type == PiiEntityType.Person &&
|
||||
entity.Source == PiiDetectionSource.Ner &&
|
||||
entity.Value.Contains(expectedNamePart, StringComparison.Ordinal) &&
|
||||
prompt.AsSpan(entity.StartIndex, entity.Length).ToString() == entity.Value);
|
||||
|
||||
entities.Should().Contain(entity => entity.Value == expectedValue);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void PredictEntities_TamilWithPhone_DetectsPersonAndLeavesPhoneToRegex()
|
||||
{
|
||||
const string prompt = "வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210";
|
||||
|
||||
var entities = Runner.PredictEntities(prompt);
|
||||
|
||||
entities.Should().Contain(entity =>
|
||||
entity.Type == PiiEntityType.Person &&
|
||||
entity.Source == PiiDetectionSource.Ner &&
|
||||
entity.Value.Contains("ராஜேஷ்", StringComparison.Ordinal));
|
||||
entities.Should().NotContain(entity => entity.Type == PiiEntityType.Phone);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void PredictEntities_CleanTamilQuestion_ReturnsNoEntities()
|
||||
{
|
||||
const string prompt = "பணத்தை திரும்பப் பெறுவது எப்படி?";
|
||||
|
||||
Runner.PredictEntities(prompt).Should().BeEmpty();
|
||||
}
|
||||
|
||||
[TestCase("Customer Senthil phone 9876543210", "Senthil")]
|
||||
public void PredictEntities_TanglishLatinScript_DoesNotInvokeTamilRunner(
|
||||
string prompt,
|
||||
string expectedNamePart)
|
||||
{
|
||||
// TamilOnnxNerRunner is script-scoped; Tanglish is handled by English routing in pipeline tests.
|
||||
// Direct Tamil runner on Latin-only text should not emit person spans.
|
||||
var entities = Runner.PredictEntities(prompt);
|
||||
|
||||
entities.Should().NotContain(entity =>
|
||||
entity.Type == PiiEntityType.Person &&
|
||||
entity.Value.Contains(expectedNamePart, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
using FluentAssertions;
|
||||
using PiiRedaction.Core.Detection;
|
||||
using PiiRedaction.Core.Models;
|
||||
using PiiRedaction.Infrastructure.Onnx;
|
||||
|
||||
namespace PiiRedaction.Infrastructure.Tests.Onnx;
|
||||
|
||||
[TestFixture]
|
||||
public sealed class RoutingOnnxNerModelRunnerTests
|
||||
{
|
||||
[Test]
|
||||
public void PredictEntities_LatinOnly_UsesEnglishRunnerOnly()
|
||||
{
|
||||
var english = new FakeLanguageNerRunner("Ravi Kumar");
|
||||
var tamil = new FakeLanguageNerRunner("தமிழ் பெயர்");
|
||||
var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
|
||||
|
||||
var entities = router.PredictEntities("Customer Ravi Kumar called.");
|
||||
|
||||
entities.Should().ContainSingle(entity => entity.Value == "Ravi Kumar");
|
||||
english.CallCount.Should().Be(1);
|
||||
tamil.CallCount.Should().Be(0);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void PredictEntities_TamilOnly_UsesTamilRunnerOnly()
|
||||
{
|
||||
var english = new FakeLanguageNerRunner("Ravi Kumar");
|
||||
var tamil = new FakeLanguageNerRunner("ராஜேஷ்");
|
||||
var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
|
||||
|
||||
var entities = router.PredictEntities("வாடிக்கையாளர் ராஜேஷ்");
|
||||
|
||||
entities.Should().ContainSingle(entity => entity.Value == "ராஜேஷ்");
|
||||
english.CallCount.Should().Be(0);
|
||||
tamil.CallCount.Should().Be(1);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void PredictEntities_Mixed_InvokesBothRunners()
|
||||
{
|
||||
var english = new FakeLanguageNerRunner("EnglishName");
|
||||
var tamil = new FakeLanguageNerRunner("தமிழ்");
|
||||
var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
|
||||
|
||||
router.PredictEntities("Rajesh மற்றும் Priya");
|
||||
|
||||
english.CallCount.Should().Be(1);
|
||||
tamil.CallCount.Should().Be(1);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void PredictEntities_NoLetters_InvokesNeither()
|
||||
{
|
||||
var english = new FakeLanguageNerRunner("ignored");
|
||||
var tamil = new FakeLanguageNerRunner("ignored");
|
||||
var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
|
||||
|
||||
router.PredictEntities("9876543210").Should().BeEmpty();
|
||||
english.CallCount.Should().Be(0);
|
||||
tamil.CallCount.Should().Be(0);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void PredictEntities_TamilDisabled_SkipsTamilRunnerForMixedText()
|
||||
{
|
||||
var english = new FakeLanguageNerRunner("EnglishName");
|
||||
var tamil = new FakeLanguageNerRunner("தமிழ்");
|
||||
var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: false);
|
||||
|
||||
router.PredictEntities("Rajesh மற்றும் Priya");
|
||||
|
||||
english.CallCount.Should().Be(1);
|
||||
tamil.CallCount.Should().Be(0);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void MergePersonSpans_PrefersLongerOverlappingSpan()
|
||||
{
|
||||
var entities = new[]
|
||||
{
|
||||
CreatePerson("Raj", 0, 3),
|
||||
CreatePerson("Rajesh", 0, 6)
|
||||
};
|
||||
|
||||
var merged = RoutingOnnxNerModelRunner.MergePersonSpans(entities);
|
||||
|
||||
merged.Should().ContainSingle(entity => entity.Value == "Rajesh");
|
||||
}
|
||||
|
||||
private static PiiEntity CreatePerson(string value, int start, int length) =>
|
||||
new(PiiEntityType.Person, value, start, length, PiiDetectionSource.Ner);
|
||||
|
||||
private sealed class FakeLanguageNerRunner : IOnnxNerModelRunner
|
||||
{
|
||||
private readonly string _personValue;
|
||||
|
||||
public FakeLanguageNerRunner(string personValue) => _personValue = personValue;
|
||||
|
||||
public int CallCount { get; private set; }
|
||||
|
||||
public bool IsModelAvailable => true;
|
||||
|
||||
public IReadOnlyList<PiiEntity> PredictEntities(string text)
|
||||
{
|
||||
CallCount++;
|
||||
return
|
||||
[
|
||||
new PiiEntity(
|
||||
PiiEntityType.Person,
|
||||
_personValue,
|
||||
0,
|
||||
_personValue.Length,
|
||||
PiiDetectionSource.Ner)
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using PiiRedaction.Infrastructure.Onnx;
|
||||
|
||||
namespace PiiRedaction.Infrastructure.Tests.Onnx;
|
||||
|
||||
[TestFixture]
|
||||
public sealed class TokenClassifierEncoderFactoryTests
|
||||
{
|
||||
[Test]
|
||||
public void Create_PrefersWordPieceWhenVocabExists()
|
||||
{
|
||||
var modelDirectory = ResolveTamilModelDirectory();
|
||||
if (!File.Exists(Path.Combine(modelDirectory, "vocab.txt")))
|
||||
{
|
||||
Assert.Ignore("Tamil vocab.txt not found.");
|
||||
}
|
||||
|
||||
var encoder = TokenClassifierEncoderFactory.Create(
|
||||
modelDirectory,
|
||||
NullLogger.Instance);
|
||||
|
||||
encoder.Should().BeOfType<BertWordPieceEncoder>();
|
||||
encoder.IsAvailable.Should().BeTrue();
|
||||
}
|
||||
|
||||
private static string ResolveTamilModelDirectory()
|
||||
{
|
||||
var directory = new DirectoryInfo(AppContext.BaseDirectory);
|
||||
while (directory is not null)
|
||||
{
|
||||
var candidate = Path.Combine(directory.FullName, "models", "ta");
|
||||
if (Directory.Exists(candidate))
|
||||
{
|
||||
return candidate;
|
||||
}
|
||||
|
||||
directory = directory.Parent;
|
||||
}
|
||||
|
||||
return Path.Combine(Environment.CurrentDirectory, "models", "ta");
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,8 @@
|
||||
<ItemGroup>
|
||||
<Compile Include="..\TestSupport.Shared\RealNerModelPaths.cs" Link="TestSupport.Shared\RealNerModelPaths.cs" />
|
||||
<Compile Include="..\TestSupport.Shared\RealNerModelFixture.cs" Link="TestSupport.Shared\RealNerModelFixture.cs" />
|
||||
<Compile Include="..\TestSupport.Shared\RealTamilNerModelPaths.cs" Link="TestSupport.Shared\RealTamilNerModelPaths.cs" />
|
||||
<Compile Include="..\TestSupport.Shared\RealTamilNerModelFixture.cs" Link="TestSupport.Shared\RealTamilNerModelFixture.cs" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
Reference in New Issue
Block a user