Bilal-Nazer-Ali/llm-pii-poc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Surface NER routing provenance (ModelOrigin, NerModelsInvoked) through the pipeline and WPF harness so English/Tamil routing is observable during POC validation. Consolidate docs into solution-guide and add NER logs, topic filtering, and batch UI fixes in the test harness.

Browse Source
This commit is contained in:
Bilal Nazer Ali
2026-07-08 12:18:27 +05:30
parent cf8f5a7232
commit 81220c6ade
51 changed files with 1530 additions and 1463 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
tests/PiiRedaction.Infrastructure.Tests/Onnx/RoutingOnnxNerModelRunnerTests.cs
View File

@@ -11,13 +11,15 @@ public sealed class RoutingOnnxNerModelRunnerTests
[Test]
public void PredictEntities_LatinOnly_UsesEnglishRunnerOnly()
{
var english = new FakeLanguageNerRunner("Ravi Kumar");
var tamil = new FakeLanguageNerRunner("தமிழ் பெயர்");
var english = new FakeLanguageNerRunner("Ravi Kumar", NerModelOrigin.English);
var tamil = new FakeLanguageNerRunner("தமிழ் பெயர்", NerModelOrigin.Tamil);
var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
var entities = router.PredictEntities("Customer Ravi Kumar called.");
var result = router.PredictEntities("Customer Ravi Kumar called.");
entities.Should().ContainSingle(entity => entity.Value == "Ravi Kumar");
result.Entities.Should().ContainSingle(entity =>
entity.Value == "Ravi Kumar" && entity.ModelOrigin == NerModelOrigin.English);
result.InvokedModels.Should().Equal(NerModelOrigin.English);
english.CallCount.Should().Be(1);
tamil.CallCount.Should().Be(0);
}
@@ -25,38 +27,46 @@ public sealed class RoutingOnnxNerModelRunnerTests
[Test]
public void PredictEntities_TamilOnly_UsesTamilRunnerOnly()
{
var english = new FakeLanguageNerRunner("Ravi Kumar");
var tamil = new FakeLanguageNerRunner("ராஜேஷ்");
var english = new FakeLanguageNerRunner("Ravi Kumar", NerModelOrigin.English);
var tamil = new FakeLanguageNerRunner("ராஜேஷ்", NerModelOrigin.Tamil);
var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
var entities = router.PredictEntities("வாடிக்கையாளர் ராஜேஷ்");
var result = router.PredictEntities("வாடிக்கையாளர் ராஜேஷ்");
entities.Should().ContainSingle(entity => entity.Value == "ராஜேஷ்");
result.Entities.Should().ContainSingle(entity =>
entity.Value == "ராஜேஷ்" && entity.ModelOrigin == NerModelOrigin.Tamil);
result.InvokedModels.Should().Equal(NerModelOrigin.Tamil);
english.CallCount.Should().Be(0);
tamil.CallCount.Should().Be(1);
}
[Test]
public void PredictEntities_Mixed_InvokesBothRunners()
public void PredictEntities_Mixed_InvokesBothRunnersAndTagsOrigins()
{
var english = new FakeLanguageNerRunner("EnglishName");
var tamil = new FakeLanguageNerRunner("தமிழ்");
var english = new FakeLanguageNerRunner("Priya", NerModelOrigin.English);
var tamil = new FakeLanguageNerRunner("மற்றும்", NerModelOrigin.Tamil);
var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
router.PredictEntities("Rajesh மற்றும் Priya");
var result = router.PredictEntities("Rajesh மற்றும் Priya");
english.CallCount.Should().Be(1);
tamil.CallCount.Should().Be(1);
result.InvokedModels.Should().Equal(NerModelOrigin.English, NerModelOrigin.Tamil);
result.Entities.Should().Contain(entity => entity.ModelOrigin == NerModelOrigin.English);
result.Entities.Should().Contain(entity => entity.ModelOrigin == NerModelOrigin.Tamil);
}
[Test]
public void PredictEntities_NoLetters_InvokesNeither()
{
var english = new FakeLanguageNerRunner("ignored");
var tamil = new FakeLanguageNerRunner("ignored");
var english = new FakeLanguageNerRunner("ignored", NerModelOrigin.English);
var tamil = new FakeLanguageNerRunner("ignored", NerModelOrigin.Tamil);
var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
router.PredictEntities("9876543210").Should().BeEmpty();
var result = router.PredictEntities("9876543210");
result.Entities.Should().BeEmpty();
result.InvokedModels.Should().BeEmpty();
english.CallCount.Should().Be(0);
tamil.CallCount.Should().Be(0);
}
@@ -64,14 +74,15 @@ public sealed class RoutingOnnxNerModelRunnerTests
[Test]
public void PredictEntities_TamilDisabled_SkipsTamilRunnerForMixedText()
{
var english = new FakeLanguageNerRunner("EnglishName");
var tamil = new FakeLanguageNerRunner("தமிழ்");
var english = new FakeLanguageNerRunner("EnglishName", NerModelOrigin.English);
var tamil = new FakeLanguageNerRunner("தமிழ்", NerModelOrigin.Tamil);
var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: false);
router.PredictEntities("Rajesh மற்றும் Priya");
var result = router.PredictEntities("Rajesh மற்றும் Priya");
english.CallCount.Should().Be(1);
tamil.CallCount.Should().Be(0);
result.InvokedModels.Should().Equal(NerModelOrigin.English);
}
[Test]
@@ -79,40 +90,54 @@ public sealed class RoutingOnnxNerModelRunnerTests
{
var entities = new[]
{
CreatePerson("Raj", 0, 3),
CreatePerson("Rajesh", 0, 6)
CreatePerson("Raj", 0, 3, NerModelOrigin.English),
CreatePerson("Rajesh", 0, 6, NerModelOrigin.Tamil)
};
var merged = RoutingOnnxNerModelRunner.MergePersonSpans(entities);
merged.Should().ContainSingle(entity => entity.Value == "Rajesh");
merged.Should().ContainSingle(entity =>
entity.Value == "Rajesh" && entity.ModelOrigin == NerModelOrigin.Tamil);
}
private static PiiEntity CreatePerson(string value, int start, int length) =>
new(PiiEntityType.Person, value, start, length, PiiDetectionSource.Ner);
private static PiiEntity CreatePerson(string value, int start, int length, NerModelOrigin origin) =>
new(PiiEntityType.Person, value, start, length, PiiDetectionSource.Ner, ModelOrigin: origin);
private sealed class FakeLanguageNerRunner : IOnnxNerModelRunner
{
private readonly string _personValue;
private readonly NerModelOrigin _origin;
public FakeLanguageNerRunner(string personValue) => _personValue = personValue;
public FakeLanguageNerRunner(string personValue, NerModelOrigin origin)
{
_personValue = personValue;
_origin = origin;
}
public int CallCount { get; private set; }
public bool IsModelAvailable => true;
public IReadOnlyList<PiiEntity> PredictEntities(string text)
public NerPredictionResult PredictEntities(string text)
{
CallCount++;
return
var start = text.IndexOf(_personValue, StringComparison.Ordinal);
if (start < 0)
{
start = 0;
}
return new NerPredictionResult(
[
new PiiEntity(
PiiEntityType.Person,
_personValue,
0,
start,
_personValue.Length,
PiiDetectionSource.Ner)
];
PiiDetectionSource.Ner,
ModelOrigin: _origin)
],
[_origin]);
}
}
}