Add Tamil NER routing and WPF test harness for POC validation.
Introduce dual-script ONNX NER routing (English/Tamil/mixed), Tamil console samples and integration tests, model download scripts, and a resizable WPF MVVM harness with click-to-load prompts, batch validation, and runtime-adjustable detection panels.
This commit is contained in:
@@ -8,6 +8,7 @@ namespace PiiRedaction.Core.Tests.Integration;
|
||||
public sealed class GoldenPromptTests
|
||||
{
|
||||
[TestCaseSource(typeof(PromptScenarioCatalog), nameof(PromptScenarioCatalog.AllScenarios))]
|
||||
[TestCaseSource(typeof(TamilPromptScenarioCatalog), nameof(TamilPromptScenarioCatalog.AllScenarios))]
|
||||
public void Sanitize_PromptScenario_ProducesExpectedOutput(PromptScenario scenario)
|
||||
{
|
||||
var sanitizer = ProductionPipelineFactory.CreateForScenario(scenario);
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
using FluentAssertions;
|
||||
using PiiRedaction.Core.Abstractions;
|
||||
using PiiRedaction.Core.Models;
|
||||
using PiiRedaction.Core.Tests.TestSupport;
|
||||
using PiiRedaction.Tests.Shared;
|
||||
|
||||
namespace PiiRedaction.Core.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end pipeline proof using routed English + Tamil ONNX NER models.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
[Category("TamilNer")]
|
||||
public sealed class RealTamilPipelineTests : RealRoutingNerModelFixture
|
||||
{
|
||||
private IPromptSanitizer _sanitizer = null!;
|
||||
|
||||
[OneTimeSetUp]
|
||||
public void OneTimeSetUpPipeline()
|
||||
{
|
||||
_sanitizer = ProductionPipelineFactory.CreateWithRoutingRealModels(EnglishRunner, TamilRunner);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Sanitize_TamilCustomerNameOnly_RedactsPerson()
|
||||
{
|
||||
const string prompt =
|
||||
"வாடிக்கையாளர் ராஜேஷ் குமார் சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்.";
|
||||
|
||||
var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
|
||||
|
||||
result.SanitizedPrompt.Should().Contain("<PERSON_1>");
|
||||
result.SanitizedPrompt.Should().NotContain("ராஜேஷ்");
|
||||
result.DetectedEntities.Should().Contain(entity =>
|
||||
entity.Type == PiiEntityType.Person && entity.Source == PiiDetectionSource.Ner);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Sanitize_TamilWithPhonePan_RedactsPersonPhoneAndPan()
|
||||
{
|
||||
const string prompt = "வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210 PAN ABCDE1234F.";
|
||||
|
||||
var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
|
||||
|
||||
result.SanitizedPrompt.Should().Contain("<PERSON_1>");
|
||||
result.SanitizedPrompt.Should().Contain("<PHONE_1>");
|
||||
result.SanitizedPrompt.Should().Contain("<PAN_1>");
|
||||
result.SanitizedPrompt.Should().NotContainAny("ராஜேஷ்", "9876543210", "ABCDE1234F");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Sanitize_TanglishCustomer_RedactsPersonAndPhone()
|
||||
{
|
||||
const string prompt = "Customer Senthil phone 9876543210 reported a failed UPI transfer.";
|
||||
|
||||
var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
|
||||
|
||||
result.SanitizedPrompt.Should().Contain("<PERSON_1>");
|
||||
result.SanitizedPrompt.Should().Contain("<PHONE_1>");
|
||||
result.SanitizedPrompt.Should().NotContainAny("Senthil", "9876543210");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Sanitize_MixedTamilEnglish_RedactsPersonAndPhone()
|
||||
{
|
||||
const string prompt = "வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge.";
|
||||
|
||||
var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
|
||||
|
||||
result.SanitizedPrompt.Should().Contain("<PERSON_1>");
|
||||
result.SanitizedPrompt.Should().Contain("<PHONE_1>");
|
||||
result.SanitizedPrompt.Should().NotContainAny("Ravi Kumar", "9876543210");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Sanitize_TamilFullFinancial_RedactsAllPiiTypes()
|
||||
{
|
||||
const string prompt =
|
||||
"வாடிக்கையாளர் ராஜேஷ் குமார் மின்னஞ்சல் ravi.kumar@gmail.com தொலைபேசி 9876543210 LoanNumber LN-456789 PAN ABCDE1234F. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்.";
|
||||
|
||||
var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
|
||||
|
||||
result.SanitizedPrompt.Should().Contain("<PERSON_1>");
|
||||
result.SanitizedPrompt.Should().Contain("<EMAIL_1>");
|
||||
result.SanitizedPrompt.Should().Contain("<PHONE_1>");
|
||||
result.SanitizedPrompt.Should().Contain("<LOAN_NUMBER_1>");
|
||||
result.SanitizedPrompt.Should().Contain("<PAN_1>");
|
||||
result.SanitizedPrompt.Should().NotContainAny(
|
||||
"ராஜேஷ்",
|
||||
"ravi.kumar@gmail.com",
|
||||
"9876543210",
|
||||
"LN-456789",
|
||||
"ABCDE1234F");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Sanitize_CleanTamilQuestion_PassesThroughWithoutPersonRedaction()
|
||||
{
|
||||
const string prompt = "பணத்தை திரும்பப் பெறுவது எப்படி?";
|
||||
|
||||
var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
|
||||
|
||||
result.SanitizedPrompt.Should().Be(prompt);
|
||||
result.DetectedEntities.Should().BeEmpty();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user