diff --git a/.gitignore b/.gitignore
index 7d13b63..55f9311 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,7 +10,11 @@ models/*.onnx
models/vocab.txt
models/ner-labels.txt
models/*.json
+models/en/*
+models/ta/*
!models/.gitkeep
+!models/en/.gitkeep
+!models/ta/.gitkeep
## IDE
.idea/
diff --git a/PiiRedaction.slnx b/PiiRedaction.slnx
index 1c9bace..eb3f0db 100644
--- a/PiiRedaction.slnx
+++ b/PiiRedaction.slnx
@@ -3,6 +3,7 @@
+
diff --git a/README.md b/README.md
index affcdfd..481fbd1 100644
--- a/README.md
+++ b/README.md
@@ -15,6 +15,10 @@ Financial and customer-service prompts often contain regulated data (names, gove
For solution design, data-flow diagrams, trust boundaries, and project responsibilities, see **[docs/architecture.md](docs/architecture.md)**.
+For English and Tamil ONNX NER model IDs, assets, routing, and reproduction steps, see **[docs/ner-models.md](docs/ner-models.md)**.
+
+**Planned:** Tamil / Tanglish person-name support via dual ONNX NER routing — see **[docs/tamil-tanglish-ner-plan.md](docs/tamil-tanglish-ner-plan.md)**.
+
## Why Three Detection Strategies?
| Strategy | Used For | Rationale |
@@ -31,15 +35,17 @@ The placeholder map (`` → original value) is kept **in-process** for
```
src/
-├── PiiRedaction.ConsoleApp/ # Presentation: input/output, DI bootstrap
-├── PiiRedaction.Core/ # Business logic: detection, redaction, models
-└── PiiRedaction.Infrastructure/ # Technical adapters: ONNX Runtime, mock LLM
-models/ # Optional ONNX model files (gitignored)
+├── PiiRedaction.ConsoleApp/ # Console demo: input/output, DI bootstrap
+├── PiiRedaction.TestHarness.Wpf/ # WPF MVVM test harness for manual POC validation
+├── PiiRedaction.Core/ # Business logic: detection, redaction, models
+└── PiiRedaction.Infrastructure/ # Technical adapters: ONNX Runtime, mock LLM
+models/ # Optional ONNX model files (gitignored)
```
| Project | Responsibility |
|---------|----------------|
| `PiiRedaction.ConsoleApp` | Read prompt, call sanitizer, display results, call LLM service |
+| `PiiRedaction.TestHarness.Wpf` | Desktop test harness: preset prompts, redact UI, batch validation |
| `PiiRedaction.Core` | PII detection abstractions, redaction, sanitization orchestration |
| `PiiRedaction.Infrastructure` | ONNX model runner, `IChatClient` mock implementation |
@@ -63,7 +69,7 @@ dotnet build
dotnet run --project src/PiiRedaction.ConsoleApp
```
-By default the console app runs **11 curated sample prompts** covering NER/person names, regex identifiers, domain IDs, combined scenarios, and a clean no-PII ticket.
+By default the console app runs **16 curated sample prompts** covering English and Tamil/Tanglish/mixed person names, regex identifiers, domain IDs, combined scenarios, and a clean no-PII ticket. No flags are required for Tamil samples — they run in the default batch alongside English.
List available samples:
@@ -78,6 +84,27 @@ dotnet run --project src/PiiRedaction.ConsoleApp -- --sample 2
dotnet run --project src/PiiRedaction.ConsoleApp -- --name MrTitlePerson
```
+### WPF Test Harness
+
+A desktop **MVVM** application for interactive POC validation with English and Tamil prompts. Requires **Windows** (`net10.0-windows`).
+
+**Prerequisites:** English and Tamil ONNX models downloaded (see [ONNX Model Setup](#onnx-model-setup)).
+
+```bash
+dotnet run --project src/PiiRedaction.TestHarness.Wpf
+```
+
+**Workflow:**
+
+1. **Select a test prompt** from the left panel (grouped by language: English, Tamil, Mixed, Tanglish) or type your own prompt in the input box.
+2. **Click a test prompt** in the left panel to load it into the input box (previous results are cleared automatically).
+3. Click **Redact** to run the full detection pipeline. The status bar shows model availability, script composition (LatinOnly / TamilOnly / Mixed), and elapsed time.
+4. Review **Sanitized Output**, detected entities, and the placeholder map in the right panel. A leak warning appears if any detected value remains in the sanitized text.
+5. Optionally click **Send Mock LLM** to send only the sanitized prompt to the mock LLM.
+6. Click **Run All** to execute all **22 curated scenarios** (16 console samples + 6 harness-only edge cases) and view pass/fail results in the batch panel.
+
+The harness uses the same DI registrations and `IPromptSanitizer` pipeline as the console app, with thin application services (`IRedactionAppService`, `ITestPromptCatalog`, `IScriptAnalysisService`, `IModelStatusService`) following SOLID principles.
+
Interactive mode (enter your own prompt):
```bash
@@ -100,7 +127,12 @@ Samples are defined in [`SamplePromptCatalog.cs`](src/PiiRedaction.ConsoleApp/Sa
| 7 | PersonWithEmailNoPhone | NER + Regex | `Customer Arjun Mehta` + email |
| 8 | AllRegexTypes | Regex | email, phone, PAN, Aadhaar, card |
| 9 | AllDomainIds | Domain | LN, CID, ACC |
-| 10 | NoPiiCleanTicket | Negative | no redaction |
+| 10 | TamilCustomerNameOnly | NER (Tamil) | `வாடிக்கையாளர் ராஜேஷ் குமார்` |
+| 11 | TamilWithPhonePan | NER (Tamil) + Regex | Tamil person + phone + PAN |
+| 12 | TanglishCustomer | NER (English/Tanglish) | `Customer Senthil` + phone |
+| 13 | MixedTamilEnglish | NER (Mixed) | `வாடிக்கையாளர் Ravi Kumar` + phone |
+| 14 | TamilFullFinancial | NER (Tamil) + Regex + Domain | Tamil canonical demo |
+| 15 | NoPiiCleanTicket | Negative | no redaction |
Person names are detected via **ONNX NER** using `dslim/bert-base-NER` (or a compatible token-classification export). A real model is **required** for person-name detection; there is no regex or heuristic fallback.
@@ -147,36 +179,41 @@ Person-name detection requires a token-classification ONNX model and companion t
| File | Purpose |
|------|---------|
-| `models/ner-model.onnx` | Exported NER model |
-| `models/vocab.txt` | BERT WordPiece vocabulary |
-| `models/ner-labels.txt` | One BIO label per line (`O`, `B-PER`, `I-PER`, etc.) |
+| `models/en/ner-model.onnx` | English BERT NER model (or legacy `models/ner-model.onnx`) |
+| `models/en/vocab.txt` | BERT WordPiece vocabulary |
+| `models/en/ner-labels.txt` | One BIO label per line (`O`, `B-PER`, `I-PER`, etc.) |
+| `models/ta/model.onnx` | Tamil IndicBERT NER model |
+| `models/ta/sentencepiece.bpe.model` | SentencePiece tokenizer for Tamil model |
+| `models/ta/ner-labels.txt` | Fine-grained Tamil NER labels |
-### Download script
+### Download scripts
From the repository root:
```powershell
.\scripts\download-ner-model.ps1
+.\scripts\download-tamil-ner-model.ps1
```
Or with Python directly:
```bash
python scripts/download-ner-model.py
+python scripts/download-tamil-ner-model.py
```
-The script exports [`dslim/bert-base-NER`](https://huggingface.co/dslim/bert-base-NER) via Hugging Face Optimum when Python is available. Otherwise it downloads the pre-exported ONNX assets from Hugging Face directly.
+The English script exports [`dslim/bert-base-NER`](https://huggingface.co/dslim/bert-base-NER) via Hugging Face Optimum when Python is available. The Tamil script exports [`prachuryyaIITG/SampurNER_Tamil_IndicBERTv2`](https://huggingface.co/prachuryyaIITG/SampurNER_Tamil_IndicBERTv2). Otherwise each script downloads pre-exported ONNX assets from Hugging Face directly.
+
+Set `EnableTamilNer` to `false` in `appsettings.json` to revert to English-only routing.
### Inference pipeline
-`OnnxNerModelRunner` performs the full pipeline:
+`RoutingOnnxNerModelRunner` classifies script composition and delegates to:
-- BERT WordPiece tokenization (`Microsoft.ML.Tokenizers`)
-- ONNX Runtime inference (`input_ids`, `attention_mask`, optional `token_type_ids`)
-- BIO label decoding (`B-PER` / `I-PER` → `PiiEntityType.Person`)
-- Character-span alignment back to the source text
+- **`EnglishOnnxNerRunner`** — BERT WordPiece tokenization for Latin script and Tanglish
+- **`TamilOnnxNerRunner`** — SentencePiece tokenization for Tamil script (U+0B80–U+0BFF)
-When the model or tokenizer files are missing, person detection returns no results.
+Both runners share `OnnxTokenClassifierRunner` for ONNX Runtime inference and BIO label decoding. Overlapping person spans from mixed-script prompts are merged (longer span wins).
## Swapping Mock LLM for Azure OpenAI
@@ -251,24 +288,29 @@ The solution includes an **NUnit** test suite across two projects:
dotnet test
dotnet test --filter "FullyQualifiedName~GoldenPromptTests"
dotnet test --filter "Category=RealModel"
+dotnet test --filter "Category=TamilNer"
dotnet test --logger "console;verbosity=detailed"
```
-Fast CI runs without the ONNX model: fake-based tests always execute; tests marked **`Category=RealModel`** are skipped when `models/ner-model.onnx` is absent. Download the model first:
+Fast CI runs without the ONNX model: fake-based tests always execute; tests marked **`Category=RealModel`** or **`Category=TamilNer`** are skipped when the corresponding ONNX models are absent. Download models first:
```powershell
.\scripts\download-ner-model.ps1
+.\scripts\download-tamil-ner-model.ps1
```
### Test architecture
-- **`PromptScenarioCatalog`** — five focused end-to-end scenarios (canonical demo, multi-regex, duplicate people, overlap stress, no-PII negative)
-- **`ProductionPipelineFactory`** — builds the same Domain → Regex → OnnxNer composite stack as production DI; `CreateWithRealModel(runner)` wires a real `OnnxNerModelRunner`
+- **`PromptScenarioCatalog`** — five focused end-to-end English scenarios (canonical demo, multi-regex, duplicate people, overlap stress, no-PII negative)
+- **`TamilPromptScenarioCatalog`** — five Tamil/Tanglish/mixed golden scenarios (fake NER for person spans)
+- **`ProductionPipelineFactory`** — builds the same Domain → Regex → OnnxNer composite stack as production DI; `CreateWithRealModel(runner)` wires a real runner; `CreateWithRoutingRealModels` wires English + Tamil routing
- **`FakeOnnxNerModelRunner`** — unit-test double for NER; golden tests inject person spans per scenario
- **`GoldenPromptTests`** — end-to-end sanitization proof across the catalog (fake NER)
- **`RealNerModelFixture`** — shared fixture that loads `models/ner-model.onnx` once per class; skips when model missing
- **`RealNerModelRunnerTests`** — direct ONNX inference with span accuracy checks
-- **`RealNerPipelineTests`** — full pipeline with real NER (canonical, multi-person, clean-ticket negative)
+- **`RealNerPipelineTests`** — full pipeline with real English NER (canonical, multi-person, clean-ticket negative)
+- **`RealTamilPipelineTests`** — full pipeline with routed English + Tamil NER (`Category=TamilNer`)
+- **`RealTamilNerModelRunnerTests`** — direct Tamil ONNX inference (`Category=TamilNer`)
- **`OnnxNerModelRunnerTests`** — unit tests for missing/invalid model paths (no download required)
- **`CompositePiiDetectorTests`** — overlap merge and source-priority rules
- **`LlmBoundaryTests`** — verifies raw PII never appears in outbound LLM messages
diff --git a/docs/architecture.md b/docs/architecture.md
index 3118415..b33ea2f 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -10,7 +10,7 @@ The POC validates a compliance-oriented pattern suitable for financial and custo
## Canonical Example
-The console application ships with a **sample catalog** (11 prompts). The canonical demo is sample `FullFinancialWithCustomer`. The table below shows the exact strings produced by the production pipeline when the ONNX NER model is loaded (run `scripts/download-ner-model.ps1` first).
+The console application ships with a **sample catalog** (16 prompts). The canonical demo is sample `FullFinancialWithCustomer`. Tamil script, Tanglish, and mixed-script samples run in the **default** `dotnet run` batch (no `--interactive` required). The table below shows the exact strings produced by the production pipeline when the ONNX NER models are loaded (run `scripts/download-ner-model.ps1` and `scripts/download-tamil-ner-model.ps1` first).
| Stage | Value |
|-------|-------|
@@ -38,7 +38,7 @@ Running `dotnet run --project src/PiiRedaction.ConsoleApp` executes all samples
### NER / person-name samples
-These prompts exercise `OnnxNerPiiDetector` and `OnnxNerModelRunner`. Person names require the ONNX model (`models/ner-model.onnx` plus `vocab.txt` and `ner-labels.txt`). Without the model, person spans are not detected.
+These prompts exercise `OnnxNerPiiDetector` and `RoutingOnnxNerModelRunner`. Person names require ONNX models (`models/en/` for English, `models/ta/` for Tamil script). Without models, person spans are not detected. Legacy `models/ner-model.onnx` is still supported for English.
| Sample | Input (excerpt) | Detected person | Sanitized (excerpt) |
|--------|-----------------|-----------------|---------------------|
@@ -50,6 +50,18 @@ These prompts exercise `OnnxNerPiiDetector` and `OnnxNerModelRunner`. Person nam
| **PersonWithDomainIds** | Customer Meera Iyer holds CID-7070… | Meera Iyer | Customer `` holds ``… |
| **PersonWithEmailNoPhone** | Customer Arjun Mehta wrote from arjun.mehta@company.in… | Arjun Mehta | Customer `` wrote from ``… |
+### Tamil / Tanglish / mixed samples
+
+These prompts exercise `RoutingOnnxNerModelRunner` script routing. Tamil script uses `models/ta/`; Latin Tanglish uses `models/en/`. Mixed prompts may invoke both models.
+
+| Sample | Input (excerpt) | Detected person | Sanitized (excerpt) |
+|--------|-----------------|-----------------|---------------------|
+| **TamilCustomerNameOnly** | வாடிக்கையாளர் ராஜேஷ் குமார் சேமிப்பு… | ராஜேஷ் குமார் | வாடிக்கையாளர் `` சேமிப்பு… |
+| **TamilWithPhonePan** | வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210 PAN… | ராஜேஷ் குமார் | `` … `` … `` |
+| **TanglishCustomer** | Customer Senthil phone 9876543210… | Senthil | Customer `` phone ``… |
+| **MixedTamilEnglish** | வாடிக்கையாளர் Ravi Kumar phone 9876543210… | Ravi Kumar | வாடிக்கையாளர் `` phone ``… |
+| **TamilFullFinancial** | வாடிக்கையாளர் ராஜேஷ் குமார் மின்னஞ்சல் ravi.kumar@gmail.com… | ராஜேஷ் குமார் | Tamil canonical — all placeholder types |
+
### Other sample categories
| Category | Sample | Purpose |
@@ -84,7 +96,9 @@ flowchart TB
end
subgraph infra [PiiRedaction.Infrastructure]
- onnxRunner["OnnxNerModelRunner"]
+ onnxRunner["RoutingOnnxNerModelRunner"]
+ enRunner["EnglishOnnxNerRunner"]
+ taRunner["TamilOnnxNerRunner"]
mockLlm["MockLlmPromptService"]
mockChat["MockChatClient"]
end
@@ -110,6 +124,8 @@ flowchart TB
di -.-> mockLlm
```
+`RoutingOnnxNerModelRunner` selects English and/or Tamil ONNX models based on script composition in the prompt. See [Dual-Model NER Routing (Tamil + English)](#dual-model-ner-routing-tamil--english) for the routing decision tree.
+
---
## Detection to Redaction Detail
@@ -123,7 +139,7 @@ flowchart LR
subgraph detectPhase [Detection Phase]
domainDet["DomainRulePiiDetector"]
regexDet["RegexPiiDetector"]
- onnxDet["OnnxNerPiiDetector"]
+ onnxDet["OnnxNerPiiDetector
(RoutingOnnxNerModelRunner)"]
composite["CompositePiiDetector"]
merge["Overlap merge and source priority"]
entityList["PiiEntity list"]
@@ -157,14 +173,144 @@ flowchart LR
2. On overlapping spans, the first registered detector wins.
3. Tie-breaking uses source priority: Domain (3) > Regex (2) > NER (1).
+The ONNX NER detector delegates to `RoutingOnnxNerModelRunner`, which routes inference to English and/or Tamil models by script composition. See [Dual-Model NER Routing (Tamil + English)](#dual-model-ner-routing-tamil--english).
+
**Placeholder assignment** (applied by `PlaceholderPiiRedactor`):
+
- Format: `<{TYPE}_{n}>` (e.g. ``, ``).
- Duplicate values of the same type reuse the same placeholder.
- Replacement proceeds from highest `StartIndex` to lowest to avoid index drift.
---
+## Dual-Model NER Routing (Tamil + English)
+
+Person-name detection uses two ONNX token-classifier models: **English** (`models/en/`, BERT WordPiece) and **Tamil** (`models/ta/`, SentencePiece or WordPiece). `OnnxNerPiiDetector` calls `RoutingOnnxNerModelRunner`, which classifies prompt script via `ScriptRouter` and dispatches to `EnglishOnnxNerRunner` and/or `TamilOnnxNerRunner`. Both runners share `OnnxTokenClassifierRunner` for BIO decoding; only **PERSON** spans are emitted.
+
+The diagram below expands the detection and NER branches summarized in [High-Level Data Flow](#high-level-data-flow) and [Detection to Redaction Detail](#detection-to-redaction-detail).
+
+### End-to-end pipeline (with NER branch)
+
+```mermaid
+flowchart TB
+ subgraph Entry["Console entry"]
+ A["Program.cs
Host + AddPiiRedactionServices()"]
+ B["PromptDemoRunner.RunAsync()"]
+ A --> B
+ end
+
+ B --> C["SanitizationRequest(OriginalPrompt)"]
+ C --> D["PromptSanitizer.Sanitize()"]
+
+ subgraph Detect["CompositePiiDetector.Detect() — registration order"]
+ direction TB
+ E1["DomainRulePiiDetector
LOAN_NUMBER, CUSTOMER_ID, ACCOUNT_NUMBER"]
+ E2["RegexPiiDetector
EMAIL, PHONE, AADHAAR, PAN, CREDIT_CARD"]
+ E3["OnnxNerPiiDetector
PERSON (via IOnnxNerModelRunner)"]
+ E1 --> MERGE
+ E2 --> MERGE
+ E3 --> MERGE
+ MERGE["Merge overlapping spans
sort: StartIndex ↑, Length ↓, Source priority ↓
(Domain=3, Regex=2, Ner=1)
first candidate wins on overlap"]
+ end
+
+ D --> Detect
+ MERGE --> F["IReadOnlyList<PiiEntity>"]
+
+ F --> G["PlaceholderPiiRedactor.Redact()
replace spans right-to-left
dedupe by Type|Value → <TYPE_n>"]
+ G --> H["SanitizationResult
SanitizedPrompt, DetectedEntities, PlaceholderMap"]
+
+ H --> I["MockLlmPromptService.SendPromptAsync(SanitizedPrompt)"]
+ I --> J["Mock LLM response
(sanitized text only)"]
+
+ subgraph NerBranch["OnnxNerPiiDetector branch"]
+ E3 --> N1{"RoutingOnnxNerModelRunner
.IsModelAvailable?"}
+ N1 -->|no| N2["return []"]
+ N1 -->|yes| N3["RoutingOnnxNerModelRunner
.PredictEntities()"]
+ end
+```
+
+### RoutingOnnxNerModelRunner decision tree
+
+`ScriptRouter.GetComposition` scans each character once. Tamil letters (U+0B80–U+0BFF) and ASCII Latin letters (`char.IsAsciiLetter`) determine the route. When both scripts appear, classification is **Mixed** (early exit).
+
+```mermaid
+flowchart TB
+ IN["text"] --> SR["ScriptRouter.GetComposition(text)
scan each char"]
+
+ SR --> C1{"LatinOnly?"}
+ SR --> C2{"TamilOnly?"}
+ SR --> C3{"Mixed?"}
+ SR --> C4{"NoLetters?"}
+
+ C1 -->|yes| EN1{"EnglishOnnxNerRunner
.IsModelAvailable?"}
+ EN1 -->|yes| EN_RUN["EnglishOnnxNerRunner.PredictEntities(text)"]
+ EN1 -->|no| SKIP1["skip English"]
+ EN_RUN --> ACC
+ SKIP1 --> ACC
+
+ C2 -->|yes| TA_GATE{"EnableTamilNer
&& TamilOnnxNerRunner
.IsModelAvailable?"}
+ TA_GATE -->|yes| TA_RUN["TamilOnnxNerRunner.PredictEntities(text)"]
+ TA_GATE -->|no| SKIP2["skip Tamil"]
+ TA_RUN --> ACC
+ SKIP2 --> ACC
+
+ C3 -->|yes| EN2{"English available?"}
+ EN2 -->|yes| EN_MIX["EnglishOnnxNerRunner.PredictEntities(text)"]
+ EN2 -->|no| SKIP3["skip English"]
+ EN_MIX --> TA_GATE2{"EnableTamilNer
&& Tamil available?"}
+ SKIP3 --> TA_GATE2
+ TA_GATE2 -->|yes| TA_MIX["TamilOnnxNerRunner.PredictEntities(text)"]
+ TA_GATE2 -->|no| SKIP4["skip Tamil"]
+ TA_MIX --> ACC
+ SKIP4 --> ACC
+
+ C4 -->|yes| EMPTY["no NER inference"]
+ EMPTY --> OUT_EMPTY["return []"]
+
+ subgraph EN_Pipeline["EnglishOnnxNerRunner"]
+ EN_RUN --> EN_ENC["BertWordPieceEncoder
(model dir vocab.txt)"]
+ EN_ENC --> EN_OCR["OnnxTokenClassifierRunner
NerLabelConfig.English
B-PER / I-PER / B-PERSON / I-PERSON"]
+ end
+
+ subgraph TA_Pipeline["TamilOnnxNerRunner"]
+ TA_RUN --> TA_ENC["TokenClassifierEncoderFactory.Create()
vocab.txt → BertWordPieceEncoder
else SentencePiece (*.bpe.model, spiece.model, tokenizer.model)"]
+ TA_ENC --> TA_OCR["OnnxTokenClassifierRunner
NerLabelConfig.Tamil
label contains 'person' (case-insensitive)"]
+ end
+
+ subgraph SharedInference["OnnxTokenClassifierRunner (shared)"]
+ ENC["Encode(text, max 128 tokens)"]
+ ONNX["ONNX InferenceSession.Run
input_ids + attention_mask [+ token_type_ids]"]
+ ARGMAX["Per-token argmax over logits"]
+ BIO["BIO decode → PiiEntityType.Person
PiiDetectionSource.Ner"]
+ ENC --> ONNX --> ARGMAX --> BIO
+ end
+
+ EN_OCR --> SharedInference
+ TA_OCR --> SharedInference
+ BIO --> ACC["accumulate entities"]
+
+ ACC --> MERGE["MergePersonSpans()
sort: Length ↓, StartIndex ↑
drop overlapping spans
(longer span wins)"]
+ MERGE --> OUT["return merged PERSON entities"]
+```
+
+### Routing rules
+
+| Rule | Source | Behavior |
+|------|--------|----------|
+| **Script classification** | `ScriptRouter.GetComposition` | Single pass over characters. Tamil letter = U+0B80–U+0BFF. Latin letter = `char.IsAsciiLetter`. Both seen → `Mixed` (early exit). Neither → `NoLetters`. Tamil only → `TamilOnly`. Latin only → `LatinOnly`. |
+| **LatinOnly** | `RoutingOnnxNerModelRunner` | Run **English only** if `englishRunner.IsModelAvailable`. |
+| **TamilOnly** | `RoutingOnnxNerModelRunner` | Run **Tamil only** if `EnableTamilNer` (default `true` in `PiiRedactionOptions`) **and** `tamilRunner.IsModelAvailable`. |
+| **Mixed** | `RoutingOnnxNerModelRunner` | Run **both** models independently on the **full text** (English if available; Tamil if `EnableTamilNer` and available). |
+| **NoLetters** | `RoutingOnnxNerModelRunner` | No NER inference; returns `[]` from routing (before merge). |
+| **Model availability gate** | `OnnxNerPiiDetector` | If `RoutingOnnxNerModelRunner.IsModelAvailable` is false, NER detector returns `[]` (English OR Tamil available when Tamil enabled). |
+| **Post-route merge** | `MergePersonSpans` | After EN/TA results are concatenated, overlapping PERSON spans are deduped; **longer span wins**, then ordered by `StartIndex`. |
+| **Composite merge** | `CompositePiiDetector` | Domain → Regex → NER all run. Overlaps resolved globally: earlier registration order + longer span + higher source priority (Domain > Regex > Ner). |
+| **Encoder choice** | `EnglishOnnxNerRunner` vs `TamilOnnxNerRunner` | English always uses `BertWordPieceEncoder`. Tamil uses factory: `vocab.txt` → WordPiece; else first SentencePiece file found; fallback WordPiece with warning. |
+| **NER output scope** | `OnnxTokenClassifierRunner` | Only **PERSON** entities decoded from BIO tags; max sequence length **128** tokens. |
+
+---
+
## Runtime Sequence
```mermaid
@@ -255,7 +401,7 @@ In the POC, `MockChatClient` simulates the external provider without network I/O
|---------|-------|----------------|
| `PiiRedaction.ConsoleApp` | Presentation | Application entry point; reads prompt (sample or interactive); bootstraps `IHost` and DI via `AddPiiRedactionServices`; orchestrates sanitization and LLM invocation; renders audit output (detected entities, sanitized text, placeholder map). |
| `PiiRedaction.Core` | Domain / Application | Defines abstractions (`IPiiDetector`, `IPiiRedactor`, `IPromptSanitizer`, `ILlmPromptService`); implements detection strategies (`RegexPiiDetector`, `DomainRulePiiDetector`, `OnnxNerPiiDetector`, `CompositePiiDetector`); implements `PlaceholderPiiRedactor` and `PromptSanitizer`; owns domain models (`PiiEntity`, `SanitizationResult`, `RedactionResult`) and configuration (`PiiRedactionOptions`). Has no dependency on ONNX Runtime or LLM SDKs. |
-| `PiiRedaction.Infrastructure` | Infrastructure | Implements technical adapters: `OnnxNerModelRunner` (ONNX Runtime inference), `MockChatClient` and `MockLlmPromptService` (`Microsoft.Extensions.AI`); depends on Core abstractions and is swappable without changing domain logic. |
+| `PiiRedaction.Infrastructure` | Infrastructure | Implements technical adapters: `RoutingOnnxNerModelRunner`, `EnglishOnnxNerRunner`, `TamilOnnxNerRunner` (ONNX Runtime inference), `MockChatClient` and `MockLlmPromptService` (`Microsoft.Extensions.AI`); depends on Core abstractions and is swappable without changing domain logic. |
| `tests/PiiRedaction.Core.Tests` | Test | Unit and integration tests for detectors, redactor, sanitizer, overlap rules, golden prompt scenarios (`PromptScenarioCatalog`), and LLM boundary assertions. |
| `tests/PiiRedaction.Infrastructure.Tests` | Test | Tests for mock LLM behavior and ONNX runner load semantics. |
@@ -272,7 +418,7 @@ In the POC, `MockChatClient` simulates the external provider without network I/O
| `IPromptSanitizer` | Core | `PromptSanitizer` | Unlikely to change; orchestrates detect + redact |
| `ILlmPromptService` | Core | `MockLlmPromptService` | Production adapter with telemetry, retry, policy |
| `IChatClient` | Microsoft.Extensions.AI | `MockChatClient` | Azure OpenAI, OpenAI, or other provider SDK |
-| `IOnnxNerModelRunner` | Core | `OnnxNerModelRunner` | BERT WordPiece tokenization, ONNX inference, BIO label decoding |
+| `IOnnxNerModelRunner` | Core | `RoutingOnnxNerModelRunner` | Script-based routing to English (BERT WordPiece) and Tamil (SentencePiece) ONNX models |
---
@@ -282,14 +428,18 @@ Runtime behavior is controlled via `appsettings.json` under the `PiiRedaction` s
| Setting | Effect |
|---------|--------|
-| `OnnxModelPath` | Path to ONNX NER model (`models/ner-model.onnx` by default). Companion files `vocab.txt` and `ner-labels.txt` must live in the same directory. |
+| `OnnxModelPath` | Legacy English model path (`models/ner-model.onnx`). Used as fallback when `models/en/` is absent. |
+| `EnglishOnnxModelPath` | Primary English ONNX model (`models/en/ner-model.onnx`). |
+| `TamilOnnxModelPath` | Tamil ONNX model (`models/ta/model.onnx`). |
+| `EnableTamilNer` | When `false`, routing uses English model only. Default `true`. |
-Download the model assets with `scripts/download-ner-model.ps1` (exports `dslim/bert-base-NER`).
+Download model assets with `scripts/download-ner-model.ps1` and `scripts/download-tamil-ner-model.ps1`.
---
## Related Documentation
+- [NER models](ner-models.md) — English/Tamil ONNX model IDs, assets, labels, and integration reference
- [README](../README.md) — build, run, configuration, and testing instructions
- [ServiceCollectionExtensions.cs](../src/PiiRedaction.ConsoleApp/DependencyInjection/ServiceCollectionExtensions.cs) — DI registration and detector ordering
- [PromptScenarioCatalog.cs](../tests/PiiRedaction.Core.Tests/TestSupport/PromptScenarioCatalog.cs) — focused golden pipeline scenarios including the canonical example
diff --git a/docs/ner-models.md b/docs/ner-models.md
new file mode 100644
index 0000000..d861d69
--- /dev/null
+++ b/docs/ner-models.md
@@ -0,0 +1,548 @@
+# NER Models for PII Redaction
+
+This document describes the **Named Entity Recognition (NER)** ONNX models at the core of the PII Redaction POC. Person-name detection is the only NER responsibility in this solution; structured identifiers (email, phone, PAN, domain IDs) are handled by regex and domain-rule detectors.
+
+For pipeline placement, trust boundaries, and routing diagrams, see [architecture.md](architecture.md). For the Tamil/Tanglish implementation plan and success metrics, see [tamil-tanglish-ner-plan.md](tamil-tanglish-ner-plan.md).
+
+---
+
+## 1. Executive Summary
+
+The POC uses **dual-model ONNX NER routing** to redact **person-name PII** before prompts reach an LLM:
+
+| Script in prompt | Model invoked | Typical use case |
+|------------------|---------------|------------------|
+| Latin only (`LatinOnly`) | English (`dslim/bert-base-NER`) | English names, Indian names in Roman script, **Tanglish** |
+| Tamil only (`TamilOnly`) | Tamil (`prachuryyaIITG/SampurNER_Tamil_IndicBERTv2`) | Tamil-script customer names |
+| Mixed (`Mixed`) | **Both** models on the full text; spans merged | Code-mixed Indian CS prompts |
+| No letters (`NoLetters`) | Neither | Digits-only or symbol-only text |
+
+`RoutingOnnxNerModelRunner` classifies script via `ScriptRouter`, delegates to `EnglishOnnxNerRunner` and/or `TamilOnnxNerRunner`, and merges overlapping PERSON spans (longer span wins). Only **PERSON** entities are emitted to the redaction pipeline; all other NER labels are discarded.
+
+---
+
+## 2. English Model
+
+### Hugging Face model ID
+
+**[`dslim/bert-base-NER`](https://huggingface.co/dslim/bert-base-NER)**
+
+### Architecture
+
+| Property | Value |
+|----------|-------|
+| Base | BERT-base (uncased), ~110M parameters |
+| Task | Token classification (NER) |
+| Tokenizer | **WordPiece** via `vocab.txt` (`BertWordPieceEncoder`) |
+| Runtime | ONNX via Microsoft.ML.OnnxRuntime |
+| Export | Hugging Face Optimum (`ORTModelForTokenClassification`) or pre-exported ONNX from HF |
+
+### Labels (BIO)
+
+The English model uses standard CoNLL-style BIO tags. The POC maps only **person** labels to `PiiEntityType.Person`:
+
+| Label | Mapped to PERSON |
+|-------|------------------|
+| `O` | No |
+| `B-PER`, `I-PER` | Yes |
+| `B-PERSON`, `I-PERSON` | Yes |
+| `B-ORG`, `I-ORG`, `B-LOC`, `I-LOC`, `B-MISC`, `I-MISC` | No |
+
+Full label list is written to `ner-labels.txt` at download time from the model `config.json` `id2label` map (typically 9 labels for this model).
+
+Label matching is implemented in `NerLabelConfig.English`:
+
+```19:22:src/PiiRedaction.Infrastructure/Onnx/NerLabelConfig.cs
+ private static bool IsEnglishPersonLabel(string label) =>
+ label is "B-PER" or "I-PER" or "B-PERSON" or "I-PERSON"
+ || (label.EndsWith("-PER", StringComparison.Ordinal) &&
+ (label.StartsWith("B-", StringComparison.Ordinal) || label.StartsWith("I-", StringComparison.Ordinal)));
+```
+
+### Asset paths
+
+| File | Primary path (`appsettings.json`) | Legacy fallback |
+|------|----------------------------------|-----------------|
+| ONNX model | `models/en/ner-model.onnx` | `models/ner-model.onnx` (`OnnxModelPath`) |
+| Vocabulary | `models/en/vocab.txt` | `models/vocab.txt` |
+| Labels | `models/en/ner-labels.txt` | `models/ner-labels.txt` |
+
+`EnglishOnnxNerRunner` resolves the model path with primary + legacy fallback:
+
+```15:22:src/PiiRedaction.Infrastructure/Onnx/EnglishOnnxNerRunner.cs
+ var modelPath = OnnxAssetPathResolver.ResolveModelPath(
+ options.Value.EnglishOnnxModelPath,
+ options.Value.OnnxModelPath);
+
+ var modelDirectory = Path.GetDirectoryName(modelPath) ?? Environment.CurrentDirectory;
+ var labels = OnnxAssetPathResolver.LoadLabels(modelDirectory);
+ var encoder = new BertWordPieceEncoder(modelDirectory, logger);
+ _runner = new OnnxTokenClassifierRunner(modelPath, encoder, NerLabelConfig.English, labels, logger);
+```
+
+> **Note:** `scripts/download-ner-model.ps1` writes assets to `models/` (repository root). For the configured primary path, copy or move them into `models/en/`, or rely on the `OnnxModelPath` fallback.
+
+### Download script
+
+```powershell
+.\scripts\download-ner-model.ps1
+```
+
+Or with Python directly:
+
+```bash
+python scripts/download-ner-model.py
+```
+
+**Behavior:**
+
+1. If Python + Optimum are available → exports `dslim/bert-base-NER` to ONNX under `models/`.
+2. Otherwise → downloads pre-exported ONNX from `https://huggingface.co/dslim/bert-base-NER/resolve/main/onnx/` (`model.onnx`, `vocab.txt`, `config.json` → `ner-labels.txt`).
+
+---
+
+## 3. Tamil Model
+
+### Hugging Face model ID
+
+**[`prachuryyaIITG/SampurNER_Tamil_IndicBERTv2`](https://huggingface.co/prachuryyaIITG/SampurNER_Tamil_IndicBERTv2)**
+
+(SampurNER Tamil IndicBERTv2 — fine-grained NER for Tamil script.)
+
+### Why SampurNER IndicBERTv2 vs MuRIL
+
+| Criterion | SampurNER Tamil IndicBERTv2 | MuRIL (fallback candidate) |
+|-----------|----------------------------|----------------------------|
+| Tamil NER training | Fine-grained SampurNER dataset (Tamil-specific labels) | General multilingual; NER requires separate fine-tune |
+| Model size | ~0.3B parameters (IndicBERTv2, ~278M base) | ~0.6B parameters |
+| POC fit | Lighter memory footprint; ONNX export path validated in this repo | Reserved for Phase 5 if Tamil recall is insufficient |
+| Indian financial context | Trained on Indian-language NER corpus; person subtypes map cleanly to PERSON | Heavier; eval-driven swap only |
+
+See [tamil-tanglish-ner-plan.md](tamil-tanglish-ner-plan.md) §3 for the original selection rationale.
+
+### Architecture
+
+| Property | Value |
+|----------|-------|
+| Base | IndicBERTv2 (AI4Bharat), ~0.3B parameters |
+| Task | Fine-grained token classification |
+| Tokenizer | **WordPiece** when `vocab.txt` is present (this repo's export path); SentencePiece fallback if `sentencepiece.bpe.model` / `spiece.model` exists |
+| Runtime | Same shared `OnnxTokenClassifierRunner` as English |
+
+### Tokenizer: WordPiece, not SentencePiece (in practice)
+
+The Hugging Face repo for this model does **not** ship a SentencePiece model file. The download scripts extract **WordPiece** assets from `tokenizer.json` → `vocab.txt`. `TokenClassifierEncoderFactory` prefers `vocab.txt`:
+
+```10:19:src/PiiRedaction.Infrastructure/Onnx/TokenClassifierEncoderFactory.cs
+ public static ITokenClassifierEncoder Create(string modelDirectory, ILogger logger)
+ {
+ var vocabPath = OnnxAssetPathResolver.ResolveAssetPath(Path.Combine(modelDirectory, "vocab.txt"));
+ if (File.Exists(vocabPath))
+ {
+ logger.LogInformation(
+ "Using WordPiece tokenizer (vocab.txt) from {ModelDirectory}.",
+ modelDirectory);
+ return new BertWordPieceEncoder(modelDirectory, logger);
+ }
+```
+
+The PowerShell Tamil download script emits an explicit warning when WordPiece assets are saved instead of SentencePiece.
+
+### Labels (fine-grained person tags)
+
+SampurNER uses fine-grained BIO tags (e.g. `B-person-politician`, `I-person-artist`, `B-location`, `O`). The POC treats **any label containing `person`** (case-insensitive) as a person span:
+
+```24:25:src/PiiRedaction.Infrastructure/Onnx/NerLabelConfig.cs
+ private static bool IsTamilPersonLabel(string label) =>
+ label.Contains("person", StringComparison.OrdinalIgnoreCase);
+```
+
+Unit tests lock this behavior:
+
+```20:27:tests/PiiRedaction.Infrastructure.Tests/Onnx/NerLabelConfigTests.cs
+ [TestCase("B-person-politician", true)]
+ [TestCase("I-person-artist", true)]
+ [TestCase("B-location", false)]
+ [TestCase("O", false)]
+ public void Tamil_IsPersonLabel_MatchesFineGrainedTags(string label, bool expected)
+ {
+ NerLabelConfig.Tamil.IsPersonLabel(label).Should().Be(expected);
+ }
+```
+
+### Asset paths
+
+| File | Path |
+|------|------|
+| ONNX model | `models/ta/model.onnx` |
+| Tokenizer | `models/ta/vocab.txt` (WordPiece, preferred) **or** `models/ta/sentencepiece.bpe.model` |
+| Labels | `models/ta/ner-labels.txt` |
+| Optional | `models/ta/tokenizer.json` (intermediate export artifact) |
+
+### Download script
+
+```powershell
+.\scripts\download-tamil-ner-model.ps1
+```
+
+Or with Python directly:
+
+```bash
+python scripts/download-tamil-ner-model.py
+```
+
+**Behavior:**
+
+1. Python + Optimum → full export to `models/ta/` including ONNX, labels, and tokenizer assets.
+2. PowerShell fallback → downloads `onnx/model.onnx` from Hugging Face when published; otherwise requires Python export (pre-exported ONNX may return 404).
+
+`TamilOnnxNerRunner` wiring:
+
+```15:19:src/PiiRedaction.Infrastructure/Onnx/TamilOnnxNerRunner.cs
+ var modelPath = OnnxAssetPathResolver.ResolveModelPath(options.Value.TamilOnnxModelPath);
+ var modelDirectory = Path.GetDirectoryName(modelPath) ?? Environment.CurrentDirectory;
+ var labels = OnnxAssetPathResolver.LoadLabels(modelDirectory);
+ var encoder = TokenClassifierEncoderFactory.Create(modelDirectory, logger);
+ _runner = new OnnxTokenClassifierRunner(modelPath, encoder, NerLabelConfig.Tamil, labels, logger);
+```
+
+---
+
+## 4. Why These Models
+
+Evidence-based rationale for this Indian financial POC:
+
+| Requirement | Decision |
+|-------------|----------|
+| **English + Indian Latin names** | `dslim/bert-base-NER` is industry-standard, pre-integrated, and handles many Indian names in Roman script (e.g. `Ravi Kumar`, `Anita Sharma`) |
+| **Tamil script names** | English BERT is out-of-vocabulary for Tamil letters (U+0B80–U+0BFF); a Tamil-trained NER model is required |
+| **Tanglish (Roman-script Tamil)** | Routed to the **English** model only (`ScriptComposition.LatinOnly`); no Tamil ONNX on Latin-only text |
+| **Code-mixed prompts** | Both models run on the full prompt; `MergePersonSpans` deduplicates overlaps |
+| **Deployable size** | English ~431 MB ONNX (FP32); Tamil IndicBERTv2 ~0.3B params — smaller than MuRIL ~0.6B |
+| **ONNX export path** | Both models export via Hugging Face Optimum; English has pre-exported ONNX on HF; Tamil may require local Python export |
+| **PERSON-only scope** | POC redacts person names via NER; org/location/misc labels are intentionally ignored to limit false positives |
+
+---
+
+## 5. How They Integrate
+
+### Configuration (`appsettings.json`)
+
+```1:8:src/PiiRedaction.ConsoleApp/appsettings.json
+{
+ "PiiRedaction": {
+ "OnnxModelPath": "models/ner-model.onnx",
+ "EnglishOnnxModelPath": "models/en/ner-model.onnx",
+ "TamilOnnxModelPath": "models/ta/model.onnx",
+ "EnableTamilNer": true
+ }
+}
+```
+
+Options type:
+
+```3:14:src/PiiRedaction.Core/Configuration/PiiRedactionOptions.cs
+public sealed class PiiRedactionOptions
+{
+ public const string SectionName = "PiiRedaction";
+
+ public string OnnxModelPath { get; set; } = "models/ner-model.onnx";
+
+ public string EnglishOnnxModelPath { get; set; } = "models/en/ner-model.onnx";
+
+ public string TamilOnnxModelPath { get; set; } = "models/ta/model.onnx";
+
+ public bool EnableTamilNer { get; set; } = true;
+}
+```
+
+Set `EnableTamilNer` to `false` for English-only routing.
+
+### Dependency injection
+
+```34:36:src/PiiRedaction.ConsoleApp/DependencyInjection/ServiceCollectionExtensions.cs
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+```
+
+`OnnxNerPiiDetector` consumes `IOnnxNerModelRunner` (the router) and returns `[]` when no model is available — **fail-open** for person detection.
+
+### Script routing (`ScriptRouter`)
+
+```11:41:src/PiiRedaction.Core/Detection/ScriptRouter.cs
+ public ScriptComposition GetComposition(string text)
+ {
+ ArgumentNullException.ThrowIfNull(text);
+
+ var hasLatin = false;
+ var hasTamil = false;
+
+ foreach (var character in text)
+ {
+ if (IsTamilLetter(character))
+ {
+ hasTamil = true;
+ }
+ else if (char.IsAsciiLetter(character))
+ {
+ hasLatin = true;
+ }
+
+ if (hasLatin && hasTamil)
+ {
+ return ScriptComposition.Mixed;
+ }
+ }
+ // ...
+ return hasTamil ? ScriptComposition.TamilOnly : ScriptComposition.LatinOnly;
+ }
+```
+
+### Routing runner (`RoutingOnnxNerModelRunner`)
+
+```39:79:src/PiiRedaction.Infrastructure/Onnx/RoutingOnnxNerModelRunner.cs
+ public IReadOnlyList PredictEntities(string text)
+ {
+ var composition = _scriptRouter.GetComposition(text);
+ var entities = new List();
+
+ switch (composition)
+ {
+ case ScriptComposition.LatinOnly:
+ if (_englishRunner.IsModelAvailable)
+ {
+ entities.AddRange(_englishRunner.PredictEntities(text));
+ }
+ break;
+ case ScriptComposition.TamilOnly:
+ if (_enableTamilNer && _tamilRunner.IsModelAvailable)
+ {
+ entities.AddRange(_tamilRunner.PredictEntities(text));
+ }
+ break;
+ case ScriptComposition.Mixed:
+ if (_englishRunner.IsModelAvailable)
+ {
+ entities.AddRange(_englishRunner.PredictEntities(text));
+ }
+ if (_enableTamilNer && _tamilRunner.IsModelAvailable)
+ {
+ entities.AddRange(_tamilRunner.PredictEntities(text));
+ }
+ break;
+ case ScriptComposition.NoLetters:
+ break;
+ }
+
+ return MergePersonSpans(entities);
+ }
+```
+
+### Shared inference (`OnnxTokenClassifierRunner`)
+
+Both runners share:
+
+- **Encode** → `input_ids`, `attention_mask`, optional `token_type_ids`
+- **Argmax** over per-token logits
+- **BIO decode** → `PiiEntityType.Person` with `PiiDetectionSource.Ner`
+- **Max sequence length: 128 tokens**
+
+```13:13:src/PiiRedaction.Infrastructure/Onnx/OnnxTokenClassifierRunner.cs
+ private const int MaxSequenceLength = 128;
+```
+
+### End-to-end flow
+
+```
+Prompt → CompositePiiDetector → OnnxNerPiiDetector
+ → RoutingOnnxNerModelRunner → ScriptRouter
+ → EnglishOnnxNerRunner / TamilOnnxNerRunner
+ → OnnxTokenClassifierRunner → PERSON entities
+ → PlaceholderPiiRedactor →
+```
+
+---
+
+## 6. Evidence from Codebase
+
+### Real-model tests (`Category=RealModel`)
+
+English direct inference — `RealNerModelRunnerTests`:
+
+```15:47:tests/PiiRedaction.Infrastructure.Tests/Onnx/RealNerModelRunnerTests.cs
+[Category("RealModel")]
+public sealed class RealNerModelRunnerTests : RealNerModelFixture
+{
+ [TestCase("Customer Ravi Kumar called about billing.", "Ravi", "Ravi Kumar")]
+ [TestCase("Mr. John Smith called about a duplicate debit.", "John", "John Smith")]
+ public void PredictEntities_DetectsPersonWithCorrectSpan(...)
+```
+
+English pipeline — `RealNerPipelineTests` (`Category=RealModel`): canonical `FullFinancialWithCustomer`, multi-person, clean-ticket negative.
+
+Fixture skips when model missing:
+
+```5:6:tests/TestSupport.Shared/RealNerModelPaths.cs
+ public const string ModelMissingMessage =
+ "ONNX model not found. Run scripts/download-ner-model.ps1 from the repository root.";
+```
+
+Resolves `models/en/ner-model.onnx` then `models/ner-model.onnx`.
+
+### Tamil tests (`Category=TamilNer`)
+
+Direct Tamil runner — `RealTamilNerModelRunnerTests`:
+
+```9:69:tests/PiiRedaction.Infrastructure.Tests/Onnx/RealTamilNerModelRunnerTests.cs
+[Category("TamilNer")]
+public sealed class RealTamilNerModelRunnerTests : RealTamilNerModelFixture
+{
+ [TestCase("வாடிக்கையாளர் ராஜேஷ் குமார் அழைத்தார்.", "ராஜேஷ்", "ராஜேஷ் குமார்")]
+ public void PredictEntities_TamilScript_DetectsPersonEntity(...)
+ // ...
+ [TestCase("Customer Senthil phone 9876543210", "Senthil")]
+ public void PredictEntities_TanglishLatinScript_DoesNotInvokeTamilRunner(...)
+```
+
+Routed pipeline — `RealTamilPipelineTests` covers Tamil-only, Tanglish (English path), mixed, full financial, and clean Tamil negative.
+
+### Console samples (`SamplePromptCatalog`)
+
+Tamil/Tanglish/mixed samples (indices 10–14):
+
+| Sample | Category | Input excerpt |
+|--------|----------|---------------|
+| `TamilCustomerNameOnly` | NER (Tamil) | `வாடிக்கையாளர் ராஜேஷ் குமார் …` |
+| `TamilWithPhonePan` | NER (Tamil) + Regex | Tamil person + phone + PAN |
+| `TanglishCustomer` | NER (English/Tanglish) | `Customer Senthil phone 9876543210 …` |
+| `MixedTamilEnglish` | NER (Mixed) | `வாடிக்கையாளர் Ravi Kumar phone …` |
+| `TamilFullFinancial` | NER (Tamil) + Regex + Domain | Tamil canonical demo |
+
+Run all samples (including Tamil) with no flags:
+
+```bash
+dotnet run --project src/PiiRedaction.ConsoleApp
+```
+
+Or a single Tamil sample:
+
+```bash
+dotnet run --project src/PiiRedaction.ConsoleApp -- --name TamilCustomerNameOnly
+```
+
+### Expected console output (English canonical)
+
+When models are loaded, person names appear as `[PERSON]` with source `Ner`:
+
+```
+Detected PII:
+ [PERSON ] Ravi Kumar (Ner)
+ [EMAIL ] ravi.kumar@gmail.com (Regex)
+ ...
+
+Sanitized Prompt:
+Customer with email ...
+```
+
+---
+
+## 7. Model Assets Table
+
+All model binaries are **gitignored**; only `.gitkeep` placeholders are committed.
+
+| Directory | File | Approx. size | Gitignored | Purpose |
+|-----------|------|--------------|------------|---------|
+| `models/` or `models/en/` | `ner-model.onnx` | ~431 MB | Yes | English BERT NER (FP32 ONNX from HF) |
+| `models/` or `models/en/` | `vocab.txt` | ~213 KB | Yes | WordPiece vocabulary |
+| `models/` or `models/en/` | `ner-labels.txt` | < 1 KB | Yes | BIO label index (one per line) |
+| `models/ta/` | `model.onnx` | ~1.2 GB (FP32 export, varies) | Yes | Tamil IndicBERT NER |
+| `models/ta/` | `vocab.txt` | varies | Yes | WordPiece vocab (preferred tokenizer) |
+| `models/ta/` | `tokenizer.json` | varies | Yes | HF tokenizer export (optional) |
+| `models/ta/` | `sentencepiece.bpe.model` | varies | Yes | SentencePiece (if present instead of vocab) |
+| `models/ta/` | `ner-labels.txt` | few KB | Yes | Fine-grained SampurNER labels |
+
+`.gitignore` entries:
+
+```
+models/*.onnx
+models/vocab.txt
+models/ner-labels.txt
+models/*.json
+models/en/*
+models/ta/*
+```
+
+English size reference: [docs/git-xenovex-setup.md](git-xenovex-setup.md) notes ~431 MB for the English ONNX file.
+
+---
+
+## 8. Limitations
+
+| Limitation | Detail |
+|------------|--------|
+| **Tanglish on English model only** | Roman-script Tanglish (`Customer Senthil`) is classified `LatinOnly` and handled by English BERT. Recall is best-effort and inconsistent for non-standard spellings. Tamil ONNX is **not** invoked on Latin-only text. |
+| **Fail-open if model missing** | `OnnxNerPiiDetector` and routing runners return `[]` when models are unavailable. Person names are **not** redacted; regex/domain layers still run. No regex fallback for names. |
+| **128 token limit** | `OnnxTokenClassifierRunner` truncates encoding at 128 tokens. Very long prompts may miss person names beyond the window. |
+| **PERSON-only NER scope** | Organization, location, and misc NER labels are ignored. Only person spans become ``. |
+| **Mixed-script merge** | When both models run, overlapping spans are deduped by length; shorter overlapping spans are dropped. |
+| **Tamil ONNX availability** | Pre-exported Tamil ONNX may not exist on Hugging Face; local Python export is often required. |
+| **No fail-closed mode** | Missing NER does not block sanitization or LLM calls (optional Phase 5 enhancement). |
+
+---
+
+## 9. How to Reproduce
+
+### Download models
+
+From the repository root:
+
+```powershell
+.\scripts\download-ner-model.ps1
+.\scripts\download-tamil-ner-model.ps1
+```
+
+If Tamil PowerShell download fails with a 404 on `onnx/model.onnx`, install Python 3.12+ and re-run:
+
+```powershell
+winget install Python.Python.3.12 --accept-package-agreements --accept-source-agreements
+.\scripts\download-tamil-ner-model.ps1 -Python "$env:LOCALAPPDATA\Programs\Python\Python312\python.exe"
+```
+
+Optional: copy English assets from `models/` to `models/en/` to match `EnglishOnnxModelPath`.
+
+### Verify with tests
+
+```bash
+dotnet build
+dotnet test
+dotnet test --filter "Category=RealModel"
+dotnet test --filter "Category=TamilNer"
+dotnet test --logger "console;verbosity=detailed"
+```
+
+Tests skip gracefully when the corresponding ONNX files are absent.
+
+### Verify with console
+
+```bash
+dotnet run --project src/PiiRedaction.ConsoleApp -- --name CustomerNameOnly
+dotnet run --project src/PiiRedaction.ConsoleApp -- --name TamilCustomerNameOnly
+dotnet run --project src/PiiRedaction.ConsoleApp -- --name TanglishCustomer
+```
+
+### Disable Tamil routing (English-only)
+
+Set in `appsettings.json`:
+
+```json
+"EnableTamilNer": false
+```
+
+---
+
+## Related Documentation
+
+- [README.md](../README.md) — build, run, and test overview
+- [architecture.md](architecture.md) — dual-model routing diagrams and trust boundary
+- [tamil-tanglish-ner-plan.md](tamil-tanglish-ner-plan.md) — implementation phases and Tanglish expectations
diff --git a/docs/tamil-tanglish-ner-plan.md b/docs/tamil-tanglish-ner-plan.md
new file mode 100644
index 0000000..088e5b6
--- /dev/null
+++ b/docs/tamil-tanglish-ner-plan.md
@@ -0,0 +1,307 @@
+# Tamil / Tanglish NER — Implementation Plan
+
+**Goal:** Raise language coverage from ~15% to production-viable for Tamil script and Tanglish (Roman-script Tamil-English) customer prompts, without changing the secure LLM boundary pattern.
+
+**Status:** Phase 1–3 implemented
+**Approach:** Dual-model ONNX NER routing (English + Tamil) + lightweight text normalization + optional Tanglish heuristics
+**Estimated effort:** 4–6 engineering days across 4 phases
+
+---
+
+## 1. Current State vs Gap
+
+| Capability | Today | Tamil script | Tanglish (Latin) |
+|------------|-------|--------------|------------------|
+| Phone, PAN, Aadhaar, email, domain IDs | Regex + domain rules | Works (ASCII digits) | Works |
+| Person names | `dslim/bert-base-NER` (English BERT) | **Fails** — out of vocabulary | **Partial** — inconsistent |
+| Script / language routing | None | N/A | N/A |
+| Tamil numerals (௦–௯) | Not normalized | **May miss** phone/Aadhaar | N/A |
+| Label-aware cues (`பெயர்`, `peru`, `enga peru`) | None | **Misses** contextual names | **Misses** |
+
+**Root cause:** Person detection is a single English-only ONNX model behind `IOnnxNerModelRunner` → `OnnxNerPiiDetector`. Regex/domain layers are already language-agnostic.
+
+---
+
+## 2. Target Architecture
+
+No change to the trust boundary: `PromptSanitizer` → `CompositePiiDetector` → `PlaceholderPiiRedactor` → sanitized text only to LLM.
+
+Only the **NER adapter** expands:
+
+```mermaid
+flowchart TD
+ text["Prompt text"]
+ router["ScriptRouter (Core)"]
+ routing["RoutingOnnxNerModelRunner"]
+ en["EnglishOnnxRunner\nBERT WordPiece"]
+ ta["TamilOnnxRunner\nIndicBERT SentencePiece"]
+ nerDet["OnnxNerPiiDetector"]
+ composite["CompositePiiDetector"]
+
+ text --> composite
+ text --> nerDet
+ nerDet --> routing
+ routing --> router
+ router -->|"LatinOnly / Mixed"| en
+ router -->|"TamilOnly / Mixed"| ta
+ en --> routing
+ ta --> routing
+```
+
+### Routing rules
+
+| `ScriptComposition` | Models invoked | Tanglish note |
+|---------------------|----------------|---------------|
+| `LatinOnly` | English NER only | Tanglish names in Roman script |
+| `TamilOnly` | Tamil NER only | Tamil script names |
+| `Mixed` | **Both**, merge person spans | Common in Indian CS prompts |
+| `NoLetters` | Neither (or English fallback off) | Digits-only prompts |
+
+**Merge inside `RoutingOnnxNerModelRunner`:** dedupe overlapping person spans (prefer longer span; tie-break Tamil vs English by start index order).
+
+---
+
+## 3. Model Selection
+
+| Role | Model | Rationale |
+|------|-------|-----------|
+| English / Tanglish (Latin) | **Keep** `dslim/bert-base-NER` | Already integrated; works for many Indian names in Latin script |
+| Tamil script | **`prachuryyaIITG/SampurNER_Tamil_IndicBERTv2`** | Tamil NER; lighter than MuRIL (~0.6B) |
+| Fallback (optional Phase 5) | MuRIL Tamil NER | Only if IndicBERT recall is insufficient on eval set |
+
+### Asset layout
+
+```
+models/
+ en/
+ ner-model.onnx # or model.onnx (BERT export)
+ vocab.txt
+ ner-labels.txt
+ ta/
+ model.onnx
+ sentencepiece.bpe.model # or tokenizer.json from HF export
+ ner-labels.txt
+ ner-model.onnx # legacy path — keep for backward compatibility
+```
+
+### Label mapping (Tamil fine-grained NER)
+
+SampurNER uses fine-grained tags (e.g. `B-person-politician`, `I-person-artist`). Map **any label containing `person`** (case-insensitive) → `PiiEntityType.Person`.
+
+English labels remain: `B-PER`, `I-PER`, `B-PERSON`, `I-PERSON`.
+
+---
+
+## 4. Implementation Phases
+
+### Phase 1 — Generic ONNX token classifier (1–2 days)
+
+**Objective:** Refactor `OnnxNerModelRunner` so BERT and SentencePiece are pluggable.
+
+| Action | Location |
+|--------|----------|
+| Add `ITokenClassifierEncoder` + `EncodedSequence` | `Infrastructure/Onnx/` |
+| `BertWordPieceEncoder` — extract from current runner | Infrastructure |
+| `SentencePieceEncoder` — IndicBERT tokenizer | Infrastructure |
+| `OnnxTokenClassifierRunner` — shared inference + BIO decode | Infrastructure |
+| `NerLabelConfig` — English vs Tamil person label predicates | Infrastructure |
+| `OnnxAssetPathResolver` — resolve model dir from repo root | Infrastructure |
+| Thin wrappers: `EnglishOnnxNerRunner`, `TamilOnnxNerRunner` | Infrastructure |
+
+**Backward compat:** If `models/en/` missing, fall back to `OnnxModelPath` (`models/ner-model.onnx`).
+
+**No behavior change** until Phase 3 wiring — existing tests must pass.
+
+---
+
+### Phase 2 — Tamil model download + config (0.5–1 day)
+
+| Action | Details |
+|--------|---------|
+| `scripts/download-tamil-ner-model.ps1` + `.py` | Mirror `download-ner-model.ps1`; export via `optimum-cli export onnx --task token-classification` |
+| Optional: `scripts/download-all-ner-models.ps1` | Calls English + Tamil scripts |
+| Extend `PiiRedactionOptions` | `EnglishOnnxModelPath`, `TamilOnnxModelPath`, `EnableTamilNer` (default `true`) |
+| Update `appsettings.json` | New paths under `PiiRedaction` section |
+| `.gitignore` | `models/ta/*`, `models/en/*` (same as today for onnx/vocab) |
+
+---
+
+### Phase 3 — Script routing + DI (0.5–1 day)
+
+| Action | Location |
+|--------|----------|
+| `ScriptRouter` + `ScriptComposition` enum | `Core/Detection/` |
+| `RoutingOnnxNerModelRunner` implements `IOnnxNerModelRunner` | Infrastructure |
+| DI registration | `ServiceCollectionExtensions.cs` |
+
+```csharp
+services.AddSingleton();
+services.AddSingleton();
+services.AddSingleton();
+```
+
+`OnnxNerPiiDetector` and `CompositePiiDetector` **unchanged**.
+
+---
+
+### Phase 4 — Tests, samples, docs (1 day)
+
+#### Unit tests
+
+| Test class | Coverage |
+|------------|----------|
+| `ScriptRouterTests` | Tamil-only, Latin-only, mixed, no-letters, boundary chars U+0B80/U+0BFF |
+| `RoutingOnnxNerModelRunnerTests` | Fake EN/TA runners; mixed script merges both |
+| `NerLabelConfigTests` | Tamil fine-grained person labels map correctly |
+
+#### Real-model tests (`Category=RealModel` or `Category=TamilNer`)
+
+| Scenario | Input example | Assert |
+|----------|---------------|--------|
+| Tamil name | `வாடிக்கையாளர் ராஜேஷ் தொலைபேசி 9876543210` | ``, phone redacted |
+| Tanglish name | `Customer Senthil phone 9876543210` | person + phone (best-effort) |
+| Mixed | `Rajesh மற்றும் Priya` | both persons redacted |
+| Clean Tamil | `பணத்தை திரும்பப் பெறுவது எப்படி?` | no false positives |
+| Canonical English | existing golden tests | no regression |
+
+Skip gracefully when `models/ta/model.onnx` missing (mirror `RealNerModelFixture`).
+
+#### Console samples
+
+Add to `SamplePromptCatalog.cs`:
+
+- `TamilCustomerName` — Tamil script person
+- `TanglishCustomerName` — `enga peru Rajesh` / `Customer Senthil`
+- `MixedTamilEnglish` — code-mixed prompt
+
+#### Docs
+
+- Update `README.md` ONNX setup (dual models)
+- Update `docs/architecture.md` NER section
+- Link this plan from README
+
+---
+
+### Phase 5 — Optional enhancements (post-MVP)
+
+| Enhancement | Benefit | Effort |
+|-------------|---------|--------|
+| **Unicode digit normalization** pre-pass | Tamil numerals → ASCII for regex | 0.5 day |
+| **Label-based regex** (`பெயர்`, `peru`, `peyar`, `enga peru`) | Tanglish recall without ML | 0.5 day |
+| **Tamil name gazetteer** `IPiiDetector` | High precision for top names | 1 day |
+| **Fail-closed policy** when NER unavailable | Compliance option | 0.5 day |
+| MuRIL model swap | Higher Tamil recall | eval-driven |
+
+---
+
+## 5. Tanglish — Realistic Expectations
+
+| Input type | Primary handler | Expected recall |
+|------------|-----------------|-----------------|
+| Tamil script names | Tamil ONNX NER | High (with eval tuning) |
+| Standard Latin Indian names (`Ravi Kumar`) | English ONNX NER | High (already works) |
+| Tanglish spellings (`Senthil`, `senthil`, `Centhil`) | English NER + optional gazetteer | Medium |
+| Code-mixed (`Rajesh oda account ACC-123456`) | English NER + domain regex | Medium–high for IDs; name variable |
+
+**MVP target:** Tamil script person names reliably redacted; Tanglish improved but not 100% without Phase 5 heuristics.
+
+---
+
+## 6. Success Metrics
+
+Before marking language gap closed, run an **eval set of 20–30 real prompts** (anonymized production samples):
+
+| Metric | MVP target |
+|--------|------------|
+| Tamil script person-name recall | ≥ 85% |
+| Tanglish person-name recall | ≥ 70% (with English model + optional heuristics) |
+| False positive rate (clean prompts) | ≤ 5% |
+| Structured PII (phone/PAN/domain) in Tamil prompts | ≥ 95% (regex layer) |
+| Regression on English canonical demo | 100% (existing golden tests) |
+
+---
+
+## 7. Files to Create / Modify (checklist)
+
+### New files
+
+- `src/PiiRedaction.Core/Detection/ScriptRouter.cs`
+- `src/PiiRedaction.Core/Detection/ScriptComposition.cs`
+- `src/PiiRedaction.Infrastructure/Onnx/OnnxTokenClassifierRunner.cs`
+- `src/PiiRedaction.Infrastructure/Onnx/BertWordPieceEncoder.cs`
+- `src/PiiRedaction.Infrastructure/Onnx/SentencePieceEncoder.cs`
+- `src/PiiRedaction.Infrastructure/Onnx/ITokenClassifierEncoder.cs`
+- `src/PiiRedaction.Infrastructure/Onnx/NerLabelConfig.cs`
+- `src/PiiRedaction.Infrastructure/Onnx/OnnxAssetPathResolver.cs`
+- `src/PiiRedaction.Infrastructure/Onnx/EnglishOnnxNerRunner.cs`
+- `src/PiiRedaction.Infrastructure/Onnx/TamilOnnxNerRunner.cs`
+- `src/PiiRedaction.Infrastructure/Onnx/RoutingOnnxNerModelRunner.cs`
+- `scripts/download-tamil-ner-model.ps1` / `.py`
+- `tests/.../ScriptRouterTests.cs`
+- `tests/.../RoutingOnnxNerModelRunnerTests.cs`
+- `tests/.../RealTamilNerPipelineTests.cs`
+- `tests/TestSupport.Shared/RealTamilModelFixture.cs`
+
+### Modified files
+
+- `PiiRedactionOptions.cs` — dual model paths
+- `ServiceCollectionExtensions.cs` — routing DI
+- `appsettings.json` — config
+- `SamplePromptCatalog.cs` — Tamil/Tanglish demos
+- `ProductionPipelineFactory.cs` — `CreateWithRoutingRealModel()` for tests
+- `RealNerModelFixture.cs` / paths — support EN + TA
+- `README.md`, `docs/architecture.md`
+- `.gitignore` — `models/en/`, `models/ta/`
+
+### Unchanged (by design)
+
+- `PromptSanitizer`, `PlaceholderPiiRedactor`, `CompositePiiDetector`
+- `RegexPiiDetector`, `DomainRulePiiDetector`
+- `MockLlmPromptService` / LLM boundary
+
+---
+
+## 8. Rollout & Risk
+
+| Risk | Mitigation |
+|------|------------|
+| Tamil model export fails on Windows | PowerShell fallback downloads pre-exported ONNX from Hugging Face |
+| Larger memory (two models) | Lazy-load Tamil runner only when `EnableTamilNer` and Tamil script detected |
+| Fine-grained label mismatch | Load labels from `ner-labels.txt`; unit test label config |
+| Tanglish disappointment | Set stakeholder expectation in README; Phase 5 heuristics |
+| CI without models | Fast tests use fakes; `Category=TamilNer` skips like `RealModel` |
+
+**Feature flag:** `EnableTamilNer=false` reverts to English-only behavior for gradual rollout.
+
+---
+
+## 9. Command Reference (after implementation)
+
+```powershell
+# Download both models
+.\scripts\download-ner-model.ps1
+.\scripts\download-tamil-ner-model.ps1
+
+# Run Tamil-focused console sample
+dotnet run --project src/PiiRedaction.ConsoleApp -- --name TamilCustomerName
+
+# Tests
+dotnet test
+dotnet test --filter "Category=TamilNer"
+dotnet test --filter "Category=RealModel"
+```
+
+---
+
+## 10. Approval Checklist
+
+- [ ] Stakeholder sign-off on dual-model approach (vs single multilingual model)
+- [ ] Tamil eval prompt set collected (20–30 samples)
+- [ ] Xenovex CI policy: models downloaded in pipeline or tests skip
+- [x] Phase 1–3 implementation PR
+- [ ] Phase 4 eval metrics met
+- [ ] Optional Phase 5 for Tanglish heuristics if recall < 70%
+
+---
+
+**Next step:** Implement Phase 1 in a feature branch (`feature/tamil-tanglish-ner`), open PR to `main` on `xts.xenovex.com/Bilal-Nazer-Ali/llm-pii-poc`.
diff --git a/models/en/.gitkeep b/models/en/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/models/ta/.gitkeep b/models/ta/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/scripts/download-tamil-ner-model.ps1 b/scripts/download-tamil-ner-model.ps1
new file mode 100644
index 0000000..c45e8c1
--- /dev/null
+++ b/scripts/download-tamil-ner-model.ps1
@@ -0,0 +1,229 @@
+# Downloads prachuryyaIITG/SampurNER_Tamil_IndicBERTv2 ONNX assets to models/ta/ for the PII Redaction POC.
+param(
+ [string]$Python = "python"
+)
+
+$ErrorActionPreference = "Stop"
+$repoRoot = Split-Path -Parent $PSScriptRoot
+$modelsDir = Join-Path (Join-Path $repoRoot "models") "ta"
+$scriptPath = Join-Path $PSScriptRoot "download-tamil-ner-model.py"
+$modelId = "prachuryyaIITG/SampurNER_Tamil_IndicBERTv2"
+$baseUrl = "https://huggingface.co/$modelId/resolve/main"
+
+function Ensure-ModelsDirectory {
+ New-Item -ItemType Directory -Force -Path $modelsDir | Out-Null
+}
+
+function Download-HuggingFaceAsset {
+ param(
+ [string]$RelativePath,
+ [string]$Destination
+ )
+
+ $url = "$baseUrl/$RelativePath"
+ Write-Host "Downloading $url"
+ Invoke-WebRequest -Uri $url -OutFile $Destination -UseBasicParsing
+}
+
+function Export-LabelsFromConfig {
+ param([string]$ConfigPath, [string]$LabelsPath)
+
+ $config = Get-Content $ConfigPath -Raw | ConvertFrom-Json
+ $labelMap = @{}
+ foreach ($property in $config.id2label.PSObject.Properties) {
+ $labelMap[[int]$property.Name] = [string]$property.Value
+ }
+
+ $labels = for ($index = 0; $index -lt $labelMap.Count; $index++) {
+ $labelMap[$index]
+ }
+
+ $labels | Set-Content -Path $LabelsPath -Encoding utf8
+}
+
+function Resolve-PythonExecutable {
+ param([string]$Preferred = "python")
+
+ if ($Preferred -ne "python") {
+ if ((Get-Command $Preferred -ErrorAction SilentlyContinue) -and
+ -not ((Get-Command $Preferred).Source -like "*WindowsApps*")) {
+ return $Preferred
+ }
+ }
+
+ $candidates = @(
+ (Join-Path $env:LOCALAPPDATA "Programs\Python\Python313\python.exe"),
+ (Join-Path $env:LOCALAPPDATA "Programs\Python\Python312\python.exe"),
+ (Join-Path $env:LOCALAPPDATA "Programs\Python\Python311\python.exe"),
+ "C:\Program Files\Python312\python.exe",
+ "C:\Program Files\Python313\python.exe"
+ )
+
+ foreach ($candidate in $candidates) {
+ if (Test-Path $candidate) {
+ return $candidate
+ }
+ }
+
+ $pythonCommand = Get-Command python -ErrorAction SilentlyContinue
+ if ($pythonCommand -and $pythonCommand.Source -notlike "*WindowsApps*") {
+ return $pythonCommand.Source
+ }
+
+ return $null
+}
+
+function Export-VocabFromTokenizerJson {
+ param([string]$TokenizerJsonPath, [string]$VocabPath)
+
+ $tokenizer = Get-Content $TokenizerJsonPath -Raw | ConvertFrom-Json
+ $vocab = $tokenizer.model.vocab
+ if (-not $vocab) {
+ return $false
+ }
+
+ $orderedTokens = $vocab.PSObject.Properties |
+ Sort-Object { [int]$_.Value } |
+ ForEach-Object { $_.Name }
+
+ $orderedTokens | Set-Content -Path $VocabPath -Encoding utf8
+ return $true
+}
+
+function Copy-TokenizerAssets {
+ param([string]$SourceDir)
+
+ foreach ($name in @("sentencepiece.bpe.model", "spiece.model", "tokenizer.model")) {
+ $source = Join-Path $SourceDir $name
+ if (Test-Path $source) {
+ $destination = Join-Path $modelsDir $name
+ Copy-Item $source $destination -Force
+ return @($destination)
+ }
+ }
+
+ $tokenizerJsonSource = Join-Path $SourceDir "tokenizer.json"
+ if (Test-Path $tokenizerJsonSource) {
+ $tokenizerJsonDestination = Join-Path $modelsDir "tokenizer.json"
+ Copy-Item $tokenizerJsonSource $tokenizerJsonDestination -Force
+ $saved = @($tokenizerJsonDestination)
+
+ $vocabPath = Join-Path $modelsDir "vocab.txt"
+ if (Export-VocabFromTokenizerJson -TokenizerJsonPath $tokenizerJsonDestination -VocabPath $vocabPath) {
+ $saved += $vocabPath
+ }
+
+ Write-Warning (
+ "No SentencePiece model on Hugging Face; saved WordPiece assets ($(
+ ($saved | ForEach-Object { Split-Path $_ -Leaf }) -join ', '
+ )). TamilOnnxNerRunner uses vocab.txt (WordPiece) when present, otherwise SentencePiece model files."
+ )
+ return $saved
+ }
+
+ return @()
+}
+
+function Download-WithPowerShell {
+ Ensure-ModelsDirectory
+
+ $modelPath = Join-Path $modelsDir "model.onnx"
+ $configPath = Join-Path $modelsDir "config.json"
+ $labelsPath = Join-Path $modelsDir "ner-labels.txt"
+ $tempDir = Join-Path $modelsDir "_download_temp"
+ New-Item -ItemType Directory -Force -Path $tempDir | Out-Null
+
+ try {
+ Download-HuggingFaceAsset -RelativePath "onnx/model.onnx" -Destination $modelPath
+ }
+ catch {
+ Write-Host "Pre-exported ONNX not found; downloading config and tokenizer for manual export..."
+ Download-HuggingFaceAsset -RelativePath "config.json" -Destination $configPath
+ Export-LabelsFromConfig -ConfigPath $configPath -LabelsPath $labelsPath
+
+ $tokenizerJsonPath = Join-Path $modelsDir "tokenizer.json"
+ try {
+ Download-HuggingFaceAsset -RelativePath "tokenizer.json" -Destination $tokenizerJsonPath
+ $vocabPath = Join-Path $modelsDir "vocab.txt"
+ Export-VocabFromTokenizerJson -TokenizerJsonPath $tokenizerJsonPath -VocabPath $vocabPath | Out-Null
+ }
+ catch {
+ Write-Host "tokenizer.json not available from Hugging Face."
+ }
+
+ foreach ($name in @("sentencepiece.bpe.model", "spiece.model", "tokenizer.model")) {
+ try {
+ Download-HuggingFaceAsset -RelativePath $name -Destination (Join-Path $modelsDir $name)
+ break
+ }
+ catch {
+ continue
+ }
+ }
+
+ $pythonExe = Resolve-PythonExecutable -Preferred $Python
+ if ($pythonExe) {
+ throw (
+ "Tamil ONNX model is not published on Hugging Face. Re-run with Python export:`n" +
+ " .\scripts\download-tamil-ner-model.ps1 -Python `"$pythonExe`""
+ )
+ }
+
+ throw @"
+Tamil ONNX model is not published on Hugging Face (onnx/model.onnx returns 404).
+Install Python 3.12+ and re-run this script:
+ winget install Python.Python.3.12 --accept-package-agreements --accept-source-agreements
+ .\scripts\download-tamil-ner-model.ps1 -Python `"`$env:LOCALAPPDATA\Programs\Python\Python312\python.exe`"
+Only config.json, ner-labels.txt, and tokenizer.json were saved under models/ta/.
+"@
+ }
+
+ Download-HuggingFaceAsset -RelativePath "config.json" -Destination $configPath
+ Export-LabelsFromConfig -ConfigPath $configPath -LabelsPath $labelsPath
+ Remove-Item $configPath -Force
+
+ foreach ($name in @("sentencepiece.bpe.model", "spiece.model")) {
+ try {
+ Download-HuggingFaceAsset -RelativePath $name -Destination (Join-Path $modelsDir $name)
+ break
+ }
+ catch {
+ continue
+ }
+ }
+
+ Write-Host ""
+ Write-Host "Tamil NER model assets saved to $modelsDir"
+}
+
+function Download-WithPython {
+ $pythonExe = Resolve-PythonExecutable -Preferred $Python
+ if (-not $pythonExe) {
+ return $false
+ }
+
+ Write-Host "Using Python: $pythonExe"
+ Write-Host "Repository root: $repoRoot"
+ Write-Host ""
+
+ Push-Location $repoRoot
+ try {
+ & $pythonExe $scriptPath
+ if ($LASTEXITCODE -ne 0) {
+ throw "Tamil model download script failed with exit code $LASTEXITCODE."
+ }
+ }
+ finally {
+ Pop-Location
+ }
+
+ return $true
+}
+
+Write-Host "Repository root: $repoRoot"
+
+if (-not (Download-WithPython)) {
+ Write-Host "Python export unavailable; downloading pre-exported ONNX assets from Hugging Face..."
+ Write-Host ""
+ Download-WithPowerShell
+}
diff --git a/scripts/download-tamil-ner-model.py b/scripts/download-tamil-ner-model.py
new file mode 100644
index 0000000..f53c01d
--- /dev/null
+++ b/scripts/download-tamil-ner-model.py
@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""Download and export prachuryyaIITG/SampurNER_Tamil_IndicBERTv2 to ONNX for the PII Redaction POC."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+MODELS_DIR = REPO_ROOT / "models" / "ta"
+MODEL_ID = "prachuryyaIITG/SampurNER_Tamil_IndicBERTv2"
+REQUIRED_PACKAGES = ("transformers", "optimum[onnxruntime]", "onnx", "torch")
+SENTENCEPIECE_CANDIDATES = (
+ "sentencepiece.bpe.model",
+ "spiece.model",
+ "tokenizer.model",
+)
+TOKENIZER_JSON = "tokenizer.json"
+VOCAB_TXT = "vocab.txt"
+
+
+def ensure_dependencies() -> None:
+ try:
+ import optimum.onnxruntime # noqa: F401
+ import transformers # noqa: F401
+ except ImportError:
+ print("Installing Python dependencies (this may take a few minutes)...")
+ subprocess.check_call(
+ [sys.executable, "-m", "pip", "install", *REQUIRED_PACKAGES],
+ stdout=sys.stdout,
+ stderr=sys.stderr,
+ )
+
+
+def copy_sentencepiece_model(source_dir: Path, target_dir: Path) -> Path | None:
+ for name in SENTENCEPIECE_CANDIDATES:
+ candidate = source_dir / name
+ if candidate.exists():
+ destination = target_dir / name
+ shutil.copy(candidate, destination)
+ return destination
+
+ return None
+
+
+def export_wordpiece_assets(source_dir: Path, target_dir: Path) -> list[Path]:
+ saved: list[Path] = []
+ tokenizer_json = source_dir / TOKENIZER_JSON
+ if not tokenizer_json.exists():
+ return saved
+
+ destination = target_dir / TOKENIZER_JSON
+ shutil.copy(tokenizer_json, destination)
+ saved.append(destination)
+
+ with tokenizer_json.open(encoding="utf-8") as tokenizer_file:
+ tokenizer_data = json.load(tokenizer_file)
+
+ vocab = tokenizer_data.get("model", {}).get("vocab")
+ if not isinstance(vocab, dict):
+ return saved
+
+ vocab_path = target_dir / VOCAB_TXT
+ ordered_tokens = [token for token, _ in sorted(vocab.items(), key=lambda item: item[1])]
+ vocab_path.write_text("\n".join(ordered_tokens), encoding="utf-8")
+ saved.append(vocab_path)
+ return saved
+
+
+def copy_tokenizer_assets(source_dir: Path, target_dir: Path) -> list[Path]:
+ sentencepiece_path = copy_sentencepiece_model(source_dir, target_dir)
+ if sentencepiece_path is not None:
+ return [sentencepiece_path]
+
+ wordpiece_assets = export_wordpiece_assets(source_dir, target_dir)
+ if wordpiece_assets:
+ print(
+ "WARNING: Hugging Face repo has no SentencePiece model; saved WordPiece "
+ f"assets ({', '.join(path.name for path in wordpiece_assets)}). "
+ "TamilOnnxNerRunner uses vocab.txt (WordPiece) when present, "
+ "otherwise SentencePiece model files."
+ )
+ return wordpiece_assets
+
+ raise FileNotFoundError(
+ f"No tokenizer assets found in {source_dir}. "
+ f"Expected one of {SENTENCEPIECE_CANDIDATES} or {TOKENIZER_JSON}."
+ )
+
+
+def export_model() -> None:
+ from optimum.onnxruntime import ORTModelForTokenClassification
+ from transformers import AutoTokenizer
+
+ MODELS_DIR.mkdir(parents=True, exist_ok=True)
+ temp_dir = MODELS_DIR / "_export_temp"
+ if temp_dir.exists():
+ shutil.rmtree(temp_dir)
+ temp_dir.mkdir()
+
+ print(f"Exporting {MODEL_ID} to ONNX...")
+ model = ORTModelForTokenClassification.from_pretrained(MODEL_ID, export=True)
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+
+ model.save_pretrained(temp_dir)
+ tokenizer.save_pretrained(temp_dir)
+
+ onnx_files = sorted(temp_dir.glob("*.onnx"))
+ if not onnx_files:
+ raise FileNotFoundError("Export completed but no .onnx file was produced.")
+
+ target_onnx = MODELS_DIR / "model.onnx"
+ shutil.copy(onnx_files[0], target_onnx)
+
+ config_path = temp_dir / "config.json"
+ with config_path.open(encoding="utf-8") as config_file:
+ config = json.load(config_file)
+
+ id2label = config.get("id2label", {})
+ labels = [id2label[str(index)] for index in range(len(id2label))]
+ labels_path = MODELS_DIR / "ner-labels.txt"
+ labels_path.write_text("\n".join(labels), encoding="utf-8")
+
+ tokenizer_assets = copy_tokenizer_assets(temp_dir, MODELS_DIR)
+ shutil.rmtree(temp_dir)
+
+ print()
+ print("Tamil NER model assets saved:")
+ print(f" {target_onnx}")
+ for asset in tokenizer_assets:
+ print(f" {asset}")
+ print(f" {labels_path}")
+ print()
+ print("Run from repository root:")
+ print(" dotnet test --filter Category=TamilNer")
+
+
+def main() -> int:
+ ensure_dependencies()
+ export_model()
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/tamil-ner-diagnostic/Program.cs b/scripts/tamil-ner-diagnostic/Program.cs
new file mode 100644
index 0000000..66a3a5d
--- /dev/null
+++ b/scripts/tamil-ner-diagnostic/Program.cs
@@ -0,0 +1,27 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Microsoft.ML.Tokenizers;
+using PiiRedaction.Core.Configuration;
+using PiiRedaction.Infrastructure.Onnx;
+
+var modelDir = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "models", "ta"));
+if (!Directory.Exists(modelDir))
+{
+ modelDir = Path.GetFullPath("models/ta");
+}
+
+var vocabPath = Path.Combine(modelDir, "vocab.txt");
+var bertOptions = new BertOptions { LowerCaseBeforeTokenization = false, ApplyBasicTokenization = false };
+var tokenizer = BertTokenizer.Create(vocabPath, bertOptions);
+var text = "வாடிக்கையாளர் ராஜேஷ் குமார் அழைத்தார்.";
+var tokens = tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true);
+Console.WriteLine($"count={tokens.Count}");
+foreach (var t in tokens) Console.WriteLine($"{t.Id}\t{t.Value}");
+
+var runnerOptions = Options.Create(new PiiRedactionOptions { TamilOnnxModelPath = Path.Combine(modelDir, "model.onnx") });
+using var runner = new TamilOnnxNerRunner(runnerOptions, NullLogger.Instance);
+Console.WriteLine($"Available: {runner.IsModelAvailable}");
+foreach (var e in runner.PredictEntities(text))
+{
+ Console.WriteLine($"Entity: '{e.Value}' [{e.StartIndex},{e.Length}]");
+}
diff --git a/scripts/tamil-ner-diagnostic/tamil-ner-diagnostic.csproj b/scripts/tamil-ner-diagnostic/tamil-ner-diagnostic.csproj
new file mode 100644
index 0000000..dd3fd3f
--- /dev/null
+++ b/scripts/tamil-ner-diagnostic/tamil-ner-diagnostic.csproj
@@ -0,0 +1,14 @@
+
+
+ Exe
+ net10.0
+ enable
+ enable
+
+
+
+
+
+
+
+
diff --git a/src/PiiRedaction.ConsoleApp/DependencyInjection/ServiceCollectionExtensions.cs b/src/PiiRedaction.ConsoleApp/DependencyInjection/ServiceCollectionExtensions.cs
index b5229e4..e96b67f 100644
--- a/src/PiiRedaction.ConsoleApp/DependencyInjection/ServiceCollectionExtensions.cs
+++ b/src/PiiRedaction.ConsoleApp/DependencyInjection/ServiceCollectionExtensions.cs
@@ -31,8 +31,9 @@ public static class ServiceCollectionExtensions
services.AddSingleton();
services.AddSingleton();
- services.AddSingleton();
- services.AddSingleton(provider => provider.GetRequiredService());
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
services.AddSingleton();
services.AddSingleton();
diff --git a/src/PiiRedaction.ConsoleApp/Program.cs b/src/PiiRedaction.ConsoleApp/Program.cs
index d45f5c3..b16deba 100644
--- a/src/PiiRedaction.ConsoleApp/Program.cs
+++ b/src/PiiRedaction.ConsoleApp/Program.cs
@@ -5,6 +5,10 @@ using PiiRedaction.ConsoleApp.Samples;
using PiiRedaction.Core.Abstractions;
using PiiRedaction.Core.Models;
+System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
+Console.InputEncoding = System.Text.Encoding.UTF8;
+Console.OutputEncoding = System.Text.Encoding.UTF8;
+
var interactive = args.Contains("--interactive", StringComparer.OrdinalIgnoreCase);
var listSamples = args.Contains("--list", StringComparer.OrdinalIgnoreCase);
diff --git a/src/PiiRedaction.ConsoleApp/Samples/PromptDemoRunner.cs b/src/PiiRedaction.ConsoleApp/Samples/PromptDemoRunner.cs
index 2d1c28e..3e5eac8 100644
--- a/src/PiiRedaction.ConsoleApp/Samples/PromptDemoRunner.cs
+++ b/src/PiiRedaction.ConsoleApp/Samples/PromptDemoRunner.cs
@@ -59,7 +59,7 @@ public sealed class PromptDemoRunner
Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --sample 2");
Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --name MrTitlePerson");
Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --list");
- Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --interactive");
+ Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --interactive # UTF-8 input recommended for non-ASCII text");
}
private static void DisplayDetectedEntities(IReadOnlyList entities)
diff --git a/src/PiiRedaction.ConsoleApp/Samples/SamplePromptCatalog.cs b/src/PiiRedaction.ConsoleApp/Samples/SamplePromptCatalog.cs
index f9768dd..1f2032c 100644
--- a/src/PiiRedaction.ConsoleApp/Samples/SamplePromptCatalog.cs
+++ b/src/PiiRedaction.ConsoleApp/Samples/SamplePromptCatalog.cs
@@ -68,6 +68,36 @@ public static class SamplePromptCatalog
"Loan number, customer ID, and account number together.",
"Please verify LN-100200 for CustomerId CID-3000 on AccountNumber ACC-400500."),
+ new(
+ "TamilCustomerNameOnly",
+ "NER (Tamil)",
+ "Tamil script person name detected via Tamil ONNX NER.",
+ "வாடிக்கையாளர் ராஜேஷ் குமார் சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்."),
+
+ new(
+ "TamilWithPhonePan",
+ "NER (Tamil) + Regex",
+ "Tamil script person plus phone and PAN (regex).",
+ "வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210 PAN ABCDE1234F."),
+
+ new(
+ "TanglishCustomer",
+ "NER (English/Tanglish)",
+ "Latin-script Tanglish person name via English ONNX NER.",
+ "Customer Senthil phone 9876543210 reported a failed UPI transfer."),
+
+ new(
+ "MixedTamilEnglish",
+ "NER (Mixed)",
+ "Code-mixed Tamil and English — both script routers may contribute person spans.",
+ "வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge."),
+
+ new(
+ "TamilFullFinancial",
+ "NER (Tamil) + Regex + Domain",
+ "Tamil person with email, phone, loan number, and PAN (canonical demo in Tamil).",
+ "வாடிக்கையாளர் ராஜேஷ் குமார் மின்னஞ்சல் ravi.kumar@gmail.com தொலைபேசி 9876543210 LoanNumber LN-456789 PAN ABCDE1234F. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்."),
+
new(
"NoPiiCleanTicket",
"Negative",
diff --git a/src/PiiRedaction.ConsoleApp/appsettings.json b/src/PiiRedaction.ConsoleApp/appsettings.json
index 4d12bf6..cb79f50 100644
--- a/src/PiiRedaction.ConsoleApp/appsettings.json
+++ b/src/PiiRedaction.ConsoleApp/appsettings.json
@@ -1,5 +1,8 @@
{
"PiiRedaction": {
- "OnnxModelPath": "models/ner-model.onnx"
+ "OnnxModelPath": "models/ner-model.onnx",
+ "EnglishOnnxModelPath": "models/en/ner-model.onnx",
+ "TamilOnnxModelPath": "models/ta/model.onnx",
+ "EnableTamilNer": true
}
}
diff --git a/src/PiiRedaction.Core/Configuration/PiiRedactionOptions.cs b/src/PiiRedaction.Core/Configuration/PiiRedactionOptions.cs
index b216a8b..ac93f46 100644
--- a/src/PiiRedaction.Core/Configuration/PiiRedactionOptions.cs
+++ b/src/PiiRedaction.Core/Configuration/PiiRedactionOptions.cs
@@ -5,4 +5,10 @@ public sealed class PiiRedactionOptions
public const string SectionName = "PiiRedaction";
public string OnnxModelPath { get; set; } = "models/ner-model.onnx";
+
+ public string EnglishOnnxModelPath { get; set; } = "models/en/ner-model.onnx";
+
+ public string TamilOnnxModelPath { get; set; } = "models/ta/model.onnx";
+
+ public bool EnableTamilNer { get; set; } = true;
}
diff --git a/src/PiiRedaction.Core/Detection/ScriptComposition.cs b/src/PiiRedaction.Core/Detection/ScriptComposition.cs
new file mode 100644
index 0000000..ecaf823
--- /dev/null
+++ b/src/PiiRedaction.Core/Detection/ScriptComposition.cs
@@ -0,0 +1,9 @@
+namespace PiiRedaction.Core.Detection;
+
+public enum ScriptComposition
+{
+ LatinOnly,
+ TamilOnly,
+ Mixed,
+ NoLetters
+}
diff --git a/src/PiiRedaction.Core/Detection/ScriptRouter.cs b/src/PiiRedaction.Core/Detection/ScriptRouter.cs
new file mode 100644
index 0000000..50ac958
--- /dev/null
+++ b/src/PiiRedaction.Core/Detection/ScriptRouter.cs
@@ -0,0 +1,45 @@
+namespace PiiRedaction.Core.Detection;
+
+///
+/// Classifies prompt text by script composition to route NER inference.
+///
+public sealed class ScriptRouter
+{
+ private const char TamilRangeStart = '\u0B80';
+ private const char TamilRangeEnd = '\u0BFF';
+
+ public ScriptComposition GetComposition(string text)
+ {
+ ArgumentNullException.ThrowIfNull(text);
+
+ var hasLatin = false;
+ var hasTamil = false;
+
+ foreach (var character in text)
+ {
+ if (IsTamilLetter(character))
+ {
+ hasTamil = true;
+ }
+ else if (char.IsAsciiLetter(character))
+ {
+ hasLatin = true;
+ }
+
+ if (hasLatin && hasTamil)
+ {
+ return ScriptComposition.Mixed;
+ }
+ }
+
+ if (!hasLatin && !hasTamil)
+ {
+ return ScriptComposition.NoLetters;
+ }
+
+ return hasTamil ? ScriptComposition.TamilOnly : ScriptComposition.LatinOnly;
+ }
+
+ internal static bool IsTamilLetter(char character) =>
+ character is >= TamilRangeStart and <= TamilRangeEnd;
+}
diff --git a/src/PiiRedaction.Infrastructure/Onnx/BertWordPieceEncoder.cs b/src/PiiRedaction.Infrastructure/Onnx/BertWordPieceEncoder.cs
new file mode 100644
index 0000000..211e6fd
--- /dev/null
+++ b/src/PiiRedaction.Infrastructure/Onnx/BertWordPieceEncoder.cs
@@ -0,0 +1,107 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.ML.Tokenizers;
+
+namespace PiiRedaction.Infrastructure.Onnx;
+
+public sealed class BertWordPieceEncoder : ITokenClassifierEncoder
+{
+ private readonly BertTokenizer? _tokenizer;
+ private readonly ILogger _logger;
+
+ public BertWordPieceEncoder(string modelDirectory, ILogger logger)
+ {
+ _logger = logger;
+ _tokenizer = TryLoadTokenizer(modelDirectory);
+ }
+
+ public bool IsAvailable => _tokenizer is not null;
+
+ public EncodedSequence? Encode(string text, int maxSequenceLength)
+ {
+ if (_tokenizer is null)
+ {
+ return null;
+ }
+
+ var encodedTokens = _tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true);
+ var wordTokens = encodedTokens.Take(Math.Max(0, maxSequenceLength - 2)).ToList();
+ if (wordTokens.Count == 0)
+ {
+ return null;
+ }
+
+ var sequenceLength = wordTokens.Count + 2;
+ var inputIds = new long[sequenceLength];
+ var attentionMask = new long[sequenceLength];
+ var tokenTypeIds = new long[sequenceLength];
+ var offsets = new (int Start, int End)[sequenceLength];
+ var tokenIds = new int[sequenceLength];
+
+ inputIds[0] = _tokenizer.ClassificationTokenId;
+ attentionMask[0] = 1;
+ tokenIds[0] = _tokenizer.ClassificationTokenId;
+ offsets[0] = (0, 0);
+
+ for (var i = 0; i < wordTokens.Count; i++)
+ {
+ var token = wordTokens[i];
+ var index = i + 1;
+ inputIds[index] = token.Id;
+ attentionMask[index] = 1;
+ tokenIds[index] = token.Id;
+ offsets[index] = ToCharOffsets(token.Offset, text.Length);
+ }
+
+ inputIds[sequenceLength - 1] = _tokenizer.SeparatorTokenId;
+ attentionMask[sequenceLength - 1] = 1;
+ tokenIds[sequenceLength - 1] = _tokenizer.SeparatorTokenId;
+ offsets[sequenceLength - 1] = (0, 0);
+
+ return new EncodedSequence(inputIds, attentionMask, tokenTypeIds, offsets, tokenIds, sequenceLength);
+ }
+
+ public bool IsSpecialToken(int tokenId) =>
+ _tokenizer is not null &&
+ (tokenId == _tokenizer.ClassificationTokenId ||
+ tokenId == _tokenizer.SeparatorTokenId ||
+ tokenId == _tokenizer.PaddingTokenId);
+
+ private BertTokenizer? TryLoadTokenizer(string modelDirectory)
+ {
+ var vocabPath = OnnxAssetPathResolver.ResolveAssetPath(Path.Combine(modelDirectory, "vocab.txt"));
+ if (!File.Exists(vocabPath))
+ {
+ _logger.LogWarning("Tokenizer vocabulary not found at {VocabPath}.", vocabPath);
+ return null;
+ }
+
+ try
+ {
+ return BertTokenizer.Create(vocabPath, CreateBertOptions(modelDirectory));
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Failed to load BERT tokenizer from {VocabPath}.", vocabPath);
+ return null;
+ }
+ }
+
+ private static BertOptions CreateBertOptions(string modelDirectory)
+ {
+ var tokenizerJsonPath = OnnxAssetPathResolver.ResolveAssetPath(
+ Path.Combine(modelDirectory, "tokenizer.json"));
+ var whitespaceOnlyPretokenization = File.Exists(tokenizerJsonPath);
+
+ return new BertOptions
+ {
+ LowerCaseBeforeTokenization = false,
+ ApplyBasicTokenization = !whitespaceOnlyPretokenization
+ };
+ }
+
+ private static (int Start, int End) ToCharOffsets(Range offset, int textLength)
+ {
+ var (start, length) = offset.GetOffsetAndLength(textLength);
+ return (start, start + length);
+ }
+}
diff --git a/src/PiiRedaction.Infrastructure/Onnx/EnglishOnnxNerRunner.cs b/src/PiiRedaction.Infrastructure/Onnx/EnglishOnnxNerRunner.cs
new file mode 100644
index 0000000..d70d2d9
--- /dev/null
+++ b/src/PiiRedaction.Infrastructure/Onnx/EnglishOnnxNerRunner.cs
@@ -0,0 +1,35 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using PiiRedaction.Core.Configuration;
+using PiiRedaction.Core.Detection;
+using PiiRedaction.Core.Models;
+
+namespace PiiRedaction.Infrastructure.Onnx;
+
+public class EnglishOnnxNerRunner : IOnnxNerModelRunner, IDisposable
+{
+ private readonly OnnxTokenClassifierRunner _runner;
+
+ protected EnglishOnnxNerRunner(IOptions options, ILogger logger)
+ {
+ var modelPath = OnnxAssetPathResolver.ResolveModelPath(
+ options.Value.EnglishOnnxModelPath,
+ options.Value.OnnxModelPath);
+
+ var modelDirectory = Path.GetDirectoryName(modelPath) ?? Environment.CurrentDirectory;
+ var labels = OnnxAssetPathResolver.LoadLabels(modelDirectory);
+ var encoder = new BertWordPieceEncoder(modelDirectory, logger);
+ _runner = new OnnxTokenClassifierRunner(modelPath, encoder, NerLabelConfig.English, labels, logger);
+ }
+
+ public EnglishOnnxNerRunner(IOptions options, ILogger logger)
+ : this(options, (ILogger)logger)
+ {
+ }
+
+ public bool IsModelAvailable => _runner.IsAvailable;
+
+ public IReadOnlyList PredictEntities(string text) => _runner.PredictEntities(text);
+
+ public void Dispose() => _runner.Dispose();
+}
diff --git a/src/PiiRedaction.Infrastructure/Onnx/ITokenClassifierEncoder.cs b/src/PiiRedaction.Infrastructure/Onnx/ITokenClassifierEncoder.cs
new file mode 100644
index 0000000..d14111d
--- /dev/null
+++ b/src/PiiRedaction.Infrastructure/Onnx/ITokenClassifierEncoder.cs
@@ -0,0 +1,18 @@
+namespace PiiRedaction.Infrastructure.Onnx;
+
+public sealed record EncodedSequence(
+ long[] InputIds,
+ long[] AttentionMask,
+ long[] TokenTypeIds,
+ (int Start, int End)[] Offsets,
+ int[] TokenIds,
+ int SequenceLength);
+
+public interface ITokenClassifierEncoder
+{
+ bool IsAvailable { get; }
+
+ EncodedSequence? Encode(string text, int maxSequenceLength);
+
+ bool IsSpecialToken(int tokenId);
+}
diff --git a/src/PiiRedaction.Infrastructure/Onnx/NerLabelConfig.cs b/src/PiiRedaction.Infrastructure/Onnx/NerLabelConfig.cs
new file mode 100644
index 0000000..415e7e3
--- /dev/null
+++ b/src/PiiRedaction.Infrastructure/Onnx/NerLabelConfig.cs
@@ -0,0 +1,26 @@
+namespace PiiRedaction.Infrastructure.Onnx;
+
+public sealed class NerLabelConfig
+{
+ private readonly Func _isPersonLabel;
+
+ private NerLabelConfig(Func isPersonLabel) => _isPersonLabel = isPersonLabel;
+
+ public static NerLabelConfig English { get; } = new(IsEnglishPersonLabel);
+
+ public static NerLabelConfig Tamil { get; } = new(IsTamilPersonLabel);
+
+ public bool IsPersonLabel(string label) => _isPersonLabel(label);
+
+ public bool IsBeginLabel(string label) => label.StartsWith("B-", StringComparison.Ordinal);
+
+ public bool IsInsideLabel(string label) => label.StartsWith("I-", StringComparison.Ordinal);
+
+ private static bool IsEnglishPersonLabel(string label) =>
+ label is "B-PER" or "I-PER" or "B-PERSON" or "I-PERSON"
+ || (label.EndsWith("-PER", StringComparison.Ordinal) &&
+ (label.StartsWith("B-", StringComparison.Ordinal) || label.StartsWith("I-", StringComparison.Ordinal)));
+
+ private static bool IsTamilPersonLabel(string label) =>
+ label.Contains("person", StringComparison.OrdinalIgnoreCase);
+}
diff --git a/src/PiiRedaction.Infrastructure/Onnx/OnnxAssetPathResolver.cs b/src/PiiRedaction.Infrastructure/Onnx/OnnxAssetPathResolver.cs
new file mode 100644
index 0000000..ff96b7a
--- /dev/null
+++ b/src/PiiRedaction.Infrastructure/Onnx/OnnxAssetPathResolver.cs
@@ -0,0 +1,53 @@
+namespace PiiRedaction.Infrastructure.Onnx;
+
+public static class OnnxAssetPathResolver
+{
+ public static string ResolveAssetPath(string configuredPath)
+ {
+ if (Path.IsPathRooted(configuredPath) && File.Exists(configuredPath))
+ {
+ return configuredPath;
+ }
+
+ var directory = new DirectoryInfo(Environment.CurrentDirectory);
+ while (directory is not null)
+ {
+ var candidate = Path.GetFullPath(Path.Combine(directory.FullName, configuredPath));
+ if (File.Exists(candidate))
+ {
+ return candidate;
+ }
+
+ directory = directory.Parent;
+ }
+
+ return Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, configuredPath));
+ }
+
+ public static string ResolveModelPath(string primaryPath, params string[] fallbackPaths)
+ {
+ foreach (var candidatePath in new[] { primaryPath }.Concat(fallbackPaths))
+ {
+ var resolved = ResolveAssetPath(candidatePath);
+ if (File.Exists(resolved))
+ {
+ return resolved;
+ }
+ }
+
+ return ResolveAssetPath(primaryPath);
+ }
+
+ public static string[] LoadLabels(string modelDirectory)
+ {
+ var labelsPath = ResolveAssetPath(Path.Combine(modelDirectory, "ner-labels.txt"));
+ if (!File.Exists(labelsPath))
+ {
+ return [];
+ }
+
+ return File.ReadAllLines(labelsPath)
+ .Where(line => !string.IsNullOrWhiteSpace(line))
+ .ToArray();
+ }
+}
diff --git a/src/PiiRedaction.Infrastructure/Onnx/OnnxNerModelRunner.cs b/src/PiiRedaction.Infrastructure/Onnx/OnnxNerModelRunner.cs
index 2b2646b..26722af 100644
--- a/src/PiiRedaction.Infrastructure/Onnx/OnnxNerModelRunner.cs
+++ b/src/PiiRedaction.Infrastructure/Onnx/OnnxNerModelRunner.cs
@@ -1,325 +1,16 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
-using Microsoft.ML.OnnxRuntime;
-using Microsoft.ML.OnnxRuntime.Tensors;
-using Microsoft.ML.Tokenizers;
using PiiRedaction.Core.Configuration;
-using PiiRedaction.Core.Detection;
-using PiiRedaction.Core.Models;
namespace PiiRedaction.Infrastructure.Onnx;
///
-/// Wraps ONNX Runtime inference for NER models.
-/// Tokenization and tensor preparation are isolated here so detectors remain model-agnostic.
+/// Backward-compatible alias for .
///
-public sealed class OnnxNerModelRunner : IOnnxNerModelRunner, IDisposable
+public sealed class OnnxNerModelRunner : EnglishOnnxNerRunner
{
- private const int MaxSequenceLength = 128;
-
- private readonly ILogger _logger;
- private readonly string _modelPath;
- private readonly BertTokenizer? _tokenizer;
- private readonly string[] _labels;
- private InferenceSession? _session;
-
public OnnxNerModelRunner(IOptions options, ILogger logger)
+ : base(options, logger)
{
- _logger = logger;
- _modelPath = ResolveAssetPath(options.Value.OnnxModelPath);
- var modelDirectory = Path.GetDirectoryName(_modelPath) ?? Environment.CurrentDirectory;
- _labels = LoadLabels(modelDirectory);
- _tokenizer = TryLoadTokenizer(modelDirectory);
- _session = TryCreateSession();
}
-
- public bool IsModelAvailable => _session is not null && _tokenizer is not null && _labels.Length > 0;
-
- public IReadOnlyList PredictEntities(string text)
- {
- if (!IsModelAvailable || _session is null || _tokenizer is null)
- {
- return [];
- }
-
- var encodedTokens = _tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true);
- var wordTokens = encodedTokens.Take(Math.Max(0, MaxSequenceLength - 2)).ToList();
- if (wordTokens.Count == 0)
- {
- return [];
- }
-
- var sequenceLength = wordTokens.Count + 2;
- var inputIds = new long[sequenceLength];
- var attentionMask = new long[sequenceLength];
- var tokenTypeIds = new long[sequenceLength];
- var offsets = new (int Start, int End)[sequenceLength];
- var tokenIds = new int[sequenceLength];
-
- inputIds[0] = _tokenizer.ClassificationTokenId;
- attentionMask[0] = 1;
- tokenIds[0] = _tokenizer.ClassificationTokenId;
- offsets[0] = (0, 0);
-
- for (var i = 0; i < wordTokens.Count; i++)
- {
- var token = wordTokens[i];
- var index = i + 1;
- inputIds[index] = token.Id;
- attentionMask[index] = 1;
- tokenIds[index] = token.Id;
- offsets[index] = ToCharOffsets(token.Offset, text.Length);
- }
-
- inputIds[sequenceLength - 1] = _tokenizer.SeparatorTokenId;
- attentionMask[sequenceLength - 1] = 1;
- tokenIds[sequenceLength - 1] = _tokenizer.SeparatorTokenId;
- offsets[sequenceLength - 1] = (0, 0);
-
- var predictedLabelIds = RunInference(inputIds, attentionMask, tokenTypeIds, sequenceLength);
- return DecodePersonEntities(text, predictedLabelIds, offsets, tokenIds, sequenceLength);
- }
-
- private int[] RunInference(long[] inputIds, long[] attentionMask, long[] tokenTypeIds, int sequenceLength)
- {
- var inputIdsTensor = CreateTensor(inputIds, sequenceLength);
- var attentionMaskTensor = CreateTensor(attentionMask, sequenceLength);
- var inputs = new List
- {
- NamedOnnxValue.CreateFromTensor(_session!.InputMetadata.Keys.First(key => key.Contains("input_ids", StringComparison.OrdinalIgnoreCase)), inputIdsTensor),
- NamedOnnxValue.CreateFromTensor(_session.InputMetadata.Keys.First(key => key.Contains("attention_mask", StringComparison.OrdinalIgnoreCase)), attentionMaskTensor)
- };
-
- var tokenTypeInputName = _session.InputMetadata.Keys.FirstOrDefault(key => key.Contains("token_type", StringComparison.OrdinalIgnoreCase));
- if (tokenTypeInputName is not null)
- {
- inputs.Add(NamedOnnxValue.CreateFromTensor(tokenTypeInputName, CreateTensor(tokenTypeIds, sequenceLength)));
- }
-
- using var results = _session.Run(inputs);
- var outputName = _session.OutputMetadata.Keys.FirstOrDefault(key =>
- key.Contains("logits", StringComparison.OrdinalIgnoreCase))
- ?? results.First().Name;
- var logits = results.First(result => result.Name == outputName).AsTensor();
- var numLabels = _labels.Length;
- var predictions = new int[sequenceLength];
-
- for (var tokenIndex = 0; tokenIndex < sequenceLength; tokenIndex++)
- {
- var bestLabel = 0;
- var bestScore = float.MinValue;
-
- for (var labelIndex = 0; labelIndex < numLabels; labelIndex++)
- {
- var score = logits[0, tokenIndex, labelIndex];
- if (score > bestScore)
- {
- bestScore = score;
- bestLabel = labelIndex;
- }
- }
-
- predictions[tokenIndex] = bestLabel;
- }
-
- return predictions;
- }
-
- private static DenseTensor CreateTensor(long[] values, int sequenceLength)
- {
- var tensor = new DenseTensor([1, sequenceLength]);
- for (var i = 0; i < sequenceLength; i++)
- {
- tensor[0, i] = values[i];
- }
-
- return tensor;
- }
-
- private IReadOnlyList DecodePersonEntities(
- string text,
- int[] predictedLabelIds,
- (int Start, int End)[] offsets,
- int[] tokenIds,
- int sequenceLength)
- {
- var entities = new List();
- int? entityStart = null;
- int? entityEnd = null;
-
- void FlushEntity()
- {
- if (!entityStart.HasValue || !entityEnd.HasValue || entityEnd.Value <= entityStart.Value)
- {
- entityStart = null;
- entityEnd = null;
- return;
- }
-
- var value = text[entityStart.Value..entityEnd.Value];
- if (!string.IsNullOrWhiteSpace(value))
- {
- entities.Add(new PiiEntity(
- PiiEntityType.Person,
- value,
- entityStart.Value,
- entityEnd.Value - entityStart.Value,
- PiiDetectionSource.Ner));
- }
-
- entityStart = null;
- entityEnd = null;
- }
-
- for (var i = 0; i < sequenceLength; i++)
- {
- if (IsSpecialToken(tokenIds[i]))
- {
- FlushEntity();
- continue;
- }
-
- var label = _labels[predictedLabelIds[i]];
- var (start, end) = offsets[i];
- var hasOffset = end > start;
-
- if (!IsPersonLabel(label))
- {
- FlushEntity();
- continue;
- }
-
- if (label.StartsWith("B-", StringComparison.Ordinal))
- {
- FlushEntity();
- if (hasOffset)
- {
- entityStart = start;
- entityEnd = end;
- }
- }
- else if (label.StartsWith("I-", StringComparison.Ordinal))
- {
- if (!entityStart.HasValue && hasOffset)
- {
- entityStart = start;
- entityEnd = end;
- }
- else if (hasOffset)
- {
- entityEnd = Math.Max(entityEnd ?? end, end);
- }
- }
- }
-
- FlushEntity();
- return entities;
- }
-
- private bool IsSpecialToken(int tokenId) =>
- tokenId == _tokenizer!.ClassificationTokenId ||
- tokenId == _tokenizer.SeparatorTokenId ||
- tokenId == _tokenizer.PaddingTokenId;
-
- private static (int Start, int End) ToCharOffsets(Range offset, int textLength)
- {
- var (start, length) = offset.GetOffsetAndLength(textLength);
- return (start, start + length);
- }
-
- private static bool IsPersonLabel(string label) =>
- label is "B-PER" or "I-PER" or "B-PERSON" or "I-PERSON"
- || (label.EndsWith("-PER", StringComparison.Ordinal) &&
- (label.StartsWith("B-", StringComparison.Ordinal) || label.StartsWith("I-", StringComparison.Ordinal)));
-
- private InferenceSession? TryCreateSession()
- {
- if (!File.Exists(_modelPath))
- {
- _logger.LogWarning(
- "ONNX NER model not found at {ModelPath}. Person-name detection will return no results.",
- _modelPath);
- return null;
- }
-
- if (_tokenizer is null || _labels.Length == 0)
- {
- _logger.LogWarning(
- "Tokenizer vocabulary or label map missing for ONNX NER model at {ModelPath}.",
- _modelPath);
- return null;
- }
-
- try
- {
- var session = new InferenceSession(_modelPath);
- _logger.LogInformation("ONNX NER model loaded from {ModelPath}.", _modelPath);
- return session;
- }
- catch (Exception ex)
- {
- _logger.LogError(ex, "Failed to load ONNX NER model from {ModelPath}.", _modelPath);
- return null;
- }
- }
-
- private BertTokenizer? TryLoadTokenizer(string modelDirectory)
- {
- var vocabPath = ResolveAssetPath(Path.Combine(modelDirectory, "vocab.txt"));
- if (!File.Exists(vocabPath))
- {
- _logger.LogWarning("Tokenizer vocabulary not found at {VocabPath}.", vocabPath);
- return null;
- }
-
- try
- {
- return BertTokenizer.Create(vocabPath, new BertOptions
- {
- LowerCaseBeforeTokenization = false
- });
- }
- catch (Exception ex)
- {
- _logger.LogError(ex, "Failed to load BERT tokenizer from {VocabPath}.", vocabPath);
- return null;
- }
- }
-
- private static string[] LoadLabels(string modelDirectory)
- {
- var labelsPath = ResolveAssetPath(Path.Combine(modelDirectory, "ner-labels.txt"));
- if (!File.Exists(labelsPath))
- {
- return [];
- }
-
- return File.ReadAllLines(labelsPath)
- .Where(line => !string.IsNullOrWhiteSpace(line))
- .ToArray();
- }
-
- internal static string ResolveAssetPath(string configuredPath)
- {
- if (Path.IsPathRooted(configuredPath) && File.Exists(configuredPath))
- {
- return configuredPath;
- }
-
- var directory = new DirectoryInfo(Environment.CurrentDirectory);
- while (directory is not null)
- {
- var candidate = Path.GetFullPath(Path.Combine(directory.FullName, configuredPath));
- if (File.Exists(candidate))
- {
- return candidate;
- }
-
- directory = directory.Parent;
- }
-
- return Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, configuredPath));
- }
-
- public void Dispose() => _session?.Dispose();
}
diff --git a/src/PiiRedaction.Infrastructure/Onnx/OnnxTokenClassifierRunner.cs b/src/PiiRedaction.Infrastructure/Onnx/OnnxTokenClassifierRunner.cs
new file mode 100644
index 0000000..0601ce1
--- /dev/null
+++ b/src/PiiRedaction.Infrastructure/Onnx/OnnxTokenClassifierRunner.cs
@@ -0,0 +1,229 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.ML.OnnxRuntime;
+using Microsoft.ML.OnnxRuntime.Tensors;
+using PiiRedaction.Core.Models;
+
+namespace PiiRedaction.Infrastructure.Onnx;
+
+///
+/// Shared ONNX token-classification inference and BIO decoding for NER models.
+///
+public sealed class OnnxTokenClassifierRunner : IDisposable
+{
+ private const int MaxSequenceLength = 128;
+
+ private readonly ITokenClassifierEncoder _encoder;
+ private readonly NerLabelConfig _labelConfig;
+ private readonly string[] _labels;
+ private readonly ILogger _logger;
+ private readonly string _modelPath;
+ private InferenceSession? _session;
+
+ public OnnxTokenClassifierRunner(
+ string modelPath,
+ ITokenClassifierEncoder encoder,
+ NerLabelConfig labelConfig,
+ string[] labels,
+ ILogger logger)
+ {
+ _modelPath = modelPath;
+ _encoder = encoder;
+ _labelConfig = labelConfig;
+ _labels = labels;
+ _logger = logger;
+ _session = TryCreateSession();
+ }
+
+ public bool IsAvailable => _session is not null && _encoder.IsAvailable && _labels.Length > 0;
+
+ public IReadOnlyList PredictEntities(string text)
+ {
+ if (!IsAvailable || _session is null)
+ {
+ return [];
+ }
+
+ var encoded = _encoder.Encode(text, MaxSequenceLength);
+ if (encoded is null)
+ {
+ return [];
+ }
+
+ var predictedLabelIds = RunInference(encoded);
+ return DecodePersonEntities(text, predictedLabelIds, encoded);
+ }
+
+ private int[] RunInference(EncodedSequence encoded)
+ {
+ var inputIdsTensor = CreateTensor(encoded.InputIds, encoded.SequenceLength);
+ var attentionMaskTensor = CreateTensor(encoded.AttentionMask, encoded.SequenceLength);
+ var inputs = new List
+ {
+ NamedOnnxValue.CreateFromTensor(
+ _session!.InputMetadata.Keys.First(key => key.Contains("input_ids", StringComparison.OrdinalIgnoreCase)),
+ inputIdsTensor),
+ NamedOnnxValue.CreateFromTensor(
+ _session.InputMetadata.Keys.First(key => key.Contains("attention_mask", StringComparison.OrdinalIgnoreCase)),
+ attentionMaskTensor)
+ };
+
+ var tokenTypeInputName = _session.InputMetadata.Keys.FirstOrDefault(key =>
+ key.Contains("token_type", StringComparison.OrdinalIgnoreCase));
+ if (tokenTypeInputName is not null)
+ {
+ inputs.Add(NamedOnnxValue.CreateFromTensor(
+ tokenTypeInputName,
+ CreateTensor(encoded.TokenTypeIds, encoded.SequenceLength)));
+ }
+
+ using var results = _session.Run(inputs);
+ var outputName = _session.OutputMetadata.Keys.FirstOrDefault(key =>
+ key.Contains("logits", StringComparison.OrdinalIgnoreCase))
+ ?? results.First().Name;
+ var logits = results.First(result => result.Name == outputName).AsTensor();
+ var numLabels = _labels.Length;
+ var predictions = new int[encoded.SequenceLength];
+
+ for (var tokenIndex = 0; tokenIndex < encoded.SequenceLength; tokenIndex++)
+ {
+ var bestLabel = 0;
+ var bestScore = float.MinValue;
+
+ for (var labelIndex = 0; labelIndex < numLabels; labelIndex++)
+ {
+ var score = logits[0, tokenIndex, labelIndex];
+ if (score > bestScore)
+ {
+ bestScore = score;
+ bestLabel = labelIndex;
+ }
+ }
+
+ predictions[tokenIndex] = bestLabel;
+ }
+
+ return predictions;
+ }
+
+ private static DenseTensor CreateTensor(long[] values, int sequenceLength)
+ {
+ var tensor = new DenseTensor([1, sequenceLength]);
+ for (var i = 0; i < sequenceLength; i++)
+ {
+ tensor[0, i] = values[i];
+ }
+
+ return tensor;
+ }
+
+ private IReadOnlyList DecodePersonEntities(
+ string text,
+ int[] predictedLabelIds,
+ EncodedSequence encoded)
+ {
+ var entities = new List();
+ int? entityStart = null;
+ int? entityEnd = null;
+
+ void FlushEntity()
+ {
+ if (!entityStart.HasValue || !entityEnd.HasValue || entityEnd.Value <= entityStart.Value)
+ {
+ entityStart = null;
+ entityEnd = null;
+ return;
+ }
+
+ var value = text[entityStart.Value..entityEnd.Value];
+ if (!string.IsNullOrWhiteSpace(value))
+ {
+ entities.Add(new PiiEntity(
+ PiiEntityType.Person,
+ value,
+ entityStart.Value,
+ entityEnd.Value - entityStart.Value,
+ PiiDetectionSource.Ner));
+ }
+
+ entityStart = null;
+ entityEnd = null;
+ }
+
+ for (var i = 0; i < encoded.SequenceLength; i++)
+ {
+ if (_encoder.IsSpecialToken(encoded.TokenIds[i]))
+ {
+ FlushEntity();
+ continue;
+ }
+
+ var label = _labels[predictedLabelIds[i]];
+ var (start, end) = encoded.Offsets[i];
+ var hasOffset = end > start;
+
+ if (!_labelConfig.IsPersonLabel(label))
+ {
+ FlushEntity();
+ continue;
+ }
+
+ if (_labelConfig.IsBeginLabel(label))
+ {
+ FlushEntity();
+ if (hasOffset)
+ {
+ entityStart = start;
+ entityEnd = end;
+ }
+ }
+ else if (_labelConfig.IsInsideLabel(label))
+ {
+ if (!entityStart.HasValue && hasOffset)
+ {
+ entityStart = start;
+ entityEnd = end;
+ }
+ else if (hasOffset)
+ {
+ entityEnd = Math.Max(entityEnd ?? end, end);
+ }
+ }
+ }
+
+ FlushEntity();
+ return entities;
+ }
+
+ private InferenceSession? TryCreateSession()
+ {
+ if (!File.Exists(_modelPath))
+ {
+ _logger.LogWarning(
+ "ONNX NER model not found at {ModelPath}. Person-name detection will return no results.",
+ _modelPath);
+ return null;
+ }
+
+ if (!_encoder.IsAvailable || _labels.Length == 0)
+ {
+ _logger.LogWarning(
+ "Tokenizer or label map missing for ONNX NER model at {ModelPath}.",
+ _modelPath);
+ return null;
+ }
+
+ try
+ {
+ var session = new InferenceSession(_modelPath);
+ _logger.LogInformation("ONNX NER model loaded from {ModelPath}.", _modelPath);
+ return session;
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Failed to load ONNX NER model from {ModelPath}.", _modelPath);
+ return null;
+ }
+ }
+
+ public void Dispose() => _session?.Dispose();
+}
diff --git a/src/PiiRedaction.Infrastructure/Onnx/RoutingOnnxNerModelRunner.cs b/src/PiiRedaction.Infrastructure/Onnx/RoutingOnnxNerModelRunner.cs
new file mode 100644
index 0000000..db8196d
--- /dev/null
+++ b/src/PiiRedaction.Infrastructure/Onnx/RoutingOnnxNerModelRunner.cs
@@ -0,0 +1,105 @@
+using Microsoft.Extensions.Options;
+using PiiRedaction.Core.Configuration;
+using PiiRedaction.Core.Detection;
+using PiiRedaction.Core.Models;
+
+namespace PiiRedaction.Infrastructure.Onnx;
+
+///
+/// Routes NER inference to English and/or Tamil ONNX models based on script composition.
+///
+public sealed class RoutingOnnxNerModelRunner : IOnnxNerModelRunner
+{
+ private readonly ScriptRouter _scriptRouter = new();
+ private readonly IOnnxNerModelRunner _englishRunner;
+ private readonly IOnnxNerModelRunner _tamilRunner;
+ private readonly bool _enableTamilNer;
+
+ public RoutingOnnxNerModelRunner(
+ EnglishOnnxNerRunner englishRunner,
+ TamilOnnxNerRunner tamilRunner,
+ IOptions options)
+ : this(englishRunner, tamilRunner, options.Value.EnableTamilNer)
+ {
+ }
+
+ internal RoutingOnnxNerModelRunner(
+ IOnnxNerModelRunner englishRunner,
+ IOnnxNerModelRunner tamilRunner,
+ bool enableTamilNer)
+ {
+ _englishRunner = englishRunner;
+ _tamilRunner = tamilRunner;
+ _enableTamilNer = enableTamilNer;
+ }
+
+ public bool IsModelAvailable =>
+ _englishRunner.IsModelAvailable || (_enableTamilNer && _tamilRunner.IsModelAvailable);
+
+ public IReadOnlyList PredictEntities(string text)
+ {
+ var composition = _scriptRouter.GetComposition(text);
+ var entities = new List();
+
+ switch (composition)
+ {
+ case ScriptComposition.LatinOnly:
+ if (_englishRunner.IsModelAvailable)
+ {
+ entities.AddRange(_englishRunner.PredictEntities(text));
+ }
+
+ break;
+
+ case ScriptComposition.TamilOnly:
+ if (_enableTamilNer && _tamilRunner.IsModelAvailable)
+ {
+ entities.AddRange(_tamilRunner.PredictEntities(text));
+ }
+
+ break;
+
+ case ScriptComposition.Mixed:
+ if (_englishRunner.IsModelAvailable)
+ {
+ entities.AddRange(_englishRunner.PredictEntities(text));
+ }
+
+ if (_enableTamilNer && _tamilRunner.IsModelAvailable)
+ {
+ entities.AddRange(_tamilRunner.PredictEntities(text));
+ }
+
+ break;
+
+ case ScriptComposition.NoLetters:
+ break;
+ }
+
+ return MergePersonSpans(entities);
+ }
+
+ internal static IReadOnlyList MergePersonSpans(IReadOnlyList entities)
+ {
+ if (entities.Count <= 1)
+ {
+ return entities;
+ }
+
+ var accepted = new List();
+ foreach (var candidate in entities.OrderByDescending(entity => entity.Length).ThenBy(entity => entity.StartIndex))
+ {
+ if (accepted.Any(existing => Overlaps(existing, candidate)))
+ {
+ continue;
+ }
+
+ accepted.Add(candidate);
+ }
+
+ return accepted.OrderBy(entity => entity.StartIndex).ToList();
+ }
+
+ private static bool Overlaps(PiiEntity left, PiiEntity right) =>
+ left.StartIndex < right.EndIndex && right.StartIndex < left.EndIndex;
+}
diff --git a/src/PiiRedaction.Infrastructure/Onnx/SentencePieceEncoder.cs b/src/PiiRedaction.Infrastructure/Onnx/SentencePieceEncoder.cs
new file mode 100644
index 0000000..06d3c95
--- /dev/null
+++ b/src/PiiRedaction.Infrastructure/Onnx/SentencePieceEncoder.cs
@@ -0,0 +1,102 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.ML.Tokenizers;
+
+namespace PiiRedaction.Infrastructure.Onnx;
+
+public sealed class SentencePieceEncoder : ITokenClassifierEncoder
+{
+ private readonly SentencePieceTokenizer? _tokenizer;
+ private readonly ILogger _logger;
+
+ public SentencePieceEncoder(string modelDirectory, ILogger logger)
+ {
+ _logger = logger;
+ _tokenizer = TryLoadTokenizer(modelDirectory);
+ }
+
+ public bool IsAvailable => _tokenizer is not null;
+
+ public EncodedSequence? Encode(string text, int maxSequenceLength)
+ {
+ if (_tokenizer is null)
+ {
+ return null;
+ }
+
+ var encodedTokens = _tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true);
+ var wordTokens = encodedTokens.Take(Math.Max(0, maxSequenceLength - 2)).ToList();
+ if (wordTokens.Count == 0)
+ {
+ return null;
+ }
+
+ var sequenceLength = wordTokens.Count + 2;
+ var inputIds = new long[sequenceLength];
+ var attentionMask = new long[sequenceLength];
+ var tokenTypeIds = new long[sequenceLength];
+ var offsets = new (int Start, int End)[sequenceLength];
+ var tokenIds = new int[sequenceLength];
+
+ inputIds[0] = _tokenizer.BeginningOfSentenceId;
+ attentionMask[0] = 1;
+ tokenIds[0] = _tokenizer.BeginningOfSentenceId;
+ offsets[0] = (0, 0);
+
+ for (var i = 0; i < wordTokens.Count; i++)
+ {
+ var token = wordTokens[i];
+ var index = i + 1;
+ inputIds[index] = token.Id;
+ attentionMask[index] = 1;
+ tokenIds[index] = token.Id;
+ offsets[index] = ToCharOffsets(token.Offset, text.Length);
+ }
+
+ inputIds[sequenceLength - 1] = _tokenizer.EndOfSentenceId;
+ attentionMask[sequenceLength - 1] = 1;
+ tokenIds[sequenceLength - 1] = _tokenizer.EndOfSentenceId;
+ offsets[sequenceLength - 1] = (0, 0);
+
+ return new EncodedSequence(inputIds, attentionMask, tokenTypeIds, offsets, tokenIds, sequenceLength);
+ }
+
+ public bool IsSpecialToken(int tokenId) =>
+ _tokenizer is not null &&
+ (tokenId == _tokenizer.BeginningOfSentenceId ||
+ tokenId == _tokenizer.EndOfSentenceId ||
+ tokenId == _tokenizer.UnknownId);
+
+ private SentencePieceTokenizer? TryLoadTokenizer(string modelDirectory)
+ {
+ foreach (var fileName in new[] { "sentencepiece.bpe.model", "spiece.model", "tokenizer.model" })
+ {
+ var modelPath = OnnxAssetPathResolver.ResolveAssetPath(Path.Combine(modelDirectory, fileName));
+ if (!File.Exists(modelPath))
+ {
+ continue;
+ }
+
+ try
+ {
+ using var stream = File.OpenRead(modelPath);
+ return SentencePieceTokenizer.Create(stream, addBeginningOfSentence: false, addEndOfSentence: false);
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Failed to load SentencePiece tokenizer from {ModelPath}.", modelPath);
+ return null;
+ }
+ }
+
+ _logger.LogWarning(
+ "SentencePiece model not found in {ModelDirectory}. Expected sentencepiece.bpe.model or spiece.model.",
+ modelDirectory);
+ return null;
+ }
+
+ private static (int Start, int End) ToCharOffsets(Range offset, int textLength)
+ {
+ var (start, length) = offset.GetOffsetAndLength(textLength);
+ return (start, start + length);
+ }
+}
diff --git a/src/PiiRedaction.Infrastructure/Onnx/TamilOnnxNerRunner.cs b/src/PiiRedaction.Infrastructure/Onnx/TamilOnnxNerRunner.cs
new file mode 100644
index 0000000..54c4150
--- /dev/null
+++ b/src/PiiRedaction.Infrastructure/Onnx/TamilOnnxNerRunner.cs
@@ -0,0 +1,27 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using PiiRedaction.Core.Configuration;
+using PiiRedaction.Core.Detection;
+using PiiRedaction.Core.Models;
+
+namespace PiiRedaction.Infrastructure.Onnx;
+
+public sealed class TamilOnnxNerRunner : IOnnxNerModelRunner, IDisposable
+{
+ private readonly OnnxTokenClassifierRunner _runner;
+
+ public TamilOnnxNerRunner(IOptions options, ILogger logger)
+ {
+ var modelPath = OnnxAssetPathResolver.ResolveModelPath(options.Value.TamilOnnxModelPath);
+ var modelDirectory = Path.GetDirectoryName(modelPath) ?? Environment.CurrentDirectory;
+ var labels = OnnxAssetPathResolver.LoadLabels(modelDirectory);
+ var encoder = TokenClassifierEncoderFactory.Create(modelDirectory, logger);
+ _runner = new OnnxTokenClassifierRunner(modelPath, encoder, NerLabelConfig.Tamil, labels, logger);
+ }
+
+ public bool IsModelAvailable => _runner.IsAvailable;
+
+ public IReadOnlyList PredictEntities(string text) => _runner.PredictEntities(text);
+
+ public void Dispose() => _runner.Dispose();
+}
diff --git a/src/PiiRedaction.Infrastructure/Onnx/TokenClassifierEncoderFactory.cs b/src/PiiRedaction.Infrastructure/Onnx/TokenClassifierEncoderFactory.cs
new file mode 100644
index 0000000..3072fc5
--- /dev/null
+++ b/src/PiiRedaction.Infrastructure/Onnx/TokenClassifierEncoderFactory.cs
@@ -0,0 +1,40 @@
+using Microsoft.Extensions.Logging;
+
+namespace PiiRedaction.Infrastructure.Onnx;
+
+public static class TokenClassifierEncoderFactory
+{
+ private static readonly string[] SentencePieceFileNames =
+ ["sentencepiece.bpe.model", "spiece.model", "tokenizer.model"];
+
+ public static ITokenClassifierEncoder Create(string modelDirectory, ILogger logger)
+ {
+ var vocabPath = OnnxAssetPathResolver.ResolveAssetPath(Path.Combine(modelDirectory, "vocab.txt"));
+ if (File.Exists(vocabPath))
+ {
+ logger.LogInformation(
+ "Using WordPiece tokenizer (vocab.txt) from {ModelDirectory}.",
+ modelDirectory);
+ return new BertWordPieceEncoder(modelDirectory, logger);
+ }
+
+ foreach (var fileName in SentencePieceFileNames)
+ {
+ var sentencePiecePath = OnnxAssetPathResolver.ResolveAssetPath(
+ Path.Combine(modelDirectory, fileName));
+ if (File.Exists(sentencePiecePath))
+ {
+ logger.LogInformation(
+ "Using SentencePiece tokenizer ({FileName}) from {ModelDirectory}.",
+ fileName,
+ modelDirectory);
+ return new SentencePieceEncoder(modelDirectory, logger);
+ }
+ }
+
+ logger.LogWarning(
+ "No tokenizer assets found in {ModelDirectory}. Expected vocab.txt or a SentencePiece model file.",
+ modelDirectory);
+ return new BertWordPieceEncoder(modelDirectory, logger);
+ }
+}
diff --git a/src/PiiRedaction.Infrastructure/PiiRedaction.Infrastructure.csproj b/src/PiiRedaction.Infrastructure/PiiRedaction.Infrastructure.csproj
index 8b1696c..ad4a12f 100644
--- a/src/PiiRedaction.Infrastructure/PiiRedaction.Infrastructure.csproj
+++ b/src/PiiRedaction.Infrastructure/PiiRedaction.Infrastructure.csproj
@@ -19,4 +19,8 @@
enable
+
+
+
+
diff --git a/src/PiiRedaction.TestHarness.Wpf/App.xaml b/src/PiiRedaction.TestHarness.Wpf/App.xaml
new file mode 100644
index 0000000..77fe965
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/App.xaml
@@ -0,0 +1,17 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/PiiRedaction.TestHarness.Wpf/App.xaml.cs b/src/PiiRedaction.TestHarness.Wpf/App.xaml.cs
new file mode 100644
index 0000000..bdb531d
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/App.xaml.cs
@@ -0,0 +1,57 @@
+using System.IO;
+using System.Windows;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using PiiRedaction.TestHarness.Wpf.DependencyInjection;
+using PiiRedaction.TestHarness.Wpf.Services;
+using PiiRedaction.TestHarness.Wpf.ViewModels;
+
+namespace PiiRedaction.TestHarness.Wpf;
+
+public partial class App : Application
+{
+ private IHost? _host;
+
+ protected override async void OnStartup(StartupEventArgs e)
+ {
+ base.OnStartup(e);
+
+ _host = Host.CreateDefaultBuilder()
+ .ConfigureAppConfiguration((_, configuration) =>
+ {
+ configuration.SetBasePath(AppContext.BaseDirectory);
+ configuration.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true);
+ configuration.AddEnvironmentVariables();
+ })
+ .ConfigureServices((context, services) =>
+ {
+ services.AddPiiRedactionServices(context.Configuration);
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ })
+ .Build();
+
+ Directory.SetCurrentDirectory(AppContext.BaseDirectory);
+
+ await _host.StartAsync().ConfigureAwait(true);
+
+ var mainWindow = _host.Services.GetRequiredService();
+ mainWindow.Show();
+ }
+
+ protected override async void OnExit(ExitEventArgs e)
+ {
+ if (_host is not null)
+ {
+ await _host.StopAsync().ConfigureAwait(true);
+ _host.Dispose();
+ }
+
+ base.OnExit(e);
+ }
+}
diff --git a/src/PiiRedaction.TestHarness.Wpf/Converters/ValueConverters.cs b/src/PiiRedaction.TestHarness.Wpf/Converters/ValueConverters.cs
new file mode 100644
index 0000000..cf7d4af
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/Converters/ValueConverters.cs
@@ -0,0 +1,71 @@
+using System.Globalization;
+using System.Windows;
+using System.Windows.Data;
+using System.Windows.Media;
+using PiiRedaction.Core.Detection;
+
+namespace PiiRedaction.TestHarness.Wpf.Converters;
+
+public sealed class ScriptCompositionToBrushConverter : IValueConverter
+{
+ public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture)
+ {
+ if (value is not ScriptComposition composition)
+ {
+ return Brushes.Gray;
+ }
+
+ return composition switch
+ {
+ ScriptComposition.LatinOnly => new SolidColorBrush(Color.FromRgb(37, 99, 235)),
+ ScriptComposition.TamilOnly => new SolidColorBrush(Color.FromRgb(124, 58, 237)),
+ ScriptComposition.Mixed => new SolidColorBrush(Color.FromRgb(217, 119, 6)),
+ ScriptComposition.NoLetters => new SolidColorBrush(Color.FromRgb(107, 114, 128)),
+ _ => Brushes.Gray
+ };
+ }
+
+ public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) =>
+ throw new NotSupportedException();
+}
+
+public sealed class BoolToVisibilityConverter : IValueConverter
+{
+ public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture) =>
+ value is true ? Visibility.Visible : Visibility.Collapsed;
+
+ public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) =>
+ throw new NotSupportedException();
+}
+
+public sealed class StringNotEmptyToVisibilityConverter : IValueConverter
+{
+ public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture) =>
+ value is string text && !string.IsNullOrWhiteSpace(text)
+ ? Visibility.Visible
+ : Visibility.Collapsed;
+
+ public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) =>
+ throw new NotSupportedException();
+}
+
+public sealed class PassFailBrushConverter : IValueConverter
+{
+ public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture)
+ {
+ if (value is true)
+ {
+ return new SolidColorBrush(Color.FromRgb(22, 163, 74));
+ }
+
+ if (value is false)
+ {
+ return new SolidColorBrush(Color.FromRgb(220, 38, 38));
+ }
+
+ return Brushes.Gray;
+ }
+
+ public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) =>
+ throw new NotSupportedException();
+}
diff --git a/src/PiiRedaction.TestHarness.Wpf/DependencyInjection/ServiceCollectionExtensions.cs b/src/PiiRedaction.TestHarness.Wpf/DependencyInjection/ServiceCollectionExtensions.cs
new file mode 100644
index 0000000..24fd93d
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/DependencyInjection/ServiceCollectionExtensions.cs
@@ -0,0 +1,42 @@
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.AI;
+using PiiRedaction.Core.Abstractions;
+using PiiRedaction.Core.Configuration;
+using PiiRedaction.Core.Detection;
+using PiiRedaction.Core.Redaction;
+using PiiRedaction.Core.Sanitization;
+using PiiRedaction.Infrastructure.Llm;
+using PiiRedaction.Infrastructure.Onnx;
+
+namespace PiiRedaction.TestHarness.Wpf.DependencyInjection;
+
+public static class ServiceCollectionExtensions
+{
+ public static IServiceCollection AddPiiRedactionServices(this IServiceCollection services, IConfiguration configuration)
+ {
+ services.Configure(configuration.GetSection(PiiRedactionOptions.SectionName));
+
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+
+ services.AddSingleton(provider => new CompositePiiDetector(
+ [
+ provider.GetRequiredService(),
+ provider.GetRequiredService(),
+ provider.GetRequiredService()
+ ]));
+
+ services.AddSingleton();
+ services.AddSingleton();
+
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+
+ return services;
+ }
+}
diff --git a/src/PiiRedaction.TestHarness.Wpf/MainWindow.xaml b/src/PiiRedaction.TestHarness.Wpf/MainWindow.xaml
new file mode 100644
index 0000000..66c80e8
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/MainWindow.xaml
@@ -0,0 +1,311 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/PiiRedaction.TestHarness.Wpf/MainWindow.xaml.cs b/src/PiiRedaction.TestHarness.Wpf/MainWindow.xaml.cs
new file mode 100644
index 0000000..8a80899
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/MainWindow.xaml.cs
@@ -0,0 +1,13 @@
+using System.Windows;
+using PiiRedaction.TestHarness.Wpf.ViewModels;
+
+namespace PiiRedaction.TestHarness.Wpf;
+
+public partial class MainWindow : Window
+{
+ public MainWindow(MainViewModel viewModel)
+ {
+ InitializeComponent();
+ DataContext = viewModel;
+ }
+}
diff --git a/src/PiiRedaction.TestHarness.Wpf/Models/BatchRunResult.cs b/src/PiiRedaction.TestHarness.Wpf/Models/BatchRunResult.cs
new file mode 100644
index 0000000..f210689
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/Models/BatchRunResult.cs
@@ -0,0 +1,23 @@
+namespace PiiRedaction.TestHarness.Wpf.Models;
+
+public sealed record RedactionOutcome(
+ string OriginalPrompt,
+ string SanitizedPrompt,
+ IReadOnlyList DetectedEntities,
+ IReadOnlyList Placeholders,
+ long ElapsedMilliseconds,
+ bool HasLeak);
+
+public sealed record BatchScenarioResult(
+ TestPromptScenario Scenario,
+ bool Passed,
+ string? FailureReason,
+ int EntityCount,
+ long ElapsedMilliseconds);
+
+public sealed record BatchRunSummary(
+ int Total,
+ int Passed,
+ int Failed,
+ IReadOnlyList Results,
+ long TotalElapsedMilliseconds);
diff --git a/src/PiiRedaction.TestHarness.Wpf/Models/ModelStatusSnapshot.cs b/src/PiiRedaction.TestHarness.Wpf/Models/ModelStatusSnapshot.cs
new file mode 100644
index 0000000..6713276
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/Models/ModelStatusSnapshot.cs
@@ -0,0 +1,21 @@
+namespace PiiRedaction.TestHarness.Wpf.Models;
+
+public enum ModelAvailability
+{
+ Ready,
+ Missing,
+ Disabled
+}
+
+public sealed record NerModelStatus(
+ string ModelName,
+ ModelAvailability Availability,
+ string Path);
+
+public sealed record ModelStatusSnapshot(
+ NerModelStatus English,
+ NerModelStatus Tamil)
+{
+ public string Summary =>
+ $"English NER: {English.Availability} | Tamil NER: {Tamil.Availability}";
+}
diff --git a/src/PiiRedaction.TestHarness.Wpf/Models/RedactionDisplayModel.cs b/src/PiiRedaction.TestHarness.Wpf/Models/RedactionDisplayModel.cs
new file mode 100644
index 0000000..e34b9ab
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/Models/RedactionDisplayModel.cs
@@ -0,0 +1,29 @@
+using PiiRedaction.Core.Models;
+
+namespace PiiRedaction.TestHarness.Wpf.Models;
+
+public sealed class RedactionDisplayModel
+{
+ public required string Type { get; init; }
+ public required string Value { get; init; }
+ public required string Source { get; init; }
+ public int StartIndex { get; init; }
+ public int Length { get; init; }
+ public string? Confidence { get; init; }
+
+ public static RedactionDisplayModel FromEntity(PiiEntity entity) => new()
+ {
+ Type = entity.Type.ToString(),
+ Value = entity.Value,
+ Source = entity.Source.ToString(),
+ StartIndex = entity.StartIndex,
+ Length = entity.Length,
+ Confidence = entity.Confidence?.ToString("F2")
+ };
+}
+
+public sealed class PlaceholderDisplayModel
+{
+ public required string Placeholder { get; init; }
+ public required string OriginalValue { get; init; }
+}
diff --git a/src/PiiRedaction.TestHarness.Wpf/Models/TestPromptScenario.cs b/src/PiiRedaction.TestHarness.Wpf/Models/TestPromptScenario.cs
new file mode 100644
index 0000000..ddd10bc
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/Models/TestPromptScenario.cs
@@ -0,0 +1,18 @@
+namespace PiiRedaction.TestHarness.Wpf.Models;
+
+public enum PromptLanguage
+{
+ English,
+ Tamil,
+ Mixed,
+ Tanglish
+}
+
+public sealed record TestPromptScenario(
+ string Id,
+ string Name,
+ PromptLanguage Language,
+ string Category,
+ string Description,
+ string Prompt,
+ bool ExpectDetections);
diff --git a/src/PiiRedaction.TestHarness.Wpf/PiiRedaction.TestHarness.Wpf.csproj b/src/PiiRedaction.TestHarness.Wpf/PiiRedaction.TestHarness.Wpf.csproj
new file mode 100644
index 0000000..daf60bc
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/PiiRedaction.TestHarness.Wpf.csproj
@@ -0,0 +1,33 @@
+
+
+
+ WinExe
+ net10.0-windows
+ enable
+ enable
+ true
+
+ PiiRedaction.TestHarness.Wpf
+ PiiRedaction.TestHarness.Wpf
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ PreserveNewest
+
+
+
+
diff --git a/src/PiiRedaction.TestHarness.Wpf/Resources/Styles.xaml b/src/PiiRedaction.TestHarness.Wpf/Resources/Styles.xaml
new file mode 100644
index 0000000..0d8b908
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/Resources/Styles.xaml
@@ -0,0 +1,68 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/PiiRedaction.TestHarness.Wpf/Services/IModelStatusService.cs b/src/PiiRedaction.TestHarness.Wpf/Services/IModelStatusService.cs
new file mode 100644
index 0000000..cd15e9f
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/Services/IModelStatusService.cs
@@ -0,0 +1,8 @@
+using PiiRedaction.TestHarness.Wpf.Models;
+
+namespace PiiRedaction.TestHarness.Wpf.Services;
+
+public interface IModelStatusService
+{
+ ModelStatusSnapshot GetStatus();
+}
diff --git a/src/PiiRedaction.TestHarness.Wpf/Services/IRedactionAppService.cs b/src/PiiRedaction.TestHarness.Wpf/Services/IRedactionAppService.cs
new file mode 100644
index 0000000..0410fe6
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/Services/IRedactionAppService.cs
@@ -0,0 +1,17 @@
+using PiiRedaction.TestHarness.Wpf.Models;
+
+namespace PiiRedaction.TestHarness.Wpf.Services;
+
+public interface IRedactionAppService
+{
+ Task RedactAsync(string prompt, CancellationToken cancellationToken = default);
+
+ Task SendToMockLlmAsync(string sanitizedPrompt, CancellationToken cancellationToken = default);
+
+ BatchScenarioResult EvaluateScenario(TestPromptScenario scenario, RedactionOutcome outcome);
+
+ Task RunAllScenariosAsync(
+ IReadOnlyList scenarios,
+ IProgress<(int Current, int Total, string Name)>? progress = null,
+ CancellationToken cancellationToken = default);
+}
diff --git a/src/PiiRedaction.TestHarness.Wpf/Services/IScriptAnalysisService.cs b/src/PiiRedaction.TestHarness.Wpf/Services/IScriptAnalysisService.cs
new file mode 100644
index 0000000..334378e
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/Services/IScriptAnalysisService.cs
@@ -0,0 +1,8 @@
+using PiiRedaction.Core.Detection;
+
+namespace PiiRedaction.TestHarness.Wpf.Services;
+
+public interface IScriptAnalysisService
+{
+ ScriptComposition GetComposition(string text);
+}
diff --git a/src/PiiRedaction.TestHarness.Wpf/Services/ITestPromptCatalog.cs b/src/PiiRedaction.TestHarness.Wpf/Services/ITestPromptCatalog.cs
new file mode 100644
index 0000000..43bce19
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/Services/ITestPromptCatalog.cs
@@ -0,0 +1,8 @@
+using PiiRedaction.TestHarness.Wpf.Models;
+
+namespace PiiRedaction.TestHarness.Wpf.Services;
+
+public interface ITestPromptCatalog
+{
+ IReadOnlyList All { get; }
+}
diff --git a/src/PiiRedaction.TestHarness.Wpf/Services/ModelStatusService.cs b/src/PiiRedaction.TestHarness.Wpf/Services/ModelStatusService.cs
new file mode 100644
index 0000000..f798a0a
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/Services/ModelStatusService.cs
@@ -0,0 +1,46 @@
+using Microsoft.Extensions.Options;
+using PiiRedaction.Core.Configuration;
+using PiiRedaction.Infrastructure.Onnx;
+using PiiRedaction.TestHarness.Wpf.Models;
+
+namespace PiiRedaction.TestHarness.Wpf.Services;
+
+public sealed class ModelStatusService : IModelStatusService
+{
+ private readonly EnglishOnnxNerRunner _englishRunner;
+ private readonly TamilOnnxNerRunner _tamilRunner;
+ private readonly PiiRedactionOptions _options;
+
+ public ModelStatusService(
+ EnglishOnnxNerRunner englishRunner,
+ TamilOnnxNerRunner tamilRunner,
+ IOptions options)
+ {
+ _englishRunner = englishRunner;
+ _tamilRunner = tamilRunner;
+ _options = options.Value;
+ }
+
+ public ModelStatusSnapshot GetStatus()
+ {
+ var englishPath = OnnxAssetPathResolver.ResolveModelPath(
+ _options.EnglishOnnxModelPath,
+ _options.OnnxModelPath);
+
+ var tamilPath = OnnxAssetPathResolver.ResolveModelPath(_options.TamilOnnxModelPath);
+
+ var englishAvailability = _englishRunner.IsModelAvailable
+ ? ModelAvailability.Ready
+ : ModelAvailability.Missing;
+
+ var tamilAvailability = !_options.EnableTamilNer
+ ? ModelAvailability.Disabled
+ : _tamilRunner.IsModelAvailable
+ ? ModelAvailability.Ready
+ : ModelAvailability.Missing;
+
+ return new ModelStatusSnapshot(
+ new NerModelStatus("English", englishAvailability, englishPath),
+ new NerModelStatus("Tamil", tamilAvailability, tamilPath));
+ }
+}
diff --git a/src/PiiRedaction.TestHarness.Wpf/Services/RedactionAppService.cs b/src/PiiRedaction.TestHarness.Wpf/Services/RedactionAppService.cs
new file mode 100644
index 0000000..a45a687
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/Services/RedactionAppService.cs
@@ -0,0 +1,116 @@
+using System.Diagnostics;
+using PiiRedaction.Core.Abstractions;
+using PiiRedaction.Core.Models;
+using PiiRedaction.TestHarness.Wpf.Models;
+
+namespace PiiRedaction.TestHarness.Wpf.Services;
+
+public sealed class RedactionAppService : IRedactionAppService
+{
+ private readonly IPromptSanitizer _sanitizer;
+ private readonly ILlmPromptService _llmPromptService;
+
+ public RedactionAppService(IPromptSanitizer sanitizer, ILlmPromptService llmPromptService)
+ {
+ _sanitizer = sanitizer;
+ _llmPromptService = llmPromptService;
+ }
+
+ public Task RedactAsync(string prompt, CancellationToken cancellationToken = default) =>
+ Task.Run(() =>
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+
+ var stopwatch = Stopwatch.StartNew();
+ var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+ stopwatch.Stop();
+
+ return MapOutcome(result, stopwatch.ElapsedMilliseconds);
+ }, cancellationToken);
+
+ public Task SendToMockLlmAsync(string sanitizedPrompt, CancellationToken cancellationToken = default) =>
+ _llmPromptService.SendPromptAsync(sanitizedPrompt, cancellationToken);
+
+ public BatchScenarioResult EvaluateScenario(TestPromptScenario scenario, RedactionOutcome outcome)
+ {
+ var entityCount = outcome.DetectedEntities.Count;
+ string? failureReason = null;
+
+ if (scenario.ExpectDetections && entityCount == 0)
+ {
+ failureReason = "Expected at least one PII detection but found none.";
+ }
+ else if (!scenario.ExpectDetections && entityCount > 0)
+ {
+ failureReason = $"Expected no detections but found {entityCount}.";
+ }
+ else if (outcome.HasLeak)
+ {
+ failureReason = "Detected PII value still present in sanitized output.";
+ }
+
+ return new BatchScenarioResult(
+ scenario,
+ failureReason is null,
+ failureReason,
+ entityCount,
+ outcome.ElapsedMilliseconds);
+ }
+
+ public async Task RunAllScenariosAsync(
+ IReadOnlyList scenarios,
+ IProgress<(int Current, int Total, string Name)>? progress = null,
+ CancellationToken cancellationToken = default)
+ {
+ var results = new List(scenarios.Count);
+ var totalStopwatch = Stopwatch.StartNew();
+
+ for (var index = 0; index < scenarios.Count; index++)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+
+ var scenario = scenarios[index];
+ progress?.Report((index + 1, scenarios.Count, scenario.Name));
+
+ var outcome = await RedactAsync(scenario.Prompt, cancellationToken).ConfigureAwait(false);
+ results.Add(EvaluateScenario(scenario, outcome));
+ }
+
+ totalStopwatch.Stop();
+
+ var passed = results.Count(result => result.Passed);
+ return new BatchRunSummary(
+ scenarios.Count,
+ passed,
+ scenarios.Count - passed,
+ results,
+ totalStopwatch.ElapsedMilliseconds);
+ }
+
+ private static RedactionOutcome MapOutcome(SanitizationResult result, long elapsedMilliseconds)
+ {
+ var entities = result.DetectedEntities
+ .Select(RedactionDisplayModel.FromEntity)
+ .ToList();
+
+ var placeholders = result.Redaction.PlaceholderMap
+ .Select(pair => new PlaceholderDisplayModel
+ {
+ Placeholder = pair.Key,
+ OriginalValue = pair.Value
+ })
+ .ToList();
+
+ var hasLeak = result.DetectedEntities.Any(entity =>
+ !string.IsNullOrWhiteSpace(entity.Value) &&
+ result.SanitizedPrompt.Contains(entity.Value, StringComparison.Ordinal));
+
+ return new RedactionOutcome(
+ result.OriginalPrompt,
+ result.SanitizedPrompt,
+ entities,
+ placeholders,
+ elapsedMilliseconds,
+ hasLeak);
+ }
+}
diff --git a/src/PiiRedaction.TestHarness.Wpf/Services/ScriptAnalysisService.cs b/src/PiiRedaction.TestHarness.Wpf/Services/ScriptAnalysisService.cs
new file mode 100644
index 0000000..263eae6
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/Services/ScriptAnalysisService.cs
@@ -0,0 +1,13 @@
+using PiiRedaction.Core.Detection;
+
+namespace PiiRedaction.TestHarness.Wpf.Services;
+
+public sealed class ScriptAnalysisService : IScriptAnalysisService
+{
+ private readonly ScriptRouter _scriptRouter = new();
+
+ public ScriptComposition GetComposition(string text) =>
+ string.IsNullOrWhiteSpace(text)
+ ? ScriptComposition.NoLetters
+ : _scriptRouter.GetComposition(text);
+}
diff --git a/src/PiiRedaction.TestHarness.Wpf/Services/TestPromptCatalog.cs b/src/PiiRedaction.TestHarness.Wpf/Services/TestPromptCatalog.cs
new file mode 100644
index 0000000..0085405
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/Services/TestPromptCatalog.cs
@@ -0,0 +1,194 @@
+using PiiRedaction.TestHarness.Wpf.Models;
+
+namespace PiiRedaction.TestHarness.Wpf.Services;
+
+public sealed class TestPromptCatalog : ITestPromptCatalog
+{
+ public IReadOnlyList All { get; } =
+ [
+ Scenario(
+ "FullFinancialWithCustomer",
+ PromptLanguage.English,
+ "NER + Regex + Domain",
+ "Canonical demo: person name plus email, phone, loan number, and PAN.",
+ "Customer Ravi Kumar with email ravi.kumar@gmail.com and phone 9876543210 has LoanNumber LN-456789 and PAN ABCDE1234F. Please summarize this customer issue.",
+ expectDetections: true),
+
+ Scenario(
+ "CustomerNameOnly",
+ PromptLanguage.English,
+ "NER",
+ "Person name detected via ONNX NER after 'Customer' keyword.",
+ "Customer Anita Sharma reported unauthorized transactions on her savings account.",
+ expectDetections: true),
+
+ Scenario(
+ "MrTitlePerson",
+ PromptLanguage.English,
+ "NER",
+ "Person detected via ONNX NER (title prefix Mr.).",
+ "Mr. John Smith called about a duplicate debit on 15 March.",
+ expectDetections: true),
+
+ Scenario(
+ "MrsTitlePerson",
+ PromptLanguage.English,
+ "NER",
+ "Person detected via ONNX NER (title prefix Mrs.).",
+ "Mrs. Lakshmi Reddy requested a callback regarding LN-112233.",
+ expectDetections: true),
+
+ Scenario(
+ "DrTitlePerson",
+ PromptLanguage.English,
+ "NER",
+ "Person detected via ONNX NER (title prefix Dr).",
+ "Dr. Jane Doe escalated a complaint about delayed loan disbursement.",
+ expectDetections: true),
+
+ Scenario(
+ "TwoCustomersInOnePrompt",
+ PromptLanguage.English,
+ "NER",
+ "Two distinct person names in the same prompt.",
+ "Customer Ravi Kumar and Customer Priya Nair disputed the same charge.",
+ expectDetections: true),
+
+ Scenario(
+ "PersonWithDomainIds",
+ PromptLanguage.English,
+ "NER + Domain",
+ "Person name combined with business identifiers.",
+ "Customer Meera Iyer holds CID-7070 and account ACC-606060 for verification.",
+ expectDetections: true),
+
+ Scenario(
+ "PersonWithEmailNoPhone",
+ PromptLanguage.English,
+ "NER + Regex",
+ "Person and email without phone number.",
+ "Customer Arjun Mehta wrote from arjun.mehta@company.in about KYC renewal.",
+ expectDetections: true),
+
+ Scenario(
+ "AllRegexTypes",
+ PromptLanguage.English,
+ "Regex",
+ "Email, phone, PAN, Aadhaar, and credit card in one prompt.",
+ "Email a@b.co phone 9001234567 PAN ABCDE1234F aadhaar 1234 5678 9012 card 4111-1111-1111-1111.",
+ expectDetections: true),
+
+ Scenario(
+ "AllDomainIds",
+ PromptLanguage.English,
+ "Domain",
+ "Loan number, customer ID, and account number together.",
+ "Please verify LN-100200 for CustomerId CID-3000 on AccountNumber ACC-400500.",
+ expectDetections: true),
+
+ Scenario(
+ "TamilCustomerNameOnly",
+ PromptLanguage.Tamil,
+ "NER (Tamil)",
+ "Tamil script person name detected via Tamil ONNX NER.",
+ "வாடிக்கையாளர் ராஜேஷ் குமார் சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்.",
+ expectDetections: true),
+
+ Scenario(
+ "TamilWithPhonePan",
+ PromptLanguage.Tamil,
+ "NER (Tamil) + Regex",
+ "Tamil script person plus phone and PAN (regex).",
+ "வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210 PAN ABCDE1234F.",
+ expectDetections: true),
+
+ Scenario(
+ "TanglishCustomer",
+ PromptLanguage.Tanglish,
+ "NER (English/Tanglish)",
+ "Latin-script Tanglish person name via English ONNX NER.",
+ "Customer Senthil phone 9876543210 reported a failed UPI transfer.",
+ expectDetections: true),
+
+ Scenario(
+ "MixedTamilEnglish",
+ PromptLanguage.Mixed,
+ "NER (Mixed)",
+ "Code-mixed Tamil and English — both script routers may contribute person spans.",
+ "வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge.",
+ expectDetections: true),
+
+ Scenario(
+ "TamilFullFinancial",
+ PromptLanguage.Tamil,
+ "NER (Tamil) + Regex + Domain",
+ "Tamil person with email, phone, loan number, and PAN (canonical demo in Tamil).",
+ "வாடிக்கையாளர் ராஜேஷ் குமார் மின்னஞ்சல் ravi.kumar@gmail.com தொலைபேசி 9876543210 LoanNumber LN-456789 PAN ABCDE1234F. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்.",
+ expectDetections: true),
+
+ Scenario(
+ "NoPiiCleanTicket",
+ PromptLanguage.English,
+ "Negative",
+ "No PII — prompt passes through unchanged.",
+ "What is the status of ticket TKT-99887 and when will the API maintenance end?",
+ expectDetections: false),
+
+ Scenario(
+ "NegativeWorkflowQuestion",
+ PromptLanguage.English,
+ "Negative",
+ "General workflow question with no regulated identifiers.",
+ "Summarize the retail loan approval workflow and typical SLA milestones.",
+ expectDetections: false),
+
+ Scenario(
+ "EdgePhoneOnly",
+ PromptLanguage.English,
+ "Edge + Regex",
+ "Digits-only phone without a person name.",
+ "Callback requested on 9123456780 regarding branch hours.",
+ expectDetections: true),
+
+ Scenario(
+ "EdgeLongMixed",
+ PromptLanguage.Mixed,
+ "Edge + NER (Mixed)",
+ "Longer mixed-language prompt with person and phone.",
+ "வாடிக்கையாளர் Priya Nair called from Chennai about a delayed NEFT transfer. She asked whether LoanNumber LN-909090 is linked to account ACC-808080 and wants an email confirmation sent to priya.nair@example.com on phone 9988776655.",
+ expectDetections: true),
+
+ Scenario(
+ "LeakCheckNestedEmail",
+ PromptLanguage.English,
+ "LeakCheck + Regex",
+ "Email embedded in a sentence — placeholders must fully replace the address.",
+ "Please forward the statement for customer.support@banking.example to the operations desk.",
+ expectDetections: true),
+
+ Scenario(
+ "TamilEdgePunctuation",
+ PromptLanguage.Tamil,
+ "TamilEdge + NER (Tamil)",
+ "Tamil name surrounded by punctuation and Tamil numerals.",
+ "வாடிக்கையாளர் (ராஜேஷ் குமார்) — தொலைபேசி ௯௮௭௬௫௪௩௨௧௦ — உதவி தேவை.",
+ expectDetections: true),
+
+ Scenario(
+ "TanglishLatinInTamilSentence",
+ PromptLanguage.Tanglish,
+ "Tanglish + NER",
+ "Latin person name inside otherwise Tamil context.",
+ "வாடிக்கையாளர் Arun Kumar அவர்களின் KYC ஆவணம் நிலுவையில் உள்ளது.",
+ expectDetections: true)
+ ];
+
+ private static TestPromptScenario Scenario(
+ string name,
+ PromptLanguage language,
+ string category,
+ string description,
+ string prompt,
+ bool expectDetections) =>
+ new(name, name, language, category, description, prompt, expectDetections);
+}
diff --git a/src/PiiRedaction.TestHarness.Wpf/ViewModels/MainViewModel.cs b/src/PiiRedaction.TestHarness.Wpf/ViewModels/MainViewModel.cs
new file mode 100644
index 0000000..2ec5699
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/ViewModels/MainViewModel.cs
@@ -0,0 +1,337 @@
+using System.Collections.ObjectModel;
+using System.ComponentModel;
+using System.Windows;
+using System.Windows.Data;
+using CommunityToolkit.Mvvm.ComponentModel;
+using CommunityToolkit.Mvvm.Input;
+using PiiRedaction.Core.Detection;
+using PiiRedaction.TestHarness.Wpf.Models;
+using PiiRedaction.TestHarness.Wpf.Services;
+
+namespace PiiRedaction.TestHarness.Wpf.ViewModels;
+
+public partial class MainViewModel : ObservableObject
+{
+ private readonly IRedactionAppService _redactionAppService;
+ private readonly ITestPromptCatalog _promptCatalog;
+ private readonly IScriptAnalysisService _scriptAnalysisService;
+ private readonly IModelStatusService _modelStatusService;
+
+ public MainViewModel(
+ IRedactionAppService redactionAppService,
+ ITestPromptCatalog promptCatalog,
+ IScriptAnalysisService scriptAnalysisService,
+ IModelStatusService modelStatusService)
+ {
+ _redactionAppService = redactionAppService;
+ _promptCatalog = promptCatalog;
+ _scriptAnalysisService = scriptAnalysisService;
+ _modelStatusService = modelStatusService;
+
+ PromptItems = new ObservableCollection(
+ _promptCatalog.All.Select(scenario => new TestPromptItemViewModel(scenario)));
+
+ PromptsView = CollectionViewSource.GetDefaultView(PromptItems);
+ PromptsView.GroupDescriptions.Add(new PropertyGroupDescription(nameof(TestPromptItemViewModel.Language)));
+ PromptsView.SortDescriptions.Add(new SortDescription(nameof(TestPromptItemViewModel.Name), ListSortDirection.Ascending));
+ PromptsView.Filter = FilterPrompt;
+
+ DetectedEntities = [];
+ PlaceholderMap = [];
+ BatchResults = [];
+
+ RefreshModelStatus();
+ UpdateScriptComposition();
+ }
+
+ public ICollectionView PromptsView { get; }
+
+ public ObservableCollection PromptItems { get; }
+
+ public ObservableCollection DetectedEntities { get; }
+
+ public ObservableCollection PlaceholderMap { get; }
+
+ public ObservableCollection BatchResults { get; }
+
+ [ObservableProperty]
+ private string _inputPrompt = string.Empty;
+
+ [ObservableProperty]
+ private string _sanitizedOutput = string.Empty;
+
+ [ObservableProperty]
+ private string _originalPrompt = string.Empty;
+
+ [ObservableProperty]
+ private string _mockLlmResponse = string.Empty;
+
+ [ObservableProperty]
+ private TestPromptItemViewModel? _selectedPrompt;
+
+ [ObservableProperty]
+ private ScriptComposition _scriptComposition = ScriptComposition.NoLetters;
+
+ [ObservableProperty]
+ private long _elapsedMilliseconds;
+
+ [ObservableProperty]
+ private int _entityCount;
+
+ [ObservableProperty]
+ private string _statusMessage = "Ready";
+
+ [ObservableProperty]
+ private bool _isBusy;
+
+ [ObservableProperty]
+ private string _modelStatus = string.Empty;
+
+ [ObservableProperty]
+ private bool _leakWarning;
+
+ [ObservableProperty]
+ private string _promptFilter = string.Empty;
+
+ [ObservableProperty]
+ private string _batchSummary = string.Empty;
+
+ [ObservableProperty]
+ private bool _isBatchExpanded;
+
+ partial void OnInputPromptChanged(string value)
+ {
+ UpdateScriptComposition();
+ RedactCommand.NotifyCanExecuteChanged();
+ }
+
+ partial void OnSelectedPromptChanged(TestPromptItemViewModel? value)
+ {
+ if (value is null)
+ {
+ return;
+ }
+
+ ClearRedactionResults();
+ InputPrompt = value.Prompt;
+ StatusMessage = $"Loaded prompt: {value.Name}";
+ }
+
+ [RelayCommand]
+ private void Clear()
+ {
+ InputPrompt = string.Empty;
+ SelectedPrompt = null;
+ ClearRedactionResults();
+ BatchResults.Clear();
+ BatchSummary = string.Empty;
+ IsBatchExpanded = false;
+ StatusMessage = "Cleared.";
+ UpdateScriptComposition();
+ }
+
+ private void ClearRedactionResults()
+ {
+ SanitizedOutput = string.Empty;
+ OriginalPrompt = string.Empty;
+ MockLlmResponse = string.Empty;
+ DetectedEntities.Clear();
+ PlaceholderMap.Clear();
+ LeakWarning = false;
+ EntityCount = 0;
+ ElapsedMilliseconds = 0;
+ SendToMockLlmCommand.NotifyCanExecuteChanged();
+ }
+
+ [RelayCommand(CanExecute = nameof(CanRedact))]
+ private async Task RedactAsync(CancellationToken cancellationToken)
+ {
+ if (string.IsNullOrWhiteSpace(InputPrompt))
+ {
+ return;
+ }
+
+ try
+ {
+ IsBusy = true;
+ StatusMessage = "Redacting...";
+
+ var outcome = await _redactionAppService.RedactAsync(InputPrompt, cancellationToken)
+ .ConfigureAwait(true);
+
+ ApplyOutcome(outcome);
+ StatusMessage = $"Redaction complete in {outcome.ElapsedMilliseconds} ms.";
+ }
+ catch (OperationCanceledException)
+ {
+ StatusMessage = "Redaction cancelled.";
+ }
+ catch (Exception ex)
+ {
+ StatusMessage = $"Redaction failed: {ex.Message}";
+ }
+ finally
+ {
+ IsBusy = false;
+ RedactCommand.NotifyCanExecuteChanged();
+ RunAllScenariosCommand.NotifyCanExecuteChanged();
+ SendToMockLlmCommand.NotifyCanExecuteChanged();
+ }
+ }
+
+ private bool CanRedact() => !IsBusy && !string.IsNullOrWhiteSpace(InputPrompt);
+
+ [RelayCommand]
+ private void CopySanitized()
+ {
+ if (string.IsNullOrWhiteSpace(SanitizedOutput))
+ {
+ StatusMessage = "Nothing to copy.";
+ return;
+ }
+
+ Clipboard.SetText(SanitizedOutput);
+ StatusMessage = "Sanitized output copied to clipboard.";
+ }
+
+ [RelayCommand(CanExecute = nameof(CanSendToMockLlm))]
+ private async Task SendToMockLlmAsync(CancellationToken cancellationToken)
+ {
+ if (string.IsNullOrWhiteSpace(SanitizedOutput))
+ {
+ return;
+ }
+
+ try
+ {
+ IsBusy = true;
+ StatusMessage = "Sending to mock LLM...";
+
+ MockLlmResponse = await _redactionAppService
+ .SendToMockLlmAsync(SanitizedOutput, cancellationToken)
+ .ConfigureAwait(true);
+
+ StatusMessage = "Mock LLM response received.";
+ }
+ catch (OperationCanceledException)
+ {
+ StatusMessage = "Mock LLM call cancelled.";
+ }
+ catch (Exception ex)
+ {
+ StatusMessage = $"Mock LLM call failed: {ex.Message}";
+ }
+ finally
+ {
+ IsBusy = false;
+ RedactCommand.NotifyCanExecuteChanged();
+ RunAllScenariosCommand.NotifyCanExecuteChanged();
+ SendToMockLlmCommand.NotifyCanExecuteChanged();
+ }
+ }
+
+ private bool CanSendToMockLlm() => !IsBusy && !string.IsNullOrWhiteSpace(SanitizedOutput);
+
+ [RelayCommand(CanExecute = nameof(CanRunBatch))]
+ private async Task RunAllScenariosAsync(CancellationToken cancellationToken)
+ {
+ try
+ {
+ IsBusy = true;
+ IsBatchExpanded = true;
+ BatchResults.Clear();
+ BatchSummary = "Running batch validation...";
+ StatusMessage = "Running all scenarios...";
+
+ var progress = new Progress<(int Current, int Total, string Name)>(report =>
+ {
+ StatusMessage = $"Batch {report.Current}/{report.Total}: {report.Name}";
+ });
+
+ var summary = await _redactionAppService
+ .RunAllScenariosAsync(_promptCatalog.All, progress, cancellationToken)
+ .ConfigureAwait(true);
+
+ BatchResults.Clear();
+ foreach (var result in summary.Results)
+ {
+ BatchResults.Add(result);
+ }
+
+ BatchSummary =
+ $"{summary.Passed}/{summary.Total} passed in {summary.TotalElapsedMilliseconds} ms";
+
+ StatusMessage = summary.Failed == 0
+ ? $"Batch complete: all {summary.Total} scenarios passed."
+ : $"Batch complete: {summary.Failed} scenario(s) failed.";
+ }
+ catch (OperationCanceledException)
+ {
+ StatusMessage = "Batch run cancelled.";
+ }
+ catch (Exception ex)
+ {
+ StatusMessage = $"Batch run failed: {ex.Message}";
+ }
+ finally
+ {
+ IsBusy = false;
+ RedactCommand.NotifyCanExecuteChanged();
+ RunAllScenariosCommand.NotifyCanExecuteChanged();
+ SendToMockLlmCommand.NotifyCanExecuteChanged();
+ }
+ }
+
+ private bool CanRunBatch() => !IsBusy;
+
+ partial void OnPromptFilterChanged(string value) => PromptsView.Refresh();
+
+ private void ApplyOutcome(RedactionOutcome outcome)
+ {
+ OriginalPrompt = outcome.OriginalPrompt;
+ SanitizedOutput = outcome.SanitizedPrompt;
+ ElapsedMilliseconds = outcome.ElapsedMilliseconds;
+ EntityCount = outcome.DetectedEntities.Count;
+ LeakWarning = outcome.HasLeak;
+
+ DetectedEntities.Clear();
+ foreach (var entity in outcome.DetectedEntities)
+ {
+ DetectedEntities.Add(entity);
+ }
+
+ PlaceholderMap.Clear();
+ foreach (var placeholder in outcome.Placeholders)
+ {
+ PlaceholderMap.Add(placeholder);
+ }
+ }
+
+ private void RefreshModelStatus()
+ {
+ var snapshot = _modelStatusService.GetStatus();
+ ModelStatus = snapshot.Summary;
+ }
+
+ private void UpdateScriptComposition() =>
+ ScriptComposition = _scriptAnalysisService.GetComposition(InputPrompt);
+
+ private bool FilterPrompt(object item)
+ {
+ if (item is not TestPromptItemViewModel promptItem)
+ {
+ return false;
+ }
+
+ if (string.IsNullOrWhiteSpace(PromptFilter))
+ {
+ return true;
+ }
+
+ var filter = PromptFilter.Trim();
+ return promptItem.Name.Contains(filter, StringComparison.OrdinalIgnoreCase)
+ || promptItem.Category.Contains(filter, StringComparison.OrdinalIgnoreCase)
+ || promptItem.Description.Contains(filter, StringComparison.OrdinalIgnoreCase)
+ || promptItem.Language.ToString().Contains(filter, StringComparison.OrdinalIgnoreCase);
+ }
+}
diff --git a/src/PiiRedaction.TestHarness.Wpf/ViewModels/TestPromptItemViewModel.cs b/src/PiiRedaction.TestHarness.Wpf/ViewModels/TestPromptItemViewModel.cs
new file mode 100644
index 0000000..e8223f6
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/ViewModels/TestPromptItemViewModel.cs
@@ -0,0 +1,25 @@
+using PiiRedaction.TestHarness.Wpf.Models;
+
+namespace PiiRedaction.TestHarness.Wpf.ViewModels;
+
+public sealed class TestPromptItemViewModel
+{
+ public TestPromptItemViewModel(TestPromptScenario scenario)
+ {
+ Scenario = scenario;
+ }
+
+ public TestPromptScenario Scenario { get; }
+
+ public string Name => Scenario.Name;
+
+ public string Category => Scenario.Category;
+
+ public PromptLanguage Language => Scenario.Language;
+
+ public string Description => Scenario.Description;
+
+ public string Prompt => Scenario.Prompt;
+
+ public string DisplayLabel => $"{Name} ({Language})";
+}
diff --git a/src/PiiRedaction.TestHarness.Wpf/appsettings.json b/src/PiiRedaction.TestHarness.Wpf/appsettings.json
new file mode 100644
index 0000000..cb79f50
--- /dev/null
+++ b/src/PiiRedaction.TestHarness.Wpf/appsettings.json
@@ -0,0 +1,8 @@
+{
+ "PiiRedaction": {
+ "OnnxModelPath": "models/ner-model.onnx",
+ "EnglishOnnxModelPath": "models/en/ner-model.onnx",
+ "TamilOnnxModelPath": "models/ta/model.onnx",
+ "EnableTamilNer": true
+ }
+}
diff --git a/tests/PiiRedaction.Core.Tests/Detection/ScriptRouterTests.cs b/tests/PiiRedaction.Core.Tests/Detection/ScriptRouterTests.cs
new file mode 100644
index 0000000..bd9226e
--- /dev/null
+++ b/tests/PiiRedaction.Core.Tests/Detection/ScriptRouterTests.cs
@@ -0,0 +1,54 @@
+using FluentAssertions;
+using PiiRedaction.Core.Detection;
+using PiiRedaction.Infrastructure.Onnx;
+
+namespace PiiRedaction.Core.Tests.Detection;
+
+[TestFixture]
+public sealed class ScriptRouterTests
+{
+ private readonly ScriptRouter _router = new();
+
+ [Test]
+ public void GetComposition_LatinOnly_ReturnsLatinOnly()
+ {
+ _router.GetComposition("Customer Ravi Kumar called about billing.")
+ .Should().Be(ScriptComposition.LatinOnly);
+ }
+
+ [Test]
+ public void GetComposition_TamilOnly_ReturnsTamilOnly()
+ {
+ _router.GetComposition("வாடிக்கையாளர் ராஜேஷ் தொலைபேசி 9876543210")
+ .Should().Be(ScriptComposition.TamilOnly);
+ }
+
+ [Test]
+ public void GetComposition_Mixed_ReturnsMixed()
+ {
+ _router.GetComposition("Rajesh மற்றும் Priya disputed the charge.")
+ .Should().Be(ScriptComposition.Mixed);
+ }
+
+ [Test]
+ public void GetComposition_MixedTamilEnglishSample_ReturnsMixed()
+ {
+ _router.GetComposition("வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge.")
+ .Should().Be(ScriptComposition.Mixed);
+ }
+
+ [Test]
+ public void GetComposition_NoLetters_ReturnsNoLetters()
+ {
+ _router.GetComposition("9876543210 12345")
+ .Should().Be(ScriptComposition.NoLetters);
+ }
+
+ [Test]
+ public void GetComposition_TamilBoundaryChars_AreClassifiedAsTamil()
+ {
+ _router.GetComposition("\u0B80").Should().Be(ScriptComposition.TamilOnly);
+ _router.GetComposition("\u0BFF").Should().Be(ScriptComposition.TamilOnly);
+ _router.GetComposition("A").Should().Be(ScriptComposition.LatinOnly);
+ }
+}
diff --git a/tests/PiiRedaction.Core.Tests/Integration/GoldenPromptTests.cs b/tests/PiiRedaction.Core.Tests/Integration/GoldenPromptTests.cs
index 607a167..d0ae8bd 100644
--- a/tests/PiiRedaction.Core.Tests/Integration/GoldenPromptTests.cs
+++ b/tests/PiiRedaction.Core.Tests/Integration/GoldenPromptTests.cs
@@ -8,6 +8,7 @@ namespace PiiRedaction.Core.Tests.Integration;
public sealed class GoldenPromptTests
{
[TestCaseSource(typeof(PromptScenarioCatalog), nameof(PromptScenarioCatalog.AllScenarios))]
+ [TestCaseSource(typeof(TamilPromptScenarioCatalog), nameof(TamilPromptScenarioCatalog.AllScenarios))]
public void Sanitize_PromptScenario_ProducesExpectedOutput(PromptScenario scenario)
{
var sanitizer = ProductionPipelineFactory.CreateForScenario(scenario);
diff --git a/tests/PiiRedaction.Core.Tests/Integration/RealTamilPipelineTests.cs b/tests/PiiRedaction.Core.Tests/Integration/RealTamilPipelineTests.cs
new file mode 100644
index 0000000..665840b
--- /dev/null
+++ b/tests/PiiRedaction.Core.Tests/Integration/RealTamilPipelineTests.cs
@@ -0,0 +1,106 @@
+using FluentAssertions;
+using PiiRedaction.Core.Abstractions;
+using PiiRedaction.Core.Models;
+using PiiRedaction.Core.Tests.TestSupport;
+using PiiRedaction.Tests.Shared;
+
+namespace PiiRedaction.Core.Tests.Integration;
+
+///
+/// End-to-end pipeline proof using routed English + Tamil ONNX NER models.
+///
+[TestFixture]
+[Category("TamilNer")]
+public sealed class RealTamilPipelineTests : RealRoutingNerModelFixture
+{
+ private IPromptSanitizer _sanitizer = null!;
+
+ [OneTimeSetUp]
+ public void OneTimeSetUpPipeline()
+ {
+ _sanitizer = ProductionPipelineFactory.CreateWithRoutingRealModels(EnglishRunner, TamilRunner);
+ }
+
+ [Test]
+ public void Sanitize_TamilCustomerNameOnly_RedactsPerson()
+ {
+ const string prompt =
+ "வாடிக்கையாளர் ராஜேஷ் குமார் சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்.";
+
+ var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+ result.SanitizedPrompt.Should().Contain("");
+ result.SanitizedPrompt.Should().NotContain("ராஜேஷ்");
+ result.DetectedEntities.Should().Contain(entity =>
+ entity.Type == PiiEntityType.Person && entity.Source == PiiDetectionSource.Ner);
+ }
+
+ [Test]
+ public void Sanitize_TamilWithPhonePan_RedactsPersonPhoneAndPan()
+ {
+ const string prompt = "வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210 PAN ABCDE1234F.";
+
+ var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+ result.SanitizedPrompt.Should().Contain("");
+ result.SanitizedPrompt.Should().Contain("");
+ result.SanitizedPrompt.Should().Contain("");
+ result.SanitizedPrompt.Should().NotContainAny("ராஜேஷ்", "9876543210", "ABCDE1234F");
+ }
+
+ [Test]
+ public void Sanitize_TanglishCustomer_RedactsPersonAndPhone()
+ {
+ const string prompt = "Customer Senthil phone 9876543210 reported a failed UPI transfer.";
+
+ var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+ result.SanitizedPrompt.Should().Contain("");
+ result.SanitizedPrompt.Should().Contain("");
+ result.SanitizedPrompt.Should().NotContainAny("Senthil", "9876543210");
+ }
+
+ [Test]
+ public void Sanitize_MixedTamilEnglish_RedactsPersonAndPhone()
+ {
+ const string prompt = "வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge.";
+
+ var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+ result.SanitizedPrompt.Should().Contain("");
+ result.SanitizedPrompt.Should().Contain("");
+ result.SanitizedPrompt.Should().NotContainAny("Ravi Kumar", "9876543210");
+ }
+
+ [Test]
+ public void Sanitize_TamilFullFinancial_RedactsAllPiiTypes()
+ {
+ const string prompt =
+ "வாடிக்கையாளர் ராஜேஷ் குமார் மின்னஞ்சல் ravi.kumar@gmail.com தொலைபேசி 9876543210 LoanNumber LN-456789 PAN ABCDE1234F. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்.";
+
+ var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+ result.SanitizedPrompt.Should().Contain("");
+ result.SanitizedPrompt.Should().Contain("");
+ result.SanitizedPrompt.Should().Contain("");
+ result.SanitizedPrompt.Should().Contain("");
+ result.SanitizedPrompt.Should().Contain("");
+ result.SanitizedPrompt.Should().NotContainAny(
+ "ராஜேஷ்",
+ "ravi.kumar@gmail.com",
+ "9876543210",
+ "LN-456789",
+ "ABCDE1234F");
+ }
+
+ [Test]
+ public void Sanitize_CleanTamilQuestion_PassesThroughWithoutPersonRedaction()
+ {
+ const string prompt = "பணத்தை திரும்பப் பெறுவது எப்படி?";
+
+ var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
+
+ result.SanitizedPrompt.Should().Be(prompt);
+ result.DetectedEntities.Should().BeEmpty();
+ }
+}
diff --git a/tests/PiiRedaction.Core.Tests/PiiRedaction.Core.Tests.csproj b/tests/PiiRedaction.Core.Tests/PiiRedaction.Core.Tests.csproj
index 36fb022..eed46ae 100644
--- a/tests/PiiRedaction.Core.Tests/PiiRedaction.Core.Tests.csproj
+++ b/tests/PiiRedaction.Core.Tests/PiiRedaction.Core.Tests.csproj
@@ -25,6 +25,9 @@
+
+
+
diff --git a/tests/PiiRedaction.Core.Tests/TestSupport/ProductionPipelineFactory.cs b/tests/PiiRedaction.Core.Tests/TestSupport/ProductionPipelineFactory.cs
index 77866dc..da48196 100644
--- a/tests/PiiRedaction.Core.Tests/TestSupport/ProductionPipelineFactory.cs
+++ b/tests/PiiRedaction.Core.Tests/TestSupport/ProductionPipelineFactory.cs
@@ -1,7 +1,10 @@
+using Microsoft.Extensions.Options;
using PiiRedaction.Core.Abstractions;
+using PiiRedaction.Core.Configuration;
using PiiRedaction.Core.Detection;
using PiiRedaction.Core.Redaction;
using PiiRedaction.Core.Sanitization;
+using PiiRedaction.Infrastructure.Onnx;
namespace PiiRedaction.Core.Tests.TestSupport;
@@ -10,6 +13,15 @@ public static class ProductionPipelineFactory
public static IPromptSanitizer CreateWithRealModel(IOnnxNerModelRunner runner) =>
new PromptSanitizer(CreateCompositeDetector(runner), new PlaceholderPiiRedactor());
+ public static IPromptSanitizer CreateWithRoutingRealModels(
+ EnglishOnnxNerRunner englishRunner,
+ TamilOnnxNerRunner tamilRunner,
+ IOptions? options = null) =>
+ CreateWithRealModel(new RoutingOnnxNerModelRunner(
+ englishRunner,
+ tamilRunner,
+ options ?? Options.Create(new PiiRedactionOptions { EnableTamilNer = true })));
+
public static IPiiDetector CreateCompositeDetector(IOnnxNerModelRunner runner) =>
new CompositePiiDetector(
[
diff --git a/tests/PiiRedaction.Core.Tests/TestSupport/TamilPromptScenarioCatalog.cs b/tests/PiiRedaction.Core.Tests/TestSupport/TamilPromptScenarioCatalog.cs
new file mode 100644
index 0000000..ab3ca9e
--- /dev/null
+++ b/tests/PiiRedaction.Core.Tests/TestSupport/TamilPromptScenarioCatalog.cs
@@ -0,0 +1,63 @@
+using NUnit.Framework;
+using PiiRedaction.Core.Models;
+
+namespace PiiRedaction.Core.Tests.TestSupport;
+
+///
+/// Golden end-to-end scenarios for Tamil script, Tanglish, and mixed-script prompts.
+/// Uses fake NER spans for person names; regex and domain rules run for real.
+///
+public static class TamilPromptScenarioCatalog
+{
+ public static IEnumerable AllScenarios()
+ {
+ foreach (var scenario in BuildScenarios())
+ {
+ yield return new TestCaseData(scenario).SetName(scenario.Name);
+ }
+ }
+
+ private static IEnumerable BuildScenarios()
+ {
+ yield return TamilCustomerNameOnly();
+ yield return TamilWithPhonePan();
+ yield return TanglishCustomer();
+ yield return MixedTamilEnglish();
+ yield return TamilFullFinancial();
+ }
+
+ private static PromptScenario TamilCustomerNameOnly() => new(
+ "Tamil_CustomerNameOnly",
+ "வாடிக்கையாளர் ராஜேஷ் குமார் சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்.",
+ "வாடிக்கையாளர் சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்.",
+ [PiiEntityType.Person],
+ ["ராஜேஷ் குமார்"]);
+
+ private static PromptScenario TamilWithPhonePan() => new(
+ "Tamil_WithPhonePan",
+ "வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210 PAN ABCDE1234F.",
+ "வாடிக்கையாளர் தொலைபேசி PAN .",
+ [PiiEntityType.Person, PiiEntityType.Phone, PiiEntityType.Pan],
+ ["ராஜேஷ் குமார்", "9876543210", "ABCDE1234F"]);
+
+ private static PromptScenario TanglishCustomer() => new(
+ "Tanglish_CustomerPhone",
+ "Customer Senthil phone 9876543210 reported a failed UPI transfer.",
+ "Customer phone reported a failed UPI transfer.",
+ [PiiEntityType.Person, PiiEntityType.Phone],
+ ["Senthil", "9876543210"]);
+
+ private static PromptScenario MixedTamilEnglish() => new(
+ "Mixed_TamilEnglish",
+ "வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge.",
+ "வாடிக்கையாளர் phone disputed the charge.",
+ [PiiEntityType.Person, PiiEntityType.Phone],
+ ["Ravi Kumar", "9876543210"]);
+
+ private static PromptScenario TamilFullFinancial() => new(
+ "Tamil_FullFinancial",
+ "வாடிக்கையாளர் ராஜேஷ் குமார் மின்னஞ்சல் ravi.kumar@gmail.com தொலைபேசி 9876543210 LoanNumber LN-456789 PAN ABCDE1234F. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்.",
+ "வாடிக்கையாளர் மின்னஞ்சல் தொலைபேசி LoanNumber PAN . இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்.",
+ [PiiEntityType.Person, PiiEntityType.Email, PiiEntityType.Phone, PiiEntityType.LoanNumber, PiiEntityType.Pan],
+ ["ராஜேஷ் குமார்", "ravi.kumar@gmail.com", "9876543210", "LN-456789", "ABCDE1234F"]);
+}
diff --git a/tests/PiiRedaction.Infrastructure.Tests/Onnx/NerLabelConfigTests.cs b/tests/PiiRedaction.Infrastructure.Tests/Onnx/NerLabelConfigTests.cs
new file mode 100644
index 0000000..c9c8994
--- /dev/null
+++ b/tests/PiiRedaction.Infrastructure.Tests/Onnx/NerLabelConfigTests.cs
@@ -0,0 +1,28 @@
+using FluentAssertions;
+using PiiRedaction.Core.Detection;
+using PiiRedaction.Core.Models;
+using PiiRedaction.Infrastructure.Onnx;
+
+namespace PiiRedaction.Infrastructure.Tests.Onnx;
+
+[TestFixture]
+public sealed class NerLabelConfigTests
+{
+ [TestCase("B-PER", true)]
+ [TestCase("I-PER", true)]
+ [TestCase("B-PERSON", true)]
+ [TestCase("B-ORG", false)]
+ public void English_IsPersonLabel_MatchesExpected(string label, bool expected)
+ {
+ NerLabelConfig.English.IsPersonLabel(label).Should().Be(expected);
+ }
+
+ [TestCase("B-person-politician", true)]
+ [TestCase("I-person-artist", true)]
+ [TestCase("B-location", false)]
+ [TestCase("O", false)]
+ public void Tamil_IsPersonLabel_MatchesFineGrainedTags(string label, bool expected)
+ {
+ NerLabelConfig.Tamil.IsPersonLabel(label).Should().Be(expected);
+ }
+}
diff --git a/tests/PiiRedaction.Infrastructure.Tests/Onnx/RealTamilNerModelRunnerTests.cs b/tests/PiiRedaction.Infrastructure.Tests/Onnx/RealTamilNerModelRunnerTests.cs
new file mode 100644
index 0000000..c05e6a8
--- /dev/null
+++ b/tests/PiiRedaction.Infrastructure.Tests/Onnx/RealTamilNerModelRunnerTests.cs
@@ -0,0 +1,70 @@
+using FluentAssertions;
+using PiiRedaction.Core.Models;
+using PiiRedaction.Infrastructure.Onnx;
+using PiiRedaction.Tests.Shared;
+
+namespace PiiRedaction.Infrastructure.Tests.Onnx;
+
+[TestFixture]
+[Category("TamilNer")]
+public sealed class RealTamilNerModelRunnerTests : RealTamilNerModelFixture
+{
+ [Test]
+ public void IsModelAvailable_LoadsOnnxAndTokenizer()
+ {
+ Runner.IsModelAvailable.Should().BeTrue();
+ }
+
+ [TestCase("வாடிக்கையாளர் ராஜேஷ் குமார் அழைத்தார்.", "ராஜேஷ்", "ராஜேஷ் குமார்")]
+ public void PredictEntities_TamilScript_DetectsPersonEntity(
+ string prompt,
+ string expectedNamePart,
+ string expectedValue)
+ {
+ var entities = Runner.PredictEntities(prompt);
+
+ entities.Should().Contain(entity =>
+ entity.Type == PiiEntityType.Person &&
+ entity.Source == PiiDetectionSource.Ner &&
+ entity.Value.Contains(expectedNamePart, StringComparison.Ordinal) &&
+ prompt.AsSpan(entity.StartIndex, entity.Length).ToString() == entity.Value);
+
+ entities.Should().Contain(entity => entity.Value == expectedValue);
+ }
+
+ [Test]
+ public void PredictEntities_TamilWithPhone_DetectsPersonAndLeavesPhoneToRegex()
+ {
+ const string prompt = "வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210";
+
+ var entities = Runner.PredictEntities(prompt);
+
+ entities.Should().Contain(entity =>
+ entity.Type == PiiEntityType.Person &&
+ entity.Source == PiiDetectionSource.Ner &&
+ entity.Value.Contains("ராஜேஷ்", StringComparison.Ordinal));
+ entities.Should().NotContain(entity => entity.Type == PiiEntityType.Phone);
+ }
+
+ [Test]
+ public void PredictEntities_CleanTamilQuestion_ReturnsNoEntities()
+ {
+ const string prompt = "பணத்தை திரும்பப் பெறுவது எப்படி?";
+
+ Runner.PredictEntities(prompt).Should().BeEmpty();
+ }
+
+ [TestCase("Customer Senthil phone 9876543210", "Senthil")]
+ public void PredictEntities_TanglishLatinScript_DoesNotInvokeTamilRunner(
+ string prompt,
+ string expectedNamePart)
+ {
+ // TamilOnnxNerRunner is script-scoped; Tanglish is handled by English routing in pipeline tests.
+ // Direct Tamil runner on Latin-only text should not emit person spans.
+ var entities = Runner.PredictEntities(prompt);
+
+ entities.Should().NotContain(entity =>
+ entity.Type == PiiEntityType.Person &&
+ entity.Value.Contains(expectedNamePart, StringComparison.OrdinalIgnoreCase));
+ }
+}
diff --git a/tests/PiiRedaction.Infrastructure.Tests/Onnx/RoutingOnnxNerModelRunnerTests.cs b/tests/PiiRedaction.Infrastructure.Tests/Onnx/RoutingOnnxNerModelRunnerTests.cs
new file mode 100644
index 0000000..d75929a
--- /dev/null
+++ b/tests/PiiRedaction.Infrastructure.Tests/Onnx/RoutingOnnxNerModelRunnerTests.cs
@@ -0,0 +1,118 @@
+using FluentAssertions;
+using PiiRedaction.Core.Detection;
+using PiiRedaction.Core.Models;
+using PiiRedaction.Infrastructure.Onnx;
+
+namespace PiiRedaction.Infrastructure.Tests.Onnx;
+
+[TestFixture]
+public sealed class RoutingOnnxNerModelRunnerTests
+{
+ [Test]
+ public void PredictEntities_LatinOnly_UsesEnglishRunnerOnly()
+ {
+ var english = new FakeLanguageNerRunner("Ravi Kumar");
+ var tamil = new FakeLanguageNerRunner("தமிழ் பெயர்");
+ var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
+
+ var entities = router.PredictEntities("Customer Ravi Kumar called.");
+
+ entities.Should().ContainSingle(entity => entity.Value == "Ravi Kumar");
+ english.CallCount.Should().Be(1);
+ tamil.CallCount.Should().Be(0);
+ }
+
+ [Test]
+ public void PredictEntities_TamilOnly_UsesTamilRunnerOnly()
+ {
+ var english = new FakeLanguageNerRunner("Ravi Kumar");
+ var tamil = new FakeLanguageNerRunner("ராஜேஷ்");
+ var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
+
+ var entities = router.PredictEntities("வாடிக்கையாளர் ராஜேஷ்");
+
+ entities.Should().ContainSingle(entity => entity.Value == "ராஜேஷ்");
+ english.CallCount.Should().Be(0);
+ tamil.CallCount.Should().Be(1);
+ }
+
+ [Test]
+ public void PredictEntities_Mixed_InvokesBothRunners()
+ {
+ var english = new FakeLanguageNerRunner("EnglishName");
+ var tamil = new FakeLanguageNerRunner("தமிழ்");
+ var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
+
+ router.PredictEntities("Rajesh மற்றும் Priya");
+
+ english.CallCount.Should().Be(1);
+ tamil.CallCount.Should().Be(1);
+ }
+
+ [Test]
+ public void PredictEntities_NoLetters_InvokesNeither()
+ {
+ var english = new FakeLanguageNerRunner("ignored");
+ var tamil = new FakeLanguageNerRunner("ignored");
+ var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
+
+ router.PredictEntities("9876543210").Should().BeEmpty();
+ english.CallCount.Should().Be(0);
+ tamil.CallCount.Should().Be(0);
+ }
+
+ [Test]
+ public void PredictEntities_TamilDisabled_SkipsTamilRunnerForMixedText()
+ {
+ var english = new FakeLanguageNerRunner("EnglishName");
+ var tamil = new FakeLanguageNerRunner("தமிழ்");
+ var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: false);
+
+ router.PredictEntities("Rajesh மற்றும் Priya");
+
+ english.CallCount.Should().Be(1);
+ tamil.CallCount.Should().Be(0);
+ }
+
+ [Test]
+ public void MergePersonSpans_PrefersLongerOverlappingSpan()
+ {
+ var entities = new[]
+ {
+ CreatePerson("Raj", 0, 3),
+ CreatePerson("Rajesh", 0, 6)
+ };
+
+ var merged = RoutingOnnxNerModelRunner.MergePersonSpans(entities);
+
+ merged.Should().ContainSingle(entity => entity.Value == "Rajesh");
+ }
+
+ private static PiiEntity CreatePerson(string value, int start, int length) =>
+ new(PiiEntityType.Person, value, start, length, PiiDetectionSource.Ner);
+
+ private sealed class FakeLanguageNerRunner : IOnnxNerModelRunner
+ {
+ private readonly string _personValue;
+
+ public FakeLanguageNerRunner(string personValue) => _personValue = personValue;
+
+ public int CallCount { get; private set; }
+
+ public bool IsModelAvailable => true;
+
+ public IReadOnlyList PredictEntities(string text)
+ {
+ CallCount++;
+ return
+ [
+ new PiiEntity(
+ PiiEntityType.Person,
+ _personValue,
+ 0,
+ _personValue.Length,
+ PiiDetectionSource.Ner)
+ ];
+ }
+ }
+}
diff --git a/tests/PiiRedaction.Infrastructure.Tests/Onnx/TokenClassifierEncoderFactoryTests.cs b/tests/PiiRedaction.Infrastructure.Tests/Onnx/TokenClassifierEncoderFactoryTests.cs
new file mode 100644
index 0000000..ec0c19a
--- /dev/null
+++ b/tests/PiiRedaction.Infrastructure.Tests/Onnx/TokenClassifierEncoderFactoryTests.cs
@@ -0,0 +1,43 @@
+using FluentAssertions;
+using Microsoft.Extensions.Logging.Abstractions;
+using PiiRedaction.Infrastructure.Onnx;
+
+namespace PiiRedaction.Infrastructure.Tests.Onnx;
+
+[TestFixture]
+public sealed class TokenClassifierEncoderFactoryTests
+{
+ [Test]
+ public void Create_PrefersWordPieceWhenVocabExists()
+ {
+ var modelDirectory = ResolveTamilModelDirectory();
+ if (!File.Exists(Path.Combine(modelDirectory, "vocab.txt")))
+ {
+ Assert.Ignore("Tamil vocab.txt not found.");
+ }
+
+ var encoder = TokenClassifierEncoderFactory.Create(
+ modelDirectory,
+ NullLogger.Instance);
+
+ encoder.Should().BeOfType();
+ encoder.IsAvailable.Should().BeTrue();
+ }
+
+ private static string ResolveTamilModelDirectory()
+ {
+ var directory = new DirectoryInfo(AppContext.BaseDirectory);
+ while (directory is not null)
+ {
+ var candidate = Path.Combine(directory.FullName, "models", "ta");
+ if (Directory.Exists(candidate))
+ {
+ return candidate;
+ }
+
+ directory = directory.Parent;
+ }
+
+ return Path.Combine(Environment.CurrentDirectory, "models", "ta");
+ }
+}
diff --git a/tests/PiiRedaction.Infrastructure.Tests/PiiRedaction.Infrastructure.Tests.csproj b/tests/PiiRedaction.Infrastructure.Tests/PiiRedaction.Infrastructure.Tests.csproj
index 699c707..324e657 100644
--- a/tests/PiiRedaction.Infrastructure.Tests/PiiRedaction.Infrastructure.Tests.csproj
+++ b/tests/PiiRedaction.Infrastructure.Tests/PiiRedaction.Infrastructure.Tests.csproj
@@ -24,6 +24,8 @@
+
+
diff --git a/tests/TestSupport.Shared/RealNerModelFixture.cs b/tests/TestSupport.Shared/RealNerModelFixture.cs
index 5ec3316..578ca13 100644
--- a/tests/TestSupport.Shared/RealNerModelFixture.cs
+++ b/tests/TestSupport.Shared/RealNerModelFixture.cs
@@ -6,12 +6,12 @@ using PiiRedaction.Infrastructure.Onnx;
namespace PiiRedaction.Tests.Shared;
///
-/// Reuses a single per fixture for performance.
+/// Reuses a single per fixture for performance.
/// Skips all tests in the class when the ONNX model is missing or cannot be loaded.
///
public abstract class RealNerModelFixture
{
- protected OnnxNerModelRunner Runner { get; private set; } = null!;
+ protected EnglishOnnxNerRunner Runner { get; private set; } = null!;
protected string ModelPath { get; private set; } = null!;
@@ -24,8 +24,12 @@ public abstract class RealNerModelFixture
Assert.Ignore(RealNerModelPaths.ModelMissingMessage);
}
- var options = Options.Create(new PiiRedactionOptions { OnnxModelPath = ModelPath });
- Runner = new OnnxNerModelRunner(options, NullLogger.Instance);
+ var options = Options.Create(new PiiRedactionOptions
+ {
+ EnglishOnnxModelPath = ModelPath,
+ OnnxModelPath = ModelPath
+ });
+ Runner = new EnglishOnnxNerRunner(options, NullLogger.Instance);
if (!Runner.IsModelAvailable)
{
diff --git a/tests/TestSupport.Shared/RealNerModelPaths.cs b/tests/TestSupport.Shared/RealNerModelPaths.cs
index 99a6072..327365d 100644
--- a/tests/TestSupport.Shared/RealNerModelPaths.cs
+++ b/tests/TestSupport.Shared/RealNerModelPaths.cs
@@ -7,16 +7,23 @@ public static class RealNerModelPaths
public static string ResolveRepoModelPath()
{
- var directory = new DirectoryInfo(AppContext.BaseDirectory);
- while (directory is not null)
+ foreach (var relativePath in new[]
+ {
+ Path.Combine("models", "en", "ner-model.onnx"),
+ Path.Combine("models", "ner-model.onnx")
+ })
{
- var candidate = Path.Combine(directory.FullName, "models", "ner-model.onnx");
- if (File.Exists(candidate))
+ var directory = new DirectoryInfo(AppContext.BaseDirectory);
+ while (directory is not null)
{
- return candidate;
- }
+ var candidate = Path.Combine(directory.FullName, relativePath);
+ if (File.Exists(candidate))
+ {
+ return candidate;
+ }
- directory = directory.Parent;
+ directory = directory.Parent;
+ }
}
return Path.Combine(Environment.CurrentDirectory, "models", "ner-model.onnx");
diff --git a/tests/TestSupport.Shared/RealRoutingNerModelFixture.cs b/tests/TestSupport.Shared/RealRoutingNerModelFixture.cs
new file mode 100644
index 0000000..dc20818
--- /dev/null
+++ b/tests/TestSupport.Shared/RealRoutingNerModelFixture.cs
@@ -0,0 +1,69 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using PiiRedaction.Core.Configuration;
+using PiiRedaction.Infrastructure.Onnx;
+
+namespace PiiRedaction.Tests.Shared;
+
+///
+/// Loads English and Tamil ONNX runners and exposes a .
+/// Skips when either model is missing or cannot be loaded.
+///
+public abstract class RealRoutingNerModelFixture
+{
+ protected RoutingOnnxNerModelRunner Runner { get; private set; } = null!;
+
+ protected EnglishOnnxNerRunner EnglishRunner { get; private set; } = null!;
+
+ protected TamilOnnxNerRunner TamilRunner { get; private set; } = null!;
+
+ [OneTimeSetUp]
+ public void OneTimeSetUpRoutingModels()
+ {
+ var englishPath = RealNerModelPaths.ResolveRepoModelPath();
+ if (!File.Exists(englishPath))
+ {
+ Assert.Ignore(RealNerModelPaths.ModelMissingMessage);
+ }
+
+ var tamilPath = RealTamilNerModelPaths.ResolveRepoModelPath();
+ if (!File.Exists(tamilPath))
+ {
+ Assert.Ignore(RealTamilNerModelPaths.ModelMissingMessage);
+ }
+
+ var options = Options.Create(new PiiRedactionOptions
+ {
+ EnglishOnnxModelPath = englishPath,
+ OnnxModelPath = englishPath,
+ TamilOnnxModelPath = tamilPath,
+ EnableTamilNer = true
+ });
+
+ EnglishRunner = new EnglishOnnxNerRunner(options, NullLogger.Instance);
+ TamilRunner = new TamilOnnxNerRunner(options, NullLogger.Instance);
+
+ if (!EnglishRunner.IsModelAvailable)
+ {
+ EnglishRunner.Dispose();
+ TamilRunner.Dispose();
+ Assert.Ignore(RealNerModelPaths.ModelMissingMessage);
+ }
+
+ if (!TamilRunner.IsModelAvailable)
+ {
+ EnglishRunner.Dispose();
+ TamilRunner.Dispose();
+ Assert.Ignore(RealTamilNerModelPaths.ModelMissingMessage);
+ }
+
+ Runner = new RoutingOnnxNerModelRunner(EnglishRunner, TamilRunner, options);
+ }
+
+ [OneTimeTearDown]
+ public void OneTimeTearDownRoutingModels()
+ {
+ EnglishRunner?.Dispose();
+ TamilRunner?.Dispose();
+ }
+}
diff --git a/tests/TestSupport.Shared/RealTamilNerModelFixture.cs b/tests/TestSupport.Shared/RealTamilNerModelFixture.cs
new file mode 100644
index 0000000..214487f
--- /dev/null
+++ b/tests/TestSupport.Shared/RealTamilNerModelFixture.cs
@@ -0,0 +1,45 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using PiiRedaction.Core.Configuration;
+using PiiRedaction.Infrastructure.Onnx;
+
+namespace PiiRedaction.Tests.Shared;
+
+///
+/// Reuses a single per fixture for performance.
+/// Skips all tests in the class when the Tamil ONNX model is missing or cannot be loaded.
+///
+public abstract class RealTamilNerModelFixture
+{
+ protected TamilOnnxNerRunner Runner { get; private set; } = null!;
+
+ protected string ModelPath { get; private set; } = null!;
+
+ [OneTimeSetUp]
+ public void OneTimeSetUpTamilModel()
+ {
+ ModelPath = RealTamilNerModelPaths.ResolveRepoModelPath();
+ if (!File.Exists(ModelPath))
+ {
+ Assert.Ignore(RealTamilNerModelPaths.ModelMissingMessage);
+ }
+
+ var options = Options.Create(new PiiRedactionOptions
+ {
+ TamilOnnxModelPath = ModelPath
+ });
+ Runner = new TamilOnnxNerRunner(options, NullLogger.Instance);
+
+ if (!Runner.IsModelAvailable)
+ {
+ Runner.Dispose();
+ Assert.Ignore(RealTamilNerModelPaths.ModelMissingMessage);
+ }
+ }
+
+ [OneTimeTearDown]
+ public void OneTimeTearDownTamilModel()
+ {
+ Runner?.Dispose();
+ }
+}
diff --git a/tests/TestSupport.Shared/RealTamilNerModelPaths.cs b/tests/TestSupport.Shared/RealTamilNerModelPaths.cs
new file mode 100644
index 0000000..ace7f66
--- /dev/null
+++ b/tests/TestSupport.Shared/RealTamilNerModelPaths.cs
@@ -0,0 +1,25 @@
+namespace PiiRedaction.Tests.Shared;
+
+public static class RealTamilNerModelPaths
+{
+ public const string ModelMissingMessage =
+ "Tamil ONNX model not found. Run scripts/download-tamil-ner-model.ps1 from the repository root.";
+
+ public static string ResolveRepoModelPath()
+ {
+ const string relativePath = "models/ta/model.onnx";
+ var directory = new DirectoryInfo(AppContext.BaseDirectory);
+ while (directory is not null)
+ {
+ var candidate = Path.Combine(directory.FullName, relativePath);
+ if (File.Exists(candidate))
+ {
+ return candidate;
+ }
+
+ directory = directory.Parent;
+ }
+
+ return Path.Combine(Environment.CurrentDirectory, relativePath);
+ }
+}