Compare commits
2 Commits
main
...
feature/ta
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
81220c6ade | ||
|
|
cf8f5a7232 |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -10,7 +10,11 @@ models/*.onnx
|
|||||||
models/vocab.txt
|
models/vocab.txt
|
||||||
models/ner-labels.txt
|
models/ner-labels.txt
|
||||||
models/*.json
|
models/*.json
|
||||||
|
models/en/*
|
||||||
|
models/ta/*
|
||||||
!models/.gitkeep
|
!models/.gitkeep
|
||||||
|
!models/en/.gitkeep
|
||||||
|
!models/ta/.gitkeep
|
||||||
|
|
||||||
## IDE
|
## IDE
|
||||||
.idea/
|
.idea/
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
<Project Path="src/PiiRedaction.ConsoleApp/PiiRedaction.ConsoleApp.csproj" />
|
<Project Path="src/PiiRedaction.ConsoleApp/PiiRedaction.ConsoleApp.csproj" />
|
||||||
<Project Path="src/PiiRedaction.Core/PiiRedaction.Core.csproj" />
|
<Project Path="src/PiiRedaction.Core/PiiRedaction.Core.csproj" />
|
||||||
<Project Path="src/PiiRedaction.Infrastructure/PiiRedaction.Infrastructure.csproj" />
|
<Project Path="src/PiiRedaction.Infrastructure/PiiRedaction.Infrastructure.csproj" />
|
||||||
|
<Project Path="src/PiiRedaction.TestHarness.Wpf/PiiRedaction.TestHarness.Wpf.csproj" />
|
||||||
</Folder>
|
</Folder>
|
||||||
<Folder Name="/tests/">
|
<Folder Name="/tests/">
|
||||||
<Project Path="tests/PiiRedaction.Core.Tests/PiiRedaction.Core.Tests.csproj" />
|
<Project Path="tests/PiiRedaction.Core.Tests/PiiRedaction.Core.Tests.csproj" />
|
||||||
|
|||||||
80
README.md
80
README.md
@@ -11,9 +11,9 @@ Financial and customer-service prompts often contain regulated data (names, gove
|
|||||||
3. Replace values with stable placeholders
|
3. Replace values with stable placeholders
|
||||||
4. Send only the **sanitized** prompt to an LLM (mocked for now)
|
4. Send only the **sanitized** prompt to an LLM (mocked for now)
|
||||||
|
|
||||||
## Architecture
|
## Documentation
|
||||||
|
|
||||||
For solution design, data-flow diagrams, trust boundaries, and project responsibilities, see **[docs/architecture.md](docs/architecture.md)**.
|
Full solution reference (architecture, NER models, routing, Tamil/Tanglish, Git setup, improvement roadmap): **[docs/solution-guide.md](docs/solution-guide.md)**
|
||||||
|
|
||||||
## Why Three Detection Strategies?
|
## Why Three Detection Strategies?
|
||||||
|
|
||||||
@@ -31,7 +31,8 @@ The placeholder map (`<PERSON_1>` → original value) is kept **in-process** for
|
|||||||
|
|
||||||
```
|
```
|
||||||
src/
|
src/
|
||||||
├── PiiRedaction.ConsoleApp/ # Presentation: input/output, DI bootstrap
|
├── PiiRedaction.ConsoleApp/ # Console demo: input/output, DI bootstrap
|
||||||
|
├── PiiRedaction.TestHarness.Wpf/ # WPF MVVM test harness for manual POC validation
|
||||||
├── PiiRedaction.Core/ # Business logic: detection, redaction, models
|
├── PiiRedaction.Core/ # Business logic: detection, redaction, models
|
||||||
└── PiiRedaction.Infrastructure/ # Technical adapters: ONNX Runtime, mock LLM
|
└── PiiRedaction.Infrastructure/ # Technical adapters: ONNX Runtime, mock LLM
|
||||||
models/ # Optional ONNX model files (gitignored)
|
models/ # Optional ONNX model files (gitignored)
|
||||||
@@ -40,6 +41,7 @@ models/ # Optional ONNX model files (gitignored)
|
|||||||
| Project | Responsibility |
|
| Project | Responsibility |
|
||||||
|---------|----------------|
|
|---------|----------------|
|
||||||
| `PiiRedaction.ConsoleApp` | Read prompt, call sanitizer, display results, call LLM service |
|
| `PiiRedaction.ConsoleApp` | Read prompt, call sanitizer, display results, call LLM service |
|
||||||
|
| `PiiRedaction.TestHarness.Wpf` | Desktop test harness: preset prompts, redact UI, batch validation |
|
||||||
| `PiiRedaction.Core` | PII detection abstractions, redaction, sanitization orchestration |
|
| `PiiRedaction.Core` | PII detection abstractions, redaction, sanitization orchestration |
|
||||||
| `PiiRedaction.Infrastructure` | ONNX model runner, `IChatClient` mock implementation |
|
| `PiiRedaction.Infrastructure` | ONNX model runner, `IChatClient` mock implementation |
|
||||||
|
|
||||||
@@ -53,7 +55,7 @@ models/ # Optional ONNX model files (gitignored)
|
|||||||
|
|
||||||
## Build and Run
|
## Build and Run
|
||||||
|
|
||||||
For pushing this repository to Xenovex Git (`xts.xenovex.com`), see **[docs/git-xenovex-setup.md](docs/git-xenovex-setup.md)**.
|
For pushing this repository to Xenovex Git (`xts.xenovex.com`), see **[docs/solution-guide.md § Git remote setup](docs/solution-guide.md#11-git-remote-setup-xenovex)**.
|
||||||
|
|
||||||
From the repository root:
|
From the repository root:
|
||||||
|
|
||||||
@@ -63,7 +65,7 @@ dotnet build
|
|||||||
dotnet run --project src/PiiRedaction.ConsoleApp
|
dotnet run --project src/PiiRedaction.ConsoleApp
|
||||||
```
|
```
|
||||||
|
|
||||||
By default the console app runs **11 curated sample prompts** covering NER/person names, regex identifiers, domain IDs, combined scenarios, and a clean no-PII ticket.
|
By default the console app runs **16 curated sample prompts** covering English and Tamil/Tanglish/mixed person names, regex identifiers, domain IDs, combined scenarios, and a clean no-PII ticket. No flags are required for Tamil samples — they run in the default batch alongside English.
|
||||||
|
|
||||||
List available samples:
|
List available samples:
|
||||||
|
|
||||||
@@ -78,6 +80,27 @@ dotnet run --project src/PiiRedaction.ConsoleApp -- --sample 2
|
|||||||
dotnet run --project src/PiiRedaction.ConsoleApp -- --name MrTitlePerson
|
dotnet run --project src/PiiRedaction.ConsoleApp -- --name MrTitlePerson
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### WPF Test Harness
|
||||||
|
|
||||||
|
A desktop **MVVM** application for interactive POC validation with English and Tamil prompts. Requires **Windows** (`net10.0-windows`).
|
||||||
|
|
||||||
|
**Prerequisites:** English and Tamil ONNX models downloaded (see [ONNX Model Setup](#onnx-model-setup)).
|
||||||
|
|
||||||
|
```bash
|
||||||
|
dotnet run --project src/PiiRedaction.TestHarness.Wpf
|
||||||
|
```
|
||||||
|
|
||||||
|
**Workflow:**
|
||||||
|
|
||||||
|
1. **Select a category** from the dropdown (e.g. **Career Guidance**, **Banking & Financial**) or leave **All** to see every prompt. Use the search box for finer filtering.
|
||||||
|
2. **Click a test prompt** in the left panel to load it into the input box (previous results are cleared automatically).
|
||||||
|
3. Click **Redact** to run the full detection pipeline. The status bar shows model availability, script composition (LatinOnly / TamilOnly / Mixed), and elapsed time.
|
||||||
|
4. Review **Sanitized Output**, detected entities, and the placeholder map in the right panel. A leak warning appears if any detected value remains in the sanitized text.
|
||||||
|
5. Optionally click **Send Mock LLM** to send only the sanitized prompt to the mock LLM.
|
||||||
|
6. Click **Run All** to execute scenarios in the **selected category** (or all when **All** is chosen) and view pass/fail results in the batch panel.
|
||||||
|
|
||||||
|
The harness uses the same DI registrations and `IPromptSanitizer` pipeline as the console app, with thin application services (`IRedactionAppService`, `ITestPromptCatalog`, `IScriptAnalysisService`, `IModelStatusService`) following SOLID principles.
|
||||||
|
|
||||||
Interactive mode (enter your own prompt):
|
Interactive mode (enter your own prompt):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -100,7 +123,12 @@ Samples are defined in [`SamplePromptCatalog.cs`](src/PiiRedaction.ConsoleApp/Sa
|
|||||||
| 7 | PersonWithEmailNoPhone | NER + Regex | `Customer Arjun Mehta` + email |
|
| 7 | PersonWithEmailNoPhone | NER + Regex | `Customer Arjun Mehta` + email |
|
||||||
| 8 | AllRegexTypes | Regex | email, phone, PAN, Aadhaar, card |
|
| 8 | AllRegexTypes | Regex | email, phone, PAN, Aadhaar, card |
|
||||||
| 9 | AllDomainIds | Domain | LN, CID, ACC |
|
| 9 | AllDomainIds | Domain | LN, CID, ACC |
|
||||||
| 10 | NoPiiCleanTicket | Negative | no redaction |
|
| 10 | TamilCustomerNameOnly | NER (Tamil) | `வாடிக்கையாளர் ராஜேஷ் குமார்` |
|
||||||
|
| 11 | TamilWithPhonePan | NER (Tamil) + Regex | Tamil person + phone + PAN |
|
||||||
|
| 12 | TanglishCustomer | NER (English/Tanglish) | `Customer Senthil` + phone |
|
||||||
|
| 13 | MixedTamilEnglish | NER (Mixed) | `வாடிக்கையாளர் Ravi Kumar` + phone |
|
||||||
|
| 14 | TamilFullFinancial | NER (Tamil) + Regex + Domain | Tamil canonical demo |
|
||||||
|
| 15 | NoPiiCleanTicket | Negative | no redaction |
|
||||||
|
|
||||||
Person names are detected via **ONNX NER** using `dslim/bert-base-NER` (or a compatible token-classification export). A real model is **required** for person-name detection; there is no regex or heuristic fallback.
|
Person names are detected via **ONNX NER** using `dslim/bert-base-NER` (or a compatible token-classification export). A real model is **required** for person-name detection; there is no regex or heuristic fallback.
|
||||||
|
|
||||||
@@ -147,36 +175,41 @@ Person-name detection requires a token-classification ONNX model and companion t
|
|||||||
|
|
||||||
| File | Purpose |
|
| File | Purpose |
|
||||||
|------|---------|
|
|------|---------|
|
||||||
| `models/ner-model.onnx` | Exported NER model |
|
| `models/en/ner-model.onnx` | English BERT NER model (or legacy `models/ner-model.onnx`) |
|
||||||
| `models/vocab.txt` | BERT WordPiece vocabulary |
|
| `models/en/vocab.txt` | BERT WordPiece vocabulary |
|
||||||
| `models/ner-labels.txt` | One BIO label per line (`O`, `B-PER`, `I-PER`, etc.) |
|
| `models/en/ner-labels.txt` | One BIO label per line (`O`, `B-PER`, `I-PER`, etc.) |
|
||||||
|
| `models/ta/model.onnx` | Tamil IndicBERT NER model |
|
||||||
|
| `models/ta/sentencepiece.bpe.model` | SentencePiece tokenizer for Tamil model |
|
||||||
|
| `models/ta/ner-labels.txt` | Fine-grained Tamil NER labels |
|
||||||
|
|
||||||
### Download script
|
### Download scripts
|
||||||
|
|
||||||
From the repository root:
|
From the repository root:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
.\scripts\download-ner-model.ps1
|
.\scripts\download-ner-model.ps1
|
||||||
|
.\scripts\download-tamil-ner-model.ps1
|
||||||
```
|
```
|
||||||
|
|
||||||
Or with Python directly:
|
Or with Python directly:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python scripts/download-ner-model.py
|
python scripts/download-ner-model.py
|
||||||
|
python scripts/download-tamil-ner-model.py
|
||||||
```
|
```
|
||||||
|
|
||||||
The script exports [`dslim/bert-base-NER`](https://huggingface.co/dslim/bert-base-NER) via Hugging Face Optimum when Python is available. Otherwise it downloads the pre-exported ONNX assets from Hugging Face directly.
|
The English script exports [`dslim/bert-base-NER`](https://huggingface.co/dslim/bert-base-NER) via Hugging Face Optimum when Python is available. The Tamil script exports [`prachuryyaIITG/SampurNER_Tamil_IndicBERTv2`](https://huggingface.co/prachuryyaIITG/SampurNER_Tamil_IndicBERTv2). Otherwise each script downloads pre-exported ONNX assets from Hugging Face directly.
|
||||||
|
|
||||||
|
Set `EnableTamilNer` to `false` in `appsettings.json` to revert to English-only routing.
|
||||||
|
|
||||||
### Inference pipeline
|
### Inference pipeline
|
||||||
|
|
||||||
`OnnxNerModelRunner` performs the full pipeline:
|
`RoutingOnnxNerModelRunner` classifies script composition and delegates to:
|
||||||
|
|
||||||
- BERT WordPiece tokenization (`Microsoft.ML.Tokenizers`)
|
- **`EnglishOnnxNerRunner`** — BERT WordPiece tokenization for Latin script and Tanglish
|
||||||
- ONNX Runtime inference (`input_ids`, `attention_mask`, optional `token_type_ids`)
|
- **`TamilOnnxNerRunner`** — SentencePiece tokenization for Tamil script (U+0B80–U+0BFF)
|
||||||
- BIO label decoding (`B-PER` / `I-PER` → `PiiEntityType.Person`)
|
|
||||||
- Character-span alignment back to the source text
|
|
||||||
|
|
||||||
When the model or tokenizer files are missing, person detection returns no results.
|
Both runners share `OnnxTokenClassifierRunner` for ONNX Runtime inference and BIO label decoding. Overlapping person spans from mixed-script prompts are merged (longer span wins).
|
||||||
|
|
||||||
## Swapping Mock LLM for Azure OpenAI
|
## Swapping Mock LLM for Azure OpenAI
|
||||||
|
|
||||||
@@ -251,24 +284,29 @@ The solution includes an **NUnit** test suite across two projects:
|
|||||||
dotnet test
|
dotnet test
|
||||||
dotnet test --filter "FullyQualifiedName~GoldenPromptTests"
|
dotnet test --filter "FullyQualifiedName~GoldenPromptTests"
|
||||||
dotnet test --filter "Category=RealModel"
|
dotnet test --filter "Category=RealModel"
|
||||||
|
dotnet test --filter "Category=TamilNer"
|
||||||
dotnet test --logger "console;verbosity=detailed"
|
dotnet test --logger "console;verbosity=detailed"
|
||||||
```
|
```
|
||||||
|
|
||||||
Fast CI runs without the ONNX model: fake-based tests always execute; tests marked **`Category=RealModel`** are skipped when `models/ner-model.onnx` is absent. Download the model first:
|
Fast CI runs without the ONNX model: fake-based tests always execute; tests marked **`Category=RealModel`** or **`Category=TamilNer`** are skipped when the corresponding ONNX models are absent. Download models first:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
.\scripts\download-ner-model.ps1
|
.\scripts\download-ner-model.ps1
|
||||||
|
.\scripts\download-tamil-ner-model.ps1
|
||||||
```
|
```
|
||||||
|
|
||||||
### Test architecture
|
### Test architecture
|
||||||
|
|
||||||
- **`PromptScenarioCatalog`** — five focused end-to-end scenarios (canonical demo, multi-regex, duplicate people, overlap stress, no-PII negative)
|
- **`PromptScenarioCatalog`** — five focused end-to-end English scenarios (canonical demo, multi-regex, duplicate people, overlap stress, no-PII negative)
|
||||||
- **`ProductionPipelineFactory`** — builds the same Domain → Regex → OnnxNer composite stack as production DI; `CreateWithRealModel(runner)` wires a real `OnnxNerModelRunner`
|
- **`TamilPromptScenarioCatalog`** — five Tamil/Tanglish/mixed golden scenarios (fake NER for person spans)
|
||||||
|
- **`ProductionPipelineFactory`** — builds the same Domain → Regex → OnnxNer composite stack as production DI; `CreateWithRealModel(runner)` wires a real runner; `CreateWithRoutingRealModels` wires English + Tamil routing
|
||||||
- **`FakeOnnxNerModelRunner`** — unit-test double for NER; golden tests inject person spans per scenario
|
- **`FakeOnnxNerModelRunner`** — unit-test double for NER; golden tests inject person spans per scenario
|
||||||
- **`GoldenPromptTests`** — end-to-end sanitization proof across the catalog (fake NER)
|
- **`GoldenPromptTests`** — end-to-end sanitization proof across the catalog (fake NER)
|
||||||
- **`RealNerModelFixture`** — shared fixture that loads `models/ner-model.onnx` once per class; skips when model missing
|
- **`RealNerModelFixture`** — shared fixture that loads `models/ner-model.onnx` once per class; skips when model missing
|
||||||
- **`RealNerModelRunnerTests`** — direct ONNX inference with span accuracy checks
|
- **`RealNerModelRunnerTests`** — direct ONNX inference with span accuracy checks
|
||||||
- **`RealNerPipelineTests`** — full pipeline with real NER (canonical, multi-person, clean-ticket negative)
|
- **`RealNerPipelineTests`** — full pipeline with real English NER (canonical, multi-person, clean-ticket negative)
|
||||||
|
- **`RealTamilPipelineTests`** — full pipeline with routed English + Tamil NER (`Category=TamilNer`)
|
||||||
|
- **`RealTamilNerModelRunnerTests`** — direct Tamil ONNX inference (`Category=TamilNer`)
|
||||||
- **`OnnxNerModelRunnerTests`** — unit tests for missing/invalid model paths (no download required)
|
- **`OnnxNerModelRunnerTests`** — unit tests for missing/invalid model paths (no download required)
|
||||||
- **`CompositePiiDetectorTests`** — overlap merge and source-priority rules
|
- **`CompositePiiDetectorTests`** — overlap merge and source-priority rules
|
||||||
- **`LlmBoundaryTests`** — verifies raw PII never appears in outbound LLM messages
|
- **`LlmBoundaryTests`** — verifies raw PII never appears in outbound LLM messages
|
||||||
|
|||||||
@@ -1,295 +0,0 @@
|
|||||||
# PII Redaction POC — Solution Architecture
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This document describes the architectural design of the **PII Redaction POC**, a .NET proof-of-concept that intercepts user prompts containing regulated personally identifiable information (PII), redacts sensitive values into stable placeholders, and transmits **only sanitized text** across the LLM trust boundary. The solution is structured for enterprise adoption: clear layer separation, interface-driven composition, dependency injection, and swappable infrastructure adapters (ONNX NER, `Microsoft.Extensions.AI` chat clients).
|
|
||||||
|
|
||||||
The POC validates a compliance-oriented pattern suitable for financial and customer-service workloads where raw PII must not leave the application process when invoking external language models.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Canonical Example
|
|
||||||
|
|
||||||
The console application ships with a **sample catalog** (11 prompts). The canonical demo is sample `FullFinancialWithCustomer`. The table below shows the exact strings produced by the production pipeline when the ONNX NER model is loaded (run `scripts/download-ner-model.ps1` first).
|
|
||||||
|
|
||||||
| Stage | Value |
|
|
||||||
|-------|-------|
|
|
||||||
| **Input** | `Customer Ravi Kumar with email ravi.kumar@gmail.com and phone 9876543210 has LoanNumber LN-456789 and PAN ABCDE1234F. Please summarize this customer issue.` |
|
|
||||||
| **Sanitized Output** | `Customer <PERSON_1> with email <EMAIL_1> and phone <PHONE_1> has LoanNumber <LOAN_NUMBER_1> and PAN <PAN_1>. Please summarize this customer issue.` |
|
|
||||||
| **Mock LLM Response** | `[Mock LLM Response] Received sanitized prompt (146 chars). No original PII was transmitted.` |
|
|
||||||
|
|
||||||
Detected entities for this prompt:
|
|
||||||
|
|
||||||
| Type | Value | Detection Source |
|
|
||||||
|------|-------|------------------|
|
|
||||||
| PERSON | Ravi Kumar | Ner |
|
|
||||||
| EMAIL | ravi.kumar@gmail.com | Regex |
|
|
||||||
| PHONE | 9876543210 | Regex |
|
|
||||||
| LOAN_NUMBER | LN-456789 | Domain |
|
|
||||||
| PAN | ABCDE1234F | Regex |
|
|
||||||
|
|
||||||
The internal placeholder map (`<PERSON_1>` → `Ravi Kumar`, etc.) is retained in-process and is **not** included in the outbound LLM request.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Console Sample Catalog
|
|
||||||
|
|
||||||
Running `dotnet run --project src/PiiRedaction.ConsoleApp` executes all samples sequentially. Use `--list`, `--sample N`, or `--name SampleName` to filter.
|
|
||||||
|
|
||||||
### NER / person-name samples
|
|
||||||
|
|
||||||
These prompts exercise `OnnxNerPiiDetector` and `OnnxNerModelRunner`. Person names require the ONNX model (`models/ner-model.onnx` plus `vocab.txt` and `ner-labels.txt`). Without the model, person spans are not detected.
|
|
||||||
|
|
||||||
| Sample | Input (excerpt) | Detected person | Sanitized (excerpt) |
|
|
||||||
|--------|-----------------|-----------------|---------------------|
|
|
||||||
| **CustomerNameOnly** | Customer Anita Sharma reported unauthorized… | Anita Sharma | Customer `<PERSON_1>` reported unauthorized… |
|
|
||||||
| **MrTitlePerson** | Mr. John Smith called about a duplicate debit… | John Smith | `<PERSON_1>` called about a duplicate debit… |
|
|
||||||
| **MrsTitlePerson** | Mrs. Lakshmi Reddy requested a callback regarding LN-112233. | Lakshmi Reddy | `<PERSON_1>` requested a callback regarding `<LOAN_NUMBER_1>`. |
|
|
||||||
| **DrTitlePerson** | Dr. Jane Doe escalated a complaint… | Jane Doe | `<PERSON_1>` escalated a complaint… |
|
|
||||||
| **TwoCustomersInOnePrompt** | Customer Ravi Kumar and Customer Priya Nair… | Ravi Kumar, Priya Nair | Customer `<PERSON_1>` and Customer `<PERSON_2>`… |
|
|
||||||
| **PersonWithDomainIds** | Customer Meera Iyer holds CID-7070… | Meera Iyer | Customer `<PERSON_1>` holds `<CUSTOMER_ID_1>`… |
|
|
||||||
| **PersonWithEmailNoPhone** | Customer Arjun Mehta wrote from arjun.mehta@company.in… | Arjun Mehta | Customer `<PERSON_1>` wrote from `<EMAIL_1>`… |
|
|
||||||
|
|
||||||
### Other sample categories
|
|
||||||
|
|
||||||
| Category | Sample | Purpose |
|
|
||||||
|----------|--------|---------|
|
|
||||||
| NER + Regex + Domain | FullFinancialWithCustomer | End-to-end financial prompt (canonical) |
|
|
||||||
| Regex only | AllRegexTypes | Email, phone, PAN, Aadhaar, credit card |
|
|
||||||
| Domain only | AllDomainIds | Loan number, customer ID, account number |
|
|
||||||
| Negative | NoPiiCleanTicket | Passthrough with no detected PII |
|
|
||||||
|
|
||||||
Sample definitions live in [`SamplePromptCatalog.cs`](../src/PiiRedaction.ConsoleApp/Samples/SamplePromptCatalog.cs).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## High-Level Data Flow
|
|
||||||
|
|
||||||
The diagram below traces the canonical example from console input through Core sanitization to the Infrastructure LLM adapter. Data labels reflect the canonical strings at each stage.
|
|
||||||
|
|
||||||
```mermaid
|
|
||||||
flowchart TB
|
|
||||||
subgraph consoleApp [PiiRedaction.ConsoleApp]
|
|
||||||
program["Program.cs"]
|
|
||||||
di["ServiceRegistration"]
|
|
||||||
end
|
|
||||||
|
|
||||||
subgraph core [PiiRedaction.Core]
|
|
||||||
sanitizer["PromptSanitizer"]
|
|
||||||
composite["CompositePiiDetector"]
|
|
||||||
regexDet["RegexPiiDetector"]
|
|
||||||
domainDet["DomainRulePiiDetector"]
|
|
||||||
onnxDet["OnnxNerPiiDetector"]
|
|
||||||
redactor["PlaceholderPiiRedactor"]
|
|
||||||
end
|
|
||||||
|
|
||||||
subgraph infra [PiiRedaction.Infrastructure]
|
|
||||||
onnxRunner["OnnxNerModelRunner"]
|
|
||||||
mockLlm["MockLlmPromptService"]
|
|
||||||
mockChat["MockChatClient"]
|
|
||||||
end
|
|
||||||
|
|
||||||
rawPrompt["Raw prompt with PII"]
|
|
||||||
sanitizedPrompt["Sanitized prompt with placeholders"]
|
|
||||||
llmResponse["Mock LLM acknowledgment"]
|
|
||||||
|
|
||||||
program -->|"Customer Ravi Kumar ... PAN ABCDE1234F"| sanitizer
|
|
||||||
sanitizer --> composite
|
|
||||||
composite --> domainDet
|
|
||||||
composite --> regexDet
|
|
||||||
composite --> onnxDet
|
|
||||||
onnxDet --> onnxRunner
|
|
||||||
sanitizer --> redactor
|
|
||||||
redactor -->|"Customer PERSON_1 ... PAN PAN_1"| sanitizedPrompt
|
|
||||||
program -->|"SanitizedPrompt only"| mockLlm
|
|
||||||
mockLlm --> mockChat
|
|
||||||
mockChat --> llmResponse
|
|
||||||
|
|
||||||
rawPrompt -.-> program
|
|
||||||
di -.-> sanitizer
|
|
||||||
di -.-> mockLlm
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Detection to Redaction Detail
|
|
||||||
|
|
||||||
`PromptSanitizer` orchestrates a two-phase pipeline: **detect** then **redact**. `CompositePiiDetector` aggregates spans from all registered detectors, resolves overlaps by registration order and source priority, and returns a merged entity list. `PlaceholderPiiRedactor` replaces spans right-to-left to preserve indices, assigns stable per-type counters, and builds the in-process placeholder map.
|
|
||||||
|
|
||||||
```mermaid
|
|
||||||
flowchart LR
|
|
||||||
inputText["Original prompt text"]
|
|
||||||
|
|
||||||
subgraph detectPhase [Detection Phase]
|
|
||||||
domainDet["DomainRulePiiDetector"]
|
|
||||||
regexDet["RegexPiiDetector"]
|
|
||||||
onnxDet["OnnxNerPiiDetector"]
|
|
||||||
composite["CompositePiiDetector"]
|
|
||||||
merge["Overlap merge and source priority"]
|
|
||||||
entityList["PiiEntity list"]
|
|
||||||
end
|
|
||||||
|
|
||||||
subgraph redactPhase [Redaction Phase]
|
|
||||||
redactor["PlaceholderPiiRedactor"]
|
|
||||||
replace["Right-to-left span replacement"]
|
|
||||||
placeholderMap["Placeholder map in-process"]
|
|
||||||
sanitizedText["Sanitized text"]
|
|
||||||
end
|
|
||||||
|
|
||||||
inputText --> domainDet
|
|
||||||
inputText --> regexDet
|
|
||||||
inputText --> onnxDet
|
|
||||||
domainDet --> composite
|
|
||||||
regexDet --> composite
|
|
||||||
onnxDet --> composite
|
|
||||||
composite --> merge
|
|
||||||
merge --> entityList
|
|
||||||
entityList --> redactor
|
|
||||||
inputText --> redactor
|
|
||||||
redactor --> replace
|
|
||||||
replace --> sanitizedText
|
|
||||||
replace --> placeholderMap
|
|
||||||
```
|
|
||||||
|
|
||||||
**Overlap resolution rules** (applied by `CompositePiiDetector`):
|
|
||||||
|
|
||||||
1. Detectors run in registration order: **Domain → Regex → ONNX NER**.
|
|
||||||
2. On overlapping spans, the first registered detector wins.
|
|
||||||
3. Tie-breaking uses source priority: Domain (3) > Regex (2) > NER (1).
|
|
||||||
|
|
||||||
**Placeholder assignment** (applied by `PlaceholderPiiRedactor`):
|
|
||||||
|
|
||||||
- Format: `<{TYPE}_{n}>` (e.g. `<EMAIL_1>`, `<PERSON_1>`).
|
|
||||||
- Duplicate values of the same type reuse the same placeholder.
|
|
||||||
- Replacement proceeds from highest `StartIndex` to lowest to avoid index drift.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Runtime Sequence
|
|
||||||
|
|
||||||
```mermaid
|
|
||||||
sequenceDiagram
|
|
||||||
participant User
|
|
||||||
participant Program as Program.cs
|
|
||||||
participant DI as ServiceProvider
|
|
||||||
participant Sanitizer as PromptSanitizer
|
|
||||||
participant Detector as CompositePiiDetector
|
|
||||||
participant Redactor as PlaceholderPiiRedactor
|
|
||||||
participant LlmSvc as MockLlmPromptService
|
|
||||||
participant Chat as MockChatClient
|
|
||||||
|
|
||||||
User->>Program: Start application
|
|
||||||
Program->>DI: Resolve IPromptSanitizer, ILlmPromptService
|
|
||||||
DI-->>Program: Sanitizer, LlmService
|
|
||||||
|
|
||||||
alt Interactive mode
|
|
||||||
User->>Program: Enter prompt via console
|
|
||||||
else Default mode
|
|
||||||
Program->>Program: Load canonical sample prompt
|
|
||||||
end
|
|
||||||
|
|
||||||
Program->>Sanitizer: Sanitize(SanitizationRequest)
|
|
||||||
Sanitizer->>Detector: Detect(originalPrompt)
|
|
||||||
Detector-->>Sanitizer: IReadOnlyList PiiEntity
|
|
||||||
Sanitizer->>Redactor: Redact(originalPrompt, entities)
|
|
||||||
Redactor-->>Sanitizer: RedactionResult
|
|
||||||
Sanitizer-->>Program: SanitizationResult
|
|
||||||
|
|
||||||
Program->>Program: Display detected entities
|
|
||||||
Program->>Program: Display sanitized prompt
|
|
||||||
Program->>Program: Display placeholder map in-process
|
|
||||||
|
|
||||||
Program->>LlmSvc: SendPromptAsync(sanitizedPrompt)
|
|
||||||
Note over Program,LlmSvc: Placeholder map never passed
|
|
||||||
LlmSvc->>Chat: GetResponseAsync(user message)
|
|
||||||
Chat-->>LlmSvc: Assistant response
|
|
||||||
LlmSvc-->>Program: Mock LLM response string
|
|
||||||
Program->>User: Write LLM response to console
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Trust Boundary
|
|
||||||
|
|
||||||
The LLM boundary is the point at which data leaves the application process via `ILlmPromptService` / `IChatClient`. Only the sanitized prompt crosses this boundary. Original PII values, detection metadata, and the placeholder-to-value map remain in-process.
|
|
||||||
|
|
||||||
```mermaid
|
|
||||||
flowchart TB
|
|
||||||
subgraph inProcess [In-Process Trust Zone]
|
|
||||||
originalPrompt["Original prompt with raw PII"]
|
|
||||||
detectedEntities["Detected PiiEntity list"]
|
|
||||||
placeholderMap["Placeholder map"]
|
|
||||||
sanitizationResult["SanitizationResult"]
|
|
||||||
consoleDisplay["Console audit output"]
|
|
||||||
end
|
|
||||||
|
|
||||||
subgraph llmBoundary [LLM Trust Boundary]
|
|
||||||
sanitizedOnly["Sanitized prompt text only"]
|
|
||||||
end
|
|
||||||
|
|
||||||
subgraph externalLlm [External LLM Provider]
|
|
||||||
chatClient["IChatClient implementation"]
|
|
||||||
modelInference["Model inference"]
|
|
||||||
end
|
|
||||||
|
|
||||||
originalPrompt --> sanitizationResult
|
|
||||||
detectedEntities --> sanitizationResult
|
|
||||||
placeholderMap --> sanitizationResult
|
|
||||||
sanitizationResult --> consoleDisplay
|
|
||||||
sanitizationResult -->|"SendPromptAsync"| sanitizedOnly
|
|
||||||
sanitizedOnly --> chatClient
|
|
||||||
chatClient --> modelInference
|
|
||||||
|
|
||||||
originalPrompt -.-x|"Never transmitted"| chatClient
|
|
||||||
placeholderMap -.-x|"Never transmitted"| chatClient
|
|
||||||
detectedEntities -.-x|"Never transmitted"| chatClient
|
|
||||||
```
|
|
||||||
|
|
||||||
In the POC, `MockChatClient` simulates the external provider without network I/O. Replacing it with Azure OpenAI or another `IChatClient` implementation does not change the trust model: `MockLlmPromptService` (or a future production adapter) continues to accept only the sanitized string.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Project Responsibilities
|
|
||||||
|
|
||||||
| Project | Layer | Responsibility |
|
|
||||||
|---------|-------|----------------|
|
|
||||||
| `PiiRedaction.ConsoleApp` | Presentation | Application entry point; reads prompt (sample or interactive); bootstraps `IHost` and DI via `AddPiiRedactionServices`; orchestrates sanitization and LLM invocation; renders audit output (detected entities, sanitized text, placeholder map). |
|
|
||||||
| `PiiRedaction.Core` | Domain / Application | Defines abstractions (`IPiiDetector`, `IPiiRedactor`, `IPromptSanitizer`, `ILlmPromptService`); implements detection strategies (`RegexPiiDetector`, `DomainRulePiiDetector`, `OnnxNerPiiDetector`, `CompositePiiDetector`); implements `PlaceholderPiiRedactor` and `PromptSanitizer`; owns domain models (`PiiEntity`, `SanitizationResult`, `RedactionResult`) and configuration (`PiiRedactionOptions`). Has no dependency on ONNX Runtime or LLM SDKs. |
|
|
||||||
| `PiiRedaction.Infrastructure` | Infrastructure | Implements technical adapters: `OnnxNerModelRunner` (ONNX Runtime inference), `MockChatClient` and `MockLlmPromptService` (`Microsoft.Extensions.AI`); depends on Core abstractions and is swappable without changing domain logic. |
|
|
||||||
| `tests/PiiRedaction.Core.Tests` | Test | Unit and integration tests for detectors, redactor, sanitizer, overlap rules, golden prompt scenarios (`PromptScenarioCatalog`), and LLM boundary assertions. |
|
|
||||||
| `tests/PiiRedaction.Infrastructure.Tests` | Test | Tests for mock LLM behavior and ONNX runner load semantics. |
|
|
||||||
|
|
||||||
**Dependency direction:** `ConsoleApp` → `Infrastructure` → `Core`. Core references no outer layers, preserving the Dependency Inversion Principle and enabling future hosts (ASP.NET Core API, worker services) to reuse the same Core and Infrastructure assemblies.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Key Abstractions and Extension Points
|
|
||||||
|
|
||||||
| Abstraction | Defined In | Default Implementation | Extension |
|
|
||||||
|-------------|------------|------------------------|-----------|
|
|
||||||
| `IPiiDetector` | Core | `CompositePiiDetector` wrapping Domain, Regex, ONNX | Add new detector; register in composite order |
|
|
||||||
| `IPiiRedactor` | Core | `PlaceholderPiiRedactor` | Replace with hashing, vault-backed tokens, etc. |
|
|
||||||
| `IPromptSanitizer` | Core | `PromptSanitizer` | Unlikely to change; orchestrates detect + redact |
|
|
||||||
| `ILlmPromptService` | Core | `MockLlmPromptService` | Production adapter with telemetry, retry, policy |
|
|
||||||
| `IChatClient` | Microsoft.Extensions.AI | `MockChatClient` | Azure OpenAI, OpenAI, or other provider SDK |
|
|
||||||
| `IOnnxNerModelRunner` | Core | `OnnxNerModelRunner` | BERT WordPiece tokenization, ONNX inference, BIO label decoding |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Configuration Surface
|
|
||||||
|
|
||||||
Runtime behavior is controlled via `appsettings.json` under the `PiiRedaction` section:
|
|
||||||
|
|
||||||
| Setting | Effect |
|
|
||||||
|---------|--------|
|
|
||||||
| `OnnxModelPath` | Path to ONNX NER model (`models/ner-model.onnx` by default). Companion files `vocab.txt` and `ner-labels.txt` must live in the same directory. |
|
|
||||||
|
|
||||||
Download the model assets with `scripts/download-ner-model.ps1` (exports `dslim/bert-base-NER`).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Related Documentation
|
|
||||||
|
|
||||||
- [README](../README.md) — build, run, configuration, and testing instructions
|
|
||||||
- [ServiceCollectionExtensions.cs](../src/PiiRedaction.ConsoleApp/DependencyInjection/ServiceCollectionExtensions.cs) — DI registration and detector ordering
|
|
||||||
- [PromptScenarioCatalog.cs](../tests/PiiRedaction.Core.Tests/TestSupport/PromptScenarioCatalog.cs) — focused golden pipeline scenarios including the canonical example
|
|
||||||
@@ -1,62 +0,0 @@
|
|||||||
# Git setup — Xenovex (xts.xenovex.com)
|
|
||||||
|
|
||||||
This repository is ready for push to your Xenovex Git server after you create a remote repository.
|
|
||||||
|
|
||||||
## Prerequisites
|
|
||||||
|
|
||||||
- Git 2.x (installed at `C:\Program Files\Git\bin\git.exe`)
|
|
||||||
- Access to https://xts.xenovex.com/explore/repos
|
|
||||||
- .NET 10 SDK for build/test
|
|
||||||
|
|
||||||
## 1. Create the remote repository
|
|
||||||
|
|
||||||
1. Sign in to **https://xts.xenovex.com**
|
|
||||||
2. Open **Explore repos** (or **New repository**)
|
|
||||||
3. Create a new empty repository, e.g. `llm-pii-poc`
|
|
||||||
4. Copy the **HTTPS** or **SSH** clone URL (example shapes):
|
|
||||||
- `https://xts.xenovex.com/<org-or-user>/llm-pii-poc.git`
|
|
||||||
- `git@xts.xenovex.com:<org-or-user>/llm-pii-poc.git`
|
|
||||||
|
|
||||||
Do **not** initialize the remote with a README if you are pushing an existing local history.
|
|
||||||
|
|
||||||
## 2. Add remote and push (from repository root)
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
cd C:\Users\bilal.n\Projects\llm-pii-poc
|
|
||||||
|
|
||||||
# Use full path if git is not on PATH
|
|
||||||
$git = "C:\Program Files\Git\bin\git.exe"
|
|
||||||
|
|
||||||
& $git remote add origin <YOUR_CLONE_URL>
|
|
||||||
& $git branch -M main
|
|
||||||
& $git push -u origin main
|
|
||||||
```
|
|
||||||
|
|
||||||
If the remote already has commits (e.g. auto-generated README), either use an empty remote or:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
& $git pull origin main --rebase
|
|
||||||
& $git push -u origin main
|
|
||||||
```
|
|
||||||
|
|
||||||
## 3. What is committed vs excluded
|
|
||||||
|
|
||||||
| Included | Excluded (`.gitignore`) |
|
|
||||||
|----------|-------------------------|
|
|
||||||
| Source (`src/`), tests, scripts, docs | `bin/`, `obj/`, `.vs/` |
|
|
||||||
| `README.md`, `PiiRedaction.slnx` | `models/*.onnx`, `vocab.txt`, `ner-labels.txt` (~431MB model) |
|
|
||||||
| `models/.gitkeep` (empty models folder) | `scratch/` |
|
|
||||||
|
|
||||||
After clone, download the NER model locally:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
.\scripts\download-ner-model.ps1
|
|
||||||
```
|
|
||||||
|
|
||||||
## 4. Verify after clone
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
dotnet build
|
|
||||||
dotnet test
|
|
||||||
dotnet run --project src/PiiRedaction.ConsoleApp -- --sample 0
|
|
||||||
```
|
|
||||||
557
docs/solution-guide.md
Normal file
557
docs/solution-guide.md
Normal file
@@ -0,0 +1,557 @@
|
|||||||
|
# PII Redaction POC — Solution Guide
|
||||||
|
|
||||||
|
Single reference for architecture, NER models, routing, operations, Tamil/Tanglish support, Git setup, and the post-POC improvement backlog.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Table of contents
|
||||||
|
|
||||||
|
1. [Purpose](#1-purpose)
|
||||||
|
2. [Canonical example](#2-canonical-example)
|
||||||
|
3. [Detection strategies](#3-detection-strategies)
|
||||||
|
4. [Solution architecture](#4-solution-architecture)
|
||||||
|
5. [Trust boundary](#5-trust-boundary)
|
||||||
|
6. [NER models](#6-ner-models)
|
||||||
|
7. [NER routing — English vs Tamil](#7-ner-routing--english-vs-tamil)
|
||||||
|
8. [Configuration and dependency injection](#8-configuration-and-dependency-injection)
|
||||||
|
9. [Running and testing](#9-running-and-testing)
|
||||||
|
10. [Tamil and Tanglish support](#10-tamil-and-tanglish-support)
|
||||||
|
11. [Git remote setup (Xenovex)](#11-git-remote-setup-xenovex)
|
||||||
|
12. [Improvement roadmap](#12-improvement-roadmap)
|
||||||
|
13. [Key source files](#13-key-source-files)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Purpose
|
||||||
|
|
||||||
|
This .NET proof-of-concept intercepts user prompts containing regulated personally identifiable information (PII), redacts sensitive values into stable placeholders, and transmits **only sanitized text** across the LLM trust boundary.
|
||||||
|
|
||||||
|
The solution uses:
|
||||||
|
|
||||||
|
- Clear layer separation (Core / Infrastructure / hosts)
|
||||||
|
- Interface-driven composition and dependency injection
|
||||||
|
- Swappable ONNX NER adapters and `Microsoft.Extensions.AI` chat clients
|
||||||
|
|
||||||
|
It targets financial and customer-service workloads where raw PII must not leave the application process when invoking external language models.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Canonical example
|
||||||
|
|
||||||
|
With English and Tamil ONNX models loaded (`scripts/download-ner-model.ps1`, `scripts/download-tamil-ner-model.ps1`):
|
||||||
|
|
||||||
|
| Stage | Value |
|
||||||
|
|-------|-------|
|
||||||
|
| **Input** | `Customer Ravi Kumar with email ravi.kumar@gmail.com and phone 9876543210 has LoanNumber LN-456789 and PAN ABCDE1234F. Please summarize this customer issue.` |
|
||||||
|
| **Sanitized output** | `Customer <PERSON_1> with email <EMAIL_1> and phone <PHONE_1> has LoanNumber <LOAN_NUMBER_1> and PAN <PAN_1>. Please summarize this customer issue.` |
|
||||||
|
| **Mock LLM response** | `[Mock LLM Response] Received sanitized prompt (146 chars). No original PII was transmitted.` |
|
||||||
|
|
||||||
|
| Type | Value | Source |
|
||||||
|
|------|-------|--------|
|
||||||
|
| PERSON | Ravi Kumar | NER |
|
||||||
|
| EMAIL | ravi.kumar@gmail.com | Regex |
|
||||||
|
| PHONE | 9876543210 | Regex |
|
||||||
|
| LOAN_NUMBER | LN-456789 | Domain |
|
||||||
|
| PAN | ABCDE1234F | Regex |
|
||||||
|
|
||||||
|
The placeholder map (`<PERSON_1>` → `Ravi Kumar`, etc.) stays **in-process** and is never sent to the LLM.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Detection strategies
|
||||||
|
|
||||||
|
| Strategy | Detects | Rationale |
|
||||||
|
|----------|---------|-----------|
|
||||||
|
| **Regex** | Email, phone, PAN, Aadhaar, credit card | Deterministic, format-bound, auditable |
|
||||||
|
| **ONNX NER** | Person names | Contextual; no rigid format |
|
||||||
|
| **Domain rules** | Loan number (`LN-`), customer ID (`CID-`), account (`ACC-`) | Business-specific identifiers |
|
||||||
|
|
||||||
|
**Overlap resolution** (`CompositePiiDetector`): detectors run in order **Domain → Regex → NER**. On overlapping spans, candidates are sorted by start index, length, and source priority (**Domain 3 > Regex 2 > NER 1**); the first non-overlapping candidate wins.
|
||||||
|
|
||||||
|
**Redaction** (`PlaceholderPiiRedactor`): format `<{TYPE}_{n}>`; duplicate values reuse placeholders; replacement is right-to-left to preserve indices.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Solution architecture
|
||||||
|
|
||||||
|
### Project structure
|
||||||
|
|
||||||
|
```
|
||||||
|
src/
|
||||||
|
├── PiiRedaction.ConsoleApp/ # Console demo, DI bootstrap
|
||||||
|
├── PiiRedaction.TestHarness.Wpf/ # WPF MVVM manual test harness
|
||||||
|
├── PiiRedaction.Core/ # Detection, redaction, abstractions
|
||||||
|
└── PiiRedaction.Infrastructure/ # ONNX runners, mock LLM
|
||||||
|
models/ # ONNX assets (gitignored)
|
||||||
|
tests/ # NUnit unit and integration tests
|
||||||
|
```
|
||||||
|
|
||||||
|
| Project | Layer | Responsibility |
|
||||||
|
|---------|-------|----------------|
|
||||||
|
| `PiiRedaction.ConsoleApp` | Presentation | Samples, interactive mode, audit output |
|
||||||
|
| `PiiRedaction.TestHarness.Wpf` | Presentation | Category-filtered prompts, redact UI, batch runner |
|
||||||
|
| `PiiRedaction.Core` | Domain | `IPiiDetector`, `IPromptSanitizer`, detectors, models |
|
||||||
|
| `PiiRedaction.Infrastructure` | Infrastructure | `RoutingOnnxNerModelRunner`, `MockLlmPromptService` |
|
||||||
|
| `tests/*` | Test | ~112 tests; `RealModel` and `TamilNer` categories |
|
||||||
|
|
||||||
|
**Dependency direction:** `ConsoleApp` / `Wpf` → `Infrastructure` → `Core`. Core has no ONNX or LLM SDK references.
|
||||||
|
|
||||||
|
### Key abstractions
|
||||||
|
|
||||||
|
| Abstraction | Default implementation | Extension |
|
||||||
|
|-------------|------------------------|-----------|
|
||||||
|
| `IPiiDetector` | `CompositePiiDetector` | Add detector; register in composite order |
|
||||||
|
| `IPiiRedactor` | `PlaceholderPiiRedactor` | Hashing, vault tokens |
|
||||||
|
| `IPromptSanitizer` | `PromptSanitizer` | Orchestrates detect + redact |
|
||||||
|
| `IOnnxNerModelRunner` | `RoutingOnnxNerModelRunner` | Script-based EN/TA routing |
|
||||||
|
| `ILlmPromptService` | `MockLlmPromptService` | Production adapter |
|
||||||
|
| `IChatClient` | `MockChatClient` | Azure OpenAI, etc. |
|
||||||
|
|
||||||
|
### Data flow
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart TB
|
||||||
|
subgraph hosts [Hosts]
|
||||||
|
console[ConsoleApp / WpfHarness]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph core [PiiRedaction.Core]
|
||||||
|
sanitizer[PromptSanitizer]
|
||||||
|
composite[CompositePiiDetector]
|
||||||
|
domain[DomainRulePiiDetector]
|
||||||
|
regex[RegexPiiDetector]
|
||||||
|
onnxDet[OnnxNerPiiDetector]
|
||||||
|
redactor[PlaceholderPiiRedactor]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph infra [PiiRedaction.Infrastructure]
|
||||||
|
router[RoutingOnnxNerModelRunner]
|
||||||
|
en[EnglishOnnxNerRunner]
|
||||||
|
ta[TamilOnnxNerRunner]
|
||||||
|
llm[MockLlmPromptService]
|
||||||
|
end
|
||||||
|
|
||||||
|
console --> sanitizer
|
||||||
|
sanitizer --> composite
|
||||||
|
composite --> domain
|
||||||
|
composite --> regex
|
||||||
|
composite --> onnxDet
|
||||||
|
onnxDet --> router
|
||||||
|
router --> en
|
||||||
|
router --> ta
|
||||||
|
sanitizer --> redactor
|
||||||
|
console -->|"sanitized text only"| llm
|
||||||
|
```
|
||||||
|
|
||||||
|
**Pipeline:** `Sanitize` → `Detect` (all detectors) → `Redact` → `SanitizationResult`. Optional: `SendPromptAsync(sanitizedPrompt)` to LLM.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Trust boundary
|
||||||
|
|
||||||
|
Only the **sanitized prompt string** crosses `ILlmPromptService` / `IChatClient`. Original PII, entity metadata, and the placeholder map remain in-process.
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart LR
|
||||||
|
subgraph inProcess [In-Process]
|
||||||
|
raw[Original prompt]
|
||||||
|
entities[Detected entities]
|
||||||
|
map[Placeholder map]
|
||||||
|
audit[Console / WPF display]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph boundary [LLM boundary]
|
||||||
|
sanitized[Sanitized prompt only]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph external [External LLM]
|
||||||
|
chat[IChatClient]
|
||||||
|
end
|
||||||
|
|
||||||
|
raw --> audit
|
||||||
|
entities --> audit
|
||||||
|
map --> audit
|
||||||
|
sanitized --> chat
|
||||||
|
raw -.->|never sent| chat
|
||||||
|
map -.->|never sent| chat
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. NER models
|
||||||
|
|
||||||
|
Person-name detection is the **only** NER responsibility. Structured PII uses regex and domain rules.
|
||||||
|
|
||||||
|
### Routing summary
|
||||||
|
|
||||||
|
| Script in prompt | Model(s) | Typical use |
|
||||||
|
|------------------|----------|-------------|
|
||||||
|
| `LatinOnly` | English | English names, Indian names in Roman script, **Tanglish** |
|
||||||
|
| `TamilOnly` | Tamil | Tamil-script names |
|
||||||
|
| `Mixed` | Both; merge spans | Code-mixed prompts |
|
||||||
|
| `NoLetters` | Neither | Digits/symbols only |
|
||||||
|
|
||||||
|
### English model
|
||||||
|
|
||||||
|
| Property | Value |
|
||||||
|
|----------|-------|
|
||||||
|
| Hugging Face ID | [`dslim/bert-base-NER`](https://huggingface.co/dslim/bert-base-NER) |
|
||||||
|
| Tokenizer | WordPiece (`vocab.txt`, `BertWordPieceEncoder`) |
|
||||||
|
| Person labels | `B-PER`, `I-PER`, `B-PERSON`, `I-PERSON` |
|
||||||
|
| Primary path | `models/en/ner-model.onnx` |
|
||||||
|
| Legacy fallback | `models/ner-model.onnx` |
|
||||||
|
| Download | `.\scripts\download-ner-model.ps1` |
|
||||||
|
|
||||||
|
### Tamil model
|
||||||
|
|
||||||
|
| Property | Value |
|
||||||
|
|----------|-------|
|
||||||
|
| Hugging Face ID | [`prachuryyaIITG/SampurNER_Tamil_IndicBERTv2`](https://huggingface.co/prachuryyaIITG/SampurNER_Tamil_IndicBERTv2) |
|
||||||
|
| Tokenizer | WordPiece when `vocab.txt` present (export path); SentencePiece fallback |
|
||||||
|
| Person labels | Any BIO tag containing `person` (case-insensitive) |
|
||||||
|
| Path | `models/ta/model.onnx` |
|
||||||
|
| Download | `.\scripts\download-tamil-ner-model.ps1` |
|
||||||
|
|
||||||
|
**Why SampurNER over MuRIL:** Tamil-specific NER training, smaller footprint (~0.3B vs ~0.6B), validated ONNX export in this repo. MuRIL reserved as eval-driven fallback.
|
||||||
|
|
||||||
|
### Model assets (gitignored)
|
||||||
|
|
||||||
|
| Directory | Key files | Approx. size |
|
||||||
|
|-----------|-----------|--------------|
|
||||||
|
| `models/en/` | `ner-model.onnx`, `vocab.txt`, `ner-labels.txt` | ~431 MB ONNX |
|
||||||
|
| `models/ta/` | `model.onnx`, `vocab.txt`, `ner-labels.txt` | ~1 GB ONNX |
|
||||||
|
|
||||||
|
Only `models/en/.gitkeep` and `models/ta/.gitkeep` are committed.
|
||||||
|
|
||||||
|
### Shared inference
|
||||||
|
|
||||||
|
Both runners use `OnnxTokenClassifierRunner`:
|
||||||
|
|
||||||
|
- Max sequence length: **128 tokens** (long prompts truncate silently)
|
||||||
|
- BIO decode → `PiiEntityType.Person` only
|
||||||
|
- Missing model → fail-open: `[]` from NER (person names not redacted)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. NER routing — English vs Tamil
|
||||||
|
|
||||||
|
### Call chain
|
||||||
|
|
||||||
|
```
|
||||||
|
PromptSanitizer → CompositePiiDetector → OnnxNerPiiDetector
|
||||||
|
→ RoutingOnnxNerModelRunner.PredictEntities()
|
||||||
|
→ ScriptRouter.GetComposition(text)
|
||||||
|
→ switch (composition) { English / Tamil / both }
|
||||||
|
→ MergePersonSpans()
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 1: `ScriptRouter` (Core)
|
||||||
|
|
||||||
|
**File:** `src/PiiRedaction.Core/Detection/ScriptRouter.cs`
|
||||||
|
|
||||||
|
Single pass over characters:
|
||||||
|
|
||||||
|
- Tamil letter: Unicode **U+0B80 – U+0BFF**
|
||||||
|
- Latin letter: `char.IsAsciiLetter`
|
||||||
|
- Both seen → `Mixed` (early exit)
|
||||||
|
- Neither → `NoLetters`
|
||||||
|
- Otherwise → `TamilOnly` or `LatinOnly`
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
// ScriptComposition enum: LatinOnly, TamilOnly, Mixed, NoLetters
|
||||||
|
public ScriptComposition GetComposition(string text) { /* scan chars */ }
|
||||||
|
```
|
||||||
|
|
||||||
|
**Tanglish** in Roman script → `LatinOnly` → **English model only**.
|
||||||
|
|
||||||
|
### Step 2: `RoutingOnnxNerModelRunner` (Infrastructure)
|
||||||
|
|
||||||
|
**File:** `src/PiiRedaction.Infrastructure/Onnx/RoutingOnnxNerModelRunner.cs`
|
||||||
|
|
||||||
|
This is the **branch decision**:
|
||||||
|
|
||||||
|
| `ScriptComposition` | English | Tamil |
|
||||||
|
|---------------------|---------|-------|
|
||||||
|
| `LatinOnly` | Yes if available | No |
|
||||||
|
| `TamilOnly` | No | Yes if `EnableTamilNer` and available |
|
||||||
|
| `Mixed` | Yes if available | Yes if `EnableTamilNer` and available |
|
||||||
|
| `NoLetters` | No | No |
|
||||||
|
|
||||||
|
Both models run on the **full prompt text** for `Mixed` (no script segmentation).
|
||||||
|
|
||||||
|
### Step 3: Merge
|
||||||
|
|
||||||
|
`MergePersonSpans`: overlapping PERSON spans → **longer span wins**; ordered by `StartIndex`.
|
||||||
|
|
||||||
|
### Routing diagram
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart TD
|
||||||
|
text[Prompt text] --> sr[ScriptRouter]
|
||||||
|
sr --> latin[LatinOnly]
|
||||||
|
sr --> tamil[TamilOnly]
|
||||||
|
sr --> mixed[Mixed]
|
||||||
|
sr --> none[NoLetters]
|
||||||
|
latin --> en[EnglishOnnxNerRunner]
|
||||||
|
tamil --> ta[TamilOnnxNerRunner]
|
||||||
|
mixed --> en
|
||||||
|
mixed --> ta
|
||||||
|
en --> merge[MergePersonSpans]
|
||||||
|
ta --> merge
|
||||||
|
none --> empty[No NER]
|
||||||
|
```
|
||||||
|
|
||||||
|
### Worked examples
|
||||||
|
|
||||||
|
| Input style | Composition | Models |
|
||||||
|
|-------------|---------------|--------|
|
||||||
|
| `Customer Ravi Kumar…` | `LatinOnly` | English |
|
||||||
|
| `வாடிக்கையாளர் ராஜேஷ் குமார்…` | `TamilOnly` | Tamil |
|
||||||
|
| `Naan Suresh, phone 9003789456…` | `LatinOnly` | English (Tanglish) |
|
||||||
|
| `வாடிக்கையாளர் Ravi Kumar phone…` | `Mixed` | Both |
|
||||||
|
| `Callback on 9123456780…` | `NoLetters` | Neither (phone via Regex) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Configuration and dependency injection
|
||||||
|
|
||||||
|
### `appsettings.json`
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"PiiRedaction": {
|
||||||
|
"OnnxModelPath": "models/ner-model.onnx",
|
||||||
|
"EnglishOnnxModelPath": "models/en/ner-model.onnx",
|
||||||
|
"TamilOnnxModelPath": "models/ta/model.onnx",
|
||||||
|
"EnableTamilNer": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
| Setting | Effect |
|
||||||
|
|---------|--------|
|
||||||
|
| `EnglishOnnxModelPath` | Primary English model |
|
||||||
|
| `OnnxModelPath` | Legacy English fallback |
|
||||||
|
| `TamilOnnxModelPath` | Tamil model |
|
||||||
|
| `EnableTamilNer` | `false` = English-only routing |
|
||||||
|
|
||||||
|
### DI registration
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
services.AddSingleton<EnglishOnnxNerRunner>();
|
||||||
|
services.AddSingleton<TamilOnnxNerRunner>();
|
||||||
|
services.AddSingleton<IOnnxNerModelRunner, RoutingOnnxNerModelRunner>();
|
||||||
|
// OnnxNerPiiDetector receives IOnnxNerModelRunner (the router)
|
||||||
|
```
|
||||||
|
|
||||||
|
Same pattern in `PiiRedaction.ConsoleApp` and `PiiRedaction.TestHarness.Wpf` `ServiceCollectionExtensions.cs`.
|
||||||
|
|
||||||
|
Detector registration order in composite: **Domain → Regex → ONNX NER**.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Running and testing
|
||||||
|
|
||||||
|
### Build
|
||||||
|
|
||||||
|
```bash
|
||||||
|
dotnet restore
|
||||||
|
dotnet build
|
||||||
|
```
|
||||||
|
|
||||||
|
### Console app
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# All 16 samples (English + Tamil/Tanglish/mixed)
|
||||||
|
dotnet run --project src/PiiRedaction.ConsoleApp
|
||||||
|
|
||||||
|
dotnet run --project src/PiiRedaction.ConsoleApp -- --list
|
||||||
|
dotnet run --project src/PiiRedaction.ConsoleApp -- --name TamilCustomerNameOnly
|
||||||
|
dotnet run --project src/PiiRedaction.ConsoleApp -- --interactive
|
||||||
|
```
|
||||||
|
|
||||||
|
Samples: `src/PiiRedaction.ConsoleApp/Samples/SamplePromptCatalog.cs`
|
||||||
|
|
||||||
|
### WPF test harness (Windows)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
dotnet run --project src/PiiRedaction.TestHarness.Wpf
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Category dropdown:** Career Guidance, Banking & Financial, Negative, Edge & Harness
|
||||||
|
- Click prompt → loads input; **Redact** runs pipeline
|
||||||
|
- **Run All** executes scenarios in the selected category
|
||||||
|
- Script badge shows predicted routing (`LatinOnly`, `TamilOnly`, `Mixed`)
|
||||||
|
|
||||||
|
### Download models
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
.\scripts\download-ner-model.ps1
|
||||||
|
.\scripts\download-tamil-ner-model.ps1
|
||||||
|
```
|
||||||
|
|
||||||
|
If Tamil PowerShell download 404s, use Python 3.12+:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
.\scripts\download-tamil-ner-model.ps1 -Python "$env:LOCALAPPDATA\Programs\Python\Python312\python.exe"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Tests
|
||||||
|
|
||||||
|
```bash
|
||||||
|
dotnet test
|
||||||
|
dotnet test --filter "Category=RealModel"
|
||||||
|
dotnet test --filter "Category=TamilNer"
|
||||||
|
dotnet test --filter "FullyQualifiedName~ScriptRouterTests|FullyQualifiedName~RoutingOnnxNerModelRunnerTests"
|
||||||
|
```
|
||||||
|
|
||||||
|
Tests skip gracefully when ONNX files are absent.
|
||||||
|
|
||||||
|
### Console sample categories
|
||||||
|
|
||||||
|
| Category | Examples |
|
||||||
|
|----------|----------|
|
||||||
|
| NER (English) | `CustomerNameOnly`, `MrTitlePerson`, `TwoCustomersInOnePrompt` |
|
||||||
|
| NER (Tamil) | `TamilCustomerNameOnly`, `TamilFullFinancial` |
|
||||||
|
| Tanglish / Mixed | `TanglishCustomer`, `MixedTamilEnglish` |
|
||||||
|
| Regex / Domain | `AllRegexTypes`, `AllDomainIds` |
|
||||||
|
| Negative | `NoPiiCleanTicket` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Tamil and Tanglish support
|
||||||
|
|
||||||
|
### Implementation status
|
||||||
|
|
||||||
|
| Phase | Status | Scope |
|
||||||
|
|-------|--------|-------|
|
||||||
|
| **1** Generic ONNX token classifier | Done | `OnnxTokenClassifierRunner`, encoders, `NerLabelConfig` |
|
||||||
|
| **2** Tamil download + config | Done | Scripts, dual paths, `EnableTamilNer` |
|
||||||
|
| **3** Script routing + DI | Done | `ScriptRouter`, `RoutingOnnxNerModelRunner` |
|
||||||
|
| **4** Tests, samples, docs | Partial | Tests/samples done; eval metrics open |
|
||||||
|
| **5** Optional enhancements | Not started | See below |
|
||||||
|
|
||||||
|
### Remaining gaps (Phase 4–5)
|
||||||
|
|
||||||
|
| Gap | Impact |
|
||||||
|
|-----|--------|
|
||||||
|
| Tamil numeral normalization (௦–௯ → 0–9) | May miss phone/Aadhaar in Tamil script |
|
||||||
|
| Tanglish heuristics (`peru`, `enga peru`) | Better Latin-name recall in Tamil context |
|
||||||
|
| Label-aware regex cues | Contextual name detection |
|
||||||
|
| Fail-closed when NER missing | Compliance hardening |
|
||||||
|
| MuRIL model swap | If Tamil recall insufficient |
|
||||||
|
|
||||||
|
### Tanglish expectations
|
||||||
|
|
||||||
|
| Input | Handler | Expected recall |
|
||||||
|
|-------|---------|-----------------|
|
||||||
|
| Tamil script names | Tamil ONNX | High (with tuning) |
|
||||||
|
| Latin Indian names (`Ravi Kumar`) | English ONNX | High |
|
||||||
|
| Tanglish spellings (`Senthil`) | English NER + optional heuristics | Medium |
|
||||||
|
| Code-mixed prompts | Both models + merge | Medium–high for IDs; names variable |
|
||||||
|
|
||||||
|
### Success metrics (eval set target)
|
||||||
|
|
||||||
|
| Metric | MVP target |
|
||||||
|
|--------|------------|
|
||||||
|
| Tamil script person recall | ≥ 85% |
|
||||||
|
| Tanglish person recall | ≥ 70% |
|
||||||
|
| False positives on clean prompts | ≤ 5% |
|
||||||
|
| Structured PII in Tamil prompts (regex) | ≥ 95% |
|
||||||
|
| English canonical regression | 100% |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11. Git remote setup (Xenovex)
|
||||||
|
|
||||||
|
Remote: `https://xts.xenovex.com/Bilal-Nazer-Ali/llm-pii-poc.git`
|
||||||
|
|
||||||
|
### Create and push
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
cd C:\Users\bilal.n\Projects\llm-pii-poc
|
||||||
|
$git = "C:\Program Files\Git\bin\git.exe"
|
||||||
|
|
||||||
|
& $git remote add origin <YOUR_CLONE_URL>
|
||||||
|
& $git branch -M main
|
||||||
|
& $git push -u origin main
|
||||||
|
```
|
||||||
|
|
||||||
|
Use an **empty** remote repository (no README) when pushing existing history.
|
||||||
|
|
||||||
|
### Committed vs gitignored
|
||||||
|
|
||||||
|
| Committed | Gitignored |
|
||||||
|
|-----------|------------|
|
||||||
|
| `src/`, `tests/`, `scripts/`, `docs/` | `bin/`, `obj/`, `.vs/` |
|
||||||
|
| `README.md`, `PiiRedaction.slnx` | `models/*.onnx`, `vocab.txt`, `models/en/*`, `models/ta/*` |
|
||||||
|
| `models/en/.gitkeep`, `models/ta/.gitkeep` | `scratch/` |
|
||||||
|
|
||||||
|
After clone, run model download scripts locally.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 12. Improvement roadmap
|
||||||
|
|
||||||
|
Post-POC items **not yet implemented**. Current maturity: strong architecture and tests; production needs fail-closed policy, API host, and observability.
|
||||||
|
|
||||||
|
### P0 — Security and correctness
|
||||||
|
|
||||||
|
| Item | Proposal |
|
||||||
|
|------|----------|
|
||||||
|
| Fail-closed when NER unavailable | `RequireNerOnStartup`, `BlockLlmWhenNerUnavailable` options |
|
||||||
|
| Outbound LLM guard | Verify sanitized text before `IChatClient` |
|
||||||
|
| Truncation warning | Surface 128-token limit in `SanitizationResult` |
|
||||||
|
| NER confidence | Populate `PiiEntity.Confidence` or hide UI column |
|
||||||
|
| WPF leak check | Use placeholder map, not naive `Contains` |
|
||||||
|
|
||||||
|
### P1 — Platform
|
||||||
|
|
||||||
|
| Item | Proposal |
|
||||||
|
|------|----------|
|
||||||
|
| `PiiRedaction.Composition` | Shared `AddPiiRedactionServices` (Console + WPF duplicate today) |
|
||||||
|
| Unified prompt catalog | Single source for console, WPF, golden tests |
|
||||||
|
| Async `IPromptSanitizer` | Replace WPF `Task.Run` wrapper |
|
||||||
|
| `PiiRedaction.Application` | Shared orchestration for API/WPF |
|
||||||
|
| Minimal API + health checks | `POST /v1/prompts/sanitize`, model readiness |
|
||||||
|
| Tamil Phase 4 | Numeral normalization, Tanglish heuristics |
|
||||||
|
|
||||||
|
### P2 — Operations
|
||||||
|
|
||||||
|
| Item | Proposal |
|
||||||
|
|------|----------|
|
||||||
|
| Placeholder audit store | TTL, encryption, correlation ID |
|
||||||
|
| Observability | Metrics, OpenTelemetry traces |
|
||||||
|
| ONNX session pool | Concurrency strategy for API load |
|
||||||
|
| CI pipeline | Fast tests without models; nightly real-model job |
|
||||||
|
|
||||||
|
### Suggested phases
|
||||||
|
|
||||||
|
1. **Production hardening** — fail-closed, composition root, API, truncation metadata (~1–2 weeks)
|
||||||
|
2. **Detection quality** — normalizer, Tamil Phase 4, regex hardening (~1 week)
|
||||||
|
3. **Operations** — audit store, metrics, session pool (~1–2 weeks)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 13. Key source files
|
||||||
|
|
||||||
|
| Topic | Path |
|
||||||
|
|-------|------|
|
||||||
|
| Script classification | `src/PiiRedaction.Core/Detection/ScriptRouter.cs` |
|
||||||
|
| **Model branch decision** | `src/PiiRedaction.Infrastructure/Onnx/RoutingOnnxNerModelRunner.cs` |
|
||||||
|
| English NER | `src/PiiRedaction.Infrastructure/Onnx/EnglishOnnxNerRunner.cs` |
|
||||||
|
| Tamil NER | `src/PiiRedaction.Infrastructure/Onnx/TamilOnnxNerRunner.cs` |
|
||||||
|
| Shared ONNX inference | `src/PiiRedaction.Infrastructure/Onnx/OnnxTokenClassifierRunner.cs` |
|
||||||
|
| NER detector | `src/PiiRedaction.Core/Detection/OnnxNerPiiDetector.cs` |
|
||||||
|
| Sanitizer | `src/PiiRedaction.Core/Sanitization/PromptSanitizer.cs` |
|
||||||
|
| Composite merge | `src/PiiRedaction.Core/Detection/CompositePiiDetector.cs` |
|
||||||
|
| Options | `src/PiiRedaction.Core/Configuration/PiiRedactionOptions.cs` |
|
||||||
|
| DI | `src/PiiRedaction.ConsoleApp/DependencyInjection/ServiceCollectionExtensions.cs` |
|
||||||
|
| Console samples | `src/PiiRedaction.ConsoleApp/Samples/SamplePromptCatalog.cs` |
|
||||||
|
| WPF catalog | `src/PiiRedaction.TestHarness.Wpf/Services/TestPromptCatalog.cs` |
|
||||||
|
| Routing tests | `tests/PiiRedaction.Infrastructure.Tests/Onnx/RoutingOnnxNerModelRunnerTests.cs` |
|
||||||
|
| Script tests | `tests/PiiRedaction.Core.Tests/Detection/ScriptRouterTests.cs` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
*Last consolidated: July 2026. Replaces separate architecture, NER models, routing reference, Tamil plan, improvement roadmap, and Git setup documents.*
|
||||||
0
models/en/.gitkeep
Normal file
0
models/en/.gitkeep
Normal file
0
models/ta/.gitkeep
Normal file
0
models/ta/.gitkeep
Normal file
229
scripts/download-tamil-ner-model.ps1
Normal file
229
scripts/download-tamil-ner-model.ps1
Normal file
@@ -0,0 +1,229 @@
|
|||||||
|
# Downloads prachuryyaIITG/SampurNER_Tamil_IndicBERTv2 ONNX assets to models/ta/ for the PII Redaction POC.
|
||||||
|
param(
|
||||||
|
[string]$Python = "python"
|
||||||
|
)
|
||||||
|
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
$repoRoot = Split-Path -Parent $PSScriptRoot
|
||||||
|
$modelsDir = Join-Path (Join-Path $repoRoot "models") "ta"
|
||||||
|
$scriptPath = Join-Path $PSScriptRoot "download-tamil-ner-model.py"
|
||||||
|
$modelId = "prachuryyaIITG/SampurNER_Tamil_IndicBERTv2"
|
||||||
|
$baseUrl = "https://huggingface.co/$modelId/resolve/main"
|
||||||
|
|
||||||
|
function Ensure-ModelsDirectory {
|
||||||
|
New-Item -ItemType Directory -Force -Path $modelsDir | Out-Null
|
||||||
|
}
|
||||||
|
|
||||||
|
function Download-HuggingFaceAsset {
|
||||||
|
param(
|
||||||
|
[string]$RelativePath,
|
||||||
|
[string]$Destination
|
||||||
|
)
|
||||||
|
|
||||||
|
$url = "$baseUrl/$RelativePath"
|
||||||
|
Write-Host "Downloading $url"
|
||||||
|
Invoke-WebRequest -Uri $url -OutFile $Destination -UseBasicParsing
|
||||||
|
}
|
||||||
|
|
||||||
|
function Export-LabelsFromConfig {
|
||||||
|
param([string]$ConfigPath, [string]$LabelsPath)
|
||||||
|
|
||||||
|
$config = Get-Content $ConfigPath -Raw | ConvertFrom-Json
|
||||||
|
$labelMap = @{}
|
||||||
|
foreach ($property in $config.id2label.PSObject.Properties) {
|
||||||
|
$labelMap[[int]$property.Name] = [string]$property.Value
|
||||||
|
}
|
||||||
|
|
||||||
|
$labels = for ($index = 0; $index -lt $labelMap.Count; $index++) {
|
||||||
|
$labelMap[$index]
|
||||||
|
}
|
||||||
|
|
||||||
|
$labels | Set-Content -Path $LabelsPath -Encoding utf8
|
||||||
|
}
|
||||||
|
|
||||||
|
function Resolve-PythonExecutable {
|
||||||
|
param([string]$Preferred = "python")
|
||||||
|
|
||||||
|
if ($Preferred -ne "python") {
|
||||||
|
if ((Get-Command $Preferred -ErrorAction SilentlyContinue) -and
|
||||||
|
-not ((Get-Command $Preferred).Source -like "*WindowsApps*")) {
|
||||||
|
return $Preferred
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$candidates = @(
|
||||||
|
(Join-Path $env:LOCALAPPDATA "Programs\Python\Python313\python.exe"),
|
||||||
|
(Join-Path $env:LOCALAPPDATA "Programs\Python\Python312\python.exe"),
|
||||||
|
(Join-Path $env:LOCALAPPDATA "Programs\Python\Python311\python.exe"),
|
||||||
|
"C:\Program Files\Python312\python.exe",
|
||||||
|
"C:\Program Files\Python313\python.exe"
|
||||||
|
)
|
||||||
|
|
||||||
|
foreach ($candidate in $candidates) {
|
||||||
|
if (Test-Path $candidate) {
|
||||||
|
return $candidate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$pythonCommand = Get-Command python -ErrorAction SilentlyContinue
|
||||||
|
if ($pythonCommand -and $pythonCommand.Source -notlike "*WindowsApps*") {
|
||||||
|
return $pythonCommand.Source
|
||||||
|
}
|
||||||
|
|
||||||
|
return $null
|
||||||
|
}
|
||||||
|
|
||||||
|
function Export-VocabFromTokenizerJson {
|
||||||
|
param([string]$TokenizerJsonPath, [string]$VocabPath)
|
||||||
|
|
||||||
|
$tokenizer = Get-Content $TokenizerJsonPath -Raw | ConvertFrom-Json
|
||||||
|
$vocab = $tokenizer.model.vocab
|
||||||
|
if (-not $vocab) {
|
||||||
|
return $false
|
||||||
|
}
|
||||||
|
|
||||||
|
$orderedTokens = $vocab.PSObject.Properties |
|
||||||
|
Sort-Object { [int]$_.Value } |
|
||||||
|
ForEach-Object { $_.Name }
|
||||||
|
|
||||||
|
$orderedTokens | Set-Content -Path $VocabPath -Encoding utf8
|
||||||
|
return $true
|
||||||
|
}
|
||||||
|
|
||||||
|
function Copy-TokenizerAssets {
|
||||||
|
param([string]$SourceDir)
|
||||||
|
|
||||||
|
foreach ($name in @("sentencepiece.bpe.model", "spiece.model", "tokenizer.model")) {
|
||||||
|
$source = Join-Path $SourceDir $name
|
||||||
|
if (Test-Path $source) {
|
||||||
|
$destination = Join-Path $modelsDir $name
|
||||||
|
Copy-Item $source $destination -Force
|
||||||
|
return @($destination)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$tokenizerJsonSource = Join-Path $SourceDir "tokenizer.json"
|
||||||
|
if (Test-Path $tokenizerJsonSource) {
|
||||||
|
$tokenizerJsonDestination = Join-Path $modelsDir "tokenizer.json"
|
||||||
|
Copy-Item $tokenizerJsonSource $tokenizerJsonDestination -Force
|
||||||
|
$saved = @($tokenizerJsonDestination)
|
||||||
|
|
||||||
|
$vocabPath = Join-Path $modelsDir "vocab.txt"
|
||||||
|
if (Export-VocabFromTokenizerJson -TokenizerJsonPath $tokenizerJsonDestination -VocabPath $vocabPath) {
|
||||||
|
$saved += $vocabPath
|
||||||
|
}
|
||||||
|
|
||||||
|
Write-Warning (
|
||||||
|
"No SentencePiece model on Hugging Face; saved WordPiece assets ($(
|
||||||
|
($saved | ForEach-Object { Split-Path $_ -Leaf }) -join ', '
|
||||||
|
)). TamilOnnxNerRunner uses vocab.txt (WordPiece) when present, otherwise SentencePiece model files."
|
||||||
|
)
|
||||||
|
return $saved
|
||||||
|
}
|
||||||
|
|
||||||
|
return @()
|
||||||
|
}
|
||||||
|
|
||||||
|
function Download-WithPowerShell {
|
||||||
|
Ensure-ModelsDirectory
|
||||||
|
|
||||||
|
$modelPath = Join-Path $modelsDir "model.onnx"
|
||||||
|
$configPath = Join-Path $modelsDir "config.json"
|
||||||
|
$labelsPath = Join-Path $modelsDir "ner-labels.txt"
|
||||||
|
$tempDir = Join-Path $modelsDir "_download_temp"
|
||||||
|
New-Item -ItemType Directory -Force -Path $tempDir | Out-Null
|
||||||
|
|
||||||
|
try {
|
||||||
|
Download-HuggingFaceAsset -RelativePath "onnx/model.onnx" -Destination $modelPath
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
Write-Host "Pre-exported ONNX not found; downloading config and tokenizer for manual export..."
|
||||||
|
Download-HuggingFaceAsset -RelativePath "config.json" -Destination $configPath
|
||||||
|
Export-LabelsFromConfig -ConfigPath $configPath -LabelsPath $labelsPath
|
||||||
|
|
||||||
|
$tokenizerJsonPath = Join-Path $modelsDir "tokenizer.json"
|
||||||
|
try {
|
||||||
|
Download-HuggingFaceAsset -RelativePath "tokenizer.json" -Destination $tokenizerJsonPath
|
||||||
|
$vocabPath = Join-Path $modelsDir "vocab.txt"
|
||||||
|
Export-VocabFromTokenizerJson -TokenizerJsonPath $tokenizerJsonPath -VocabPath $vocabPath | Out-Null
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
Write-Host "tokenizer.json not available from Hugging Face."
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach ($name in @("sentencepiece.bpe.model", "spiece.model", "tokenizer.model")) {
|
||||||
|
try {
|
||||||
|
Download-HuggingFaceAsset -RelativePath $name -Destination (Join-Path $modelsDir $name)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$pythonExe = Resolve-PythonExecutable -Preferred $Python
|
||||||
|
if ($pythonExe) {
|
||||||
|
throw (
|
||||||
|
"Tamil ONNX model is not published on Hugging Face. Re-run with Python export:`n" +
|
||||||
|
" .\scripts\download-tamil-ner-model.ps1 -Python `"$pythonExe`""
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
throw @"
|
||||||
|
Tamil ONNX model is not published on Hugging Face (onnx/model.onnx returns 404).
|
||||||
|
Install Python 3.12+ and re-run this script:
|
||||||
|
winget install Python.Python.3.12 --accept-package-agreements --accept-source-agreements
|
||||||
|
.\scripts\download-tamil-ner-model.ps1 -Python `"`$env:LOCALAPPDATA\Programs\Python\Python312\python.exe`"
|
||||||
|
Only config.json, ner-labels.txt, and tokenizer.json were saved under models/ta/.
|
||||||
|
"@
|
||||||
|
}
|
||||||
|
|
||||||
|
Download-HuggingFaceAsset -RelativePath "config.json" -Destination $configPath
|
||||||
|
Export-LabelsFromConfig -ConfigPath $configPath -LabelsPath $labelsPath
|
||||||
|
Remove-Item $configPath -Force
|
||||||
|
|
||||||
|
foreach ($name in @("sentencepiece.bpe.model", "spiece.model")) {
|
||||||
|
try {
|
||||||
|
Download-HuggingFaceAsset -RelativePath $name -Destination (Join-Path $modelsDir $name)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host "Tamil NER model assets saved to $modelsDir"
|
||||||
|
}
|
||||||
|
|
||||||
|
function Download-WithPython {
|
||||||
|
$pythonExe = Resolve-PythonExecutable -Preferred $Python
|
||||||
|
if (-not $pythonExe) {
|
||||||
|
return $false
|
||||||
|
}
|
||||||
|
|
||||||
|
Write-Host "Using Python: $pythonExe"
|
||||||
|
Write-Host "Repository root: $repoRoot"
|
||||||
|
Write-Host ""
|
||||||
|
|
||||||
|
Push-Location $repoRoot
|
||||||
|
try {
|
||||||
|
& $pythonExe $scriptPath
|
||||||
|
if ($LASTEXITCODE -ne 0) {
|
||||||
|
throw "Tamil model download script failed with exit code $LASTEXITCODE."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
Pop-Location
|
||||||
|
}
|
||||||
|
|
||||||
|
return $true
|
||||||
|
}
|
||||||
|
|
||||||
|
Write-Host "Repository root: $repoRoot"
|
||||||
|
|
||||||
|
if (-not (Download-WithPython)) {
|
||||||
|
Write-Host "Python export unavailable; downloading pre-exported ONNX assets from Hugging Face..."
|
||||||
|
Write-Host ""
|
||||||
|
Download-WithPowerShell
|
||||||
|
}
|
||||||
148
scripts/download-tamil-ner-model.py
Normal file
148
scripts/download-tamil-ner-model.py
Normal file
@@ -0,0 +1,148 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Download and export prachuryyaIITG/SampurNER_Tamil_IndicBERTv2 to ONNX for the PII Redaction POC."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
MODELS_DIR = REPO_ROOT / "models" / "ta"
|
||||||
|
MODEL_ID = "prachuryyaIITG/SampurNER_Tamil_IndicBERTv2"
|
||||||
|
REQUIRED_PACKAGES = ("transformers", "optimum[onnxruntime]", "onnx", "torch")
|
||||||
|
SENTENCEPIECE_CANDIDATES = (
|
||||||
|
"sentencepiece.bpe.model",
|
||||||
|
"spiece.model",
|
||||||
|
"tokenizer.model",
|
||||||
|
)
|
||||||
|
TOKENIZER_JSON = "tokenizer.json"
|
||||||
|
VOCAB_TXT = "vocab.txt"
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_dependencies() -> None:
|
||||||
|
try:
|
||||||
|
import optimum.onnxruntime # noqa: F401
|
||||||
|
import transformers # noqa: F401
|
||||||
|
except ImportError:
|
||||||
|
print("Installing Python dependencies (this may take a few minutes)...")
|
||||||
|
subprocess.check_call(
|
||||||
|
[sys.executable, "-m", "pip", "install", *REQUIRED_PACKAGES],
|
||||||
|
stdout=sys.stdout,
|
||||||
|
stderr=sys.stderr,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def copy_sentencepiece_model(source_dir: Path, target_dir: Path) -> Path | None:
|
||||||
|
for name in SENTENCEPIECE_CANDIDATES:
|
||||||
|
candidate = source_dir / name
|
||||||
|
if candidate.exists():
|
||||||
|
destination = target_dir / name
|
||||||
|
shutil.copy(candidate, destination)
|
||||||
|
return destination
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def export_wordpiece_assets(source_dir: Path, target_dir: Path) -> list[Path]:
|
||||||
|
saved: list[Path] = []
|
||||||
|
tokenizer_json = source_dir / TOKENIZER_JSON
|
||||||
|
if not tokenizer_json.exists():
|
||||||
|
return saved
|
||||||
|
|
||||||
|
destination = target_dir / TOKENIZER_JSON
|
||||||
|
shutil.copy(tokenizer_json, destination)
|
||||||
|
saved.append(destination)
|
||||||
|
|
||||||
|
with tokenizer_json.open(encoding="utf-8") as tokenizer_file:
|
||||||
|
tokenizer_data = json.load(tokenizer_file)
|
||||||
|
|
||||||
|
vocab = tokenizer_data.get("model", {}).get("vocab")
|
||||||
|
if not isinstance(vocab, dict):
|
||||||
|
return saved
|
||||||
|
|
||||||
|
vocab_path = target_dir / VOCAB_TXT
|
||||||
|
ordered_tokens = [token for token, _ in sorted(vocab.items(), key=lambda item: item[1])]
|
||||||
|
vocab_path.write_text("\n".join(ordered_tokens), encoding="utf-8")
|
||||||
|
saved.append(vocab_path)
|
||||||
|
return saved
|
||||||
|
|
||||||
|
|
||||||
|
def copy_tokenizer_assets(source_dir: Path, target_dir: Path) -> list[Path]:
|
||||||
|
sentencepiece_path = copy_sentencepiece_model(source_dir, target_dir)
|
||||||
|
if sentencepiece_path is not None:
|
||||||
|
return [sentencepiece_path]
|
||||||
|
|
||||||
|
wordpiece_assets = export_wordpiece_assets(source_dir, target_dir)
|
||||||
|
if wordpiece_assets:
|
||||||
|
print(
|
||||||
|
"WARNING: Hugging Face repo has no SentencePiece model; saved WordPiece "
|
||||||
|
f"assets ({', '.join(path.name for path in wordpiece_assets)}). "
|
||||||
|
"TamilOnnxNerRunner uses vocab.txt (WordPiece) when present, "
|
||||||
|
"otherwise SentencePiece model files."
|
||||||
|
)
|
||||||
|
return wordpiece_assets
|
||||||
|
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"No tokenizer assets found in {source_dir}. "
|
||||||
|
f"Expected one of {SENTENCEPIECE_CANDIDATES} or {TOKENIZER_JSON}."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def export_model() -> None:
|
||||||
|
from optimum.onnxruntime import ORTModelForTokenClassification
|
||||||
|
from transformers import AutoTokenizer
|
||||||
|
|
||||||
|
MODELS_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
temp_dir = MODELS_DIR / "_export_temp"
|
||||||
|
if temp_dir.exists():
|
||||||
|
shutil.rmtree(temp_dir)
|
||||||
|
temp_dir.mkdir()
|
||||||
|
|
||||||
|
print(f"Exporting {MODEL_ID} to ONNX...")
|
||||||
|
model = ORTModelForTokenClassification.from_pretrained(MODEL_ID, export=True)
|
||||||
|
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
|
||||||
|
|
||||||
|
model.save_pretrained(temp_dir)
|
||||||
|
tokenizer.save_pretrained(temp_dir)
|
||||||
|
|
||||||
|
onnx_files = sorted(temp_dir.glob("*.onnx"))
|
||||||
|
if not onnx_files:
|
||||||
|
raise FileNotFoundError("Export completed but no .onnx file was produced.")
|
||||||
|
|
||||||
|
target_onnx = MODELS_DIR / "model.onnx"
|
||||||
|
shutil.copy(onnx_files[0], target_onnx)
|
||||||
|
|
||||||
|
config_path = temp_dir / "config.json"
|
||||||
|
with config_path.open(encoding="utf-8") as config_file:
|
||||||
|
config = json.load(config_file)
|
||||||
|
|
||||||
|
id2label = config.get("id2label", {})
|
||||||
|
labels = [id2label[str(index)] for index in range(len(id2label))]
|
||||||
|
labels_path = MODELS_DIR / "ner-labels.txt"
|
||||||
|
labels_path.write_text("\n".join(labels), encoding="utf-8")
|
||||||
|
|
||||||
|
tokenizer_assets = copy_tokenizer_assets(temp_dir, MODELS_DIR)
|
||||||
|
shutil.rmtree(temp_dir)
|
||||||
|
|
||||||
|
print()
|
||||||
|
print("Tamil NER model assets saved:")
|
||||||
|
print(f" {target_onnx}")
|
||||||
|
for asset in tokenizer_assets:
|
||||||
|
print(f" {asset}")
|
||||||
|
print(f" {labels_path}")
|
||||||
|
print()
|
||||||
|
print("Run from repository root:")
|
||||||
|
print(" dotnet test --filter Category=TamilNer")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
ensure_dependencies()
|
||||||
|
export_model()
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
27
scripts/tamil-ner-diagnostic/Program.cs
Normal file
27
scripts/tamil-ner-diagnostic/Program.cs
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
using Microsoft.ML.Tokenizers;
|
||||||
|
using PiiRedaction.Core.Configuration;
|
||||||
|
using PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
var modelDir = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "models", "ta"));
|
||||||
|
if (!Directory.Exists(modelDir))
|
||||||
|
{
|
||||||
|
modelDir = Path.GetFullPath("models/ta");
|
||||||
|
}
|
||||||
|
|
||||||
|
var vocabPath = Path.Combine(modelDir, "vocab.txt");
|
||||||
|
var bertOptions = new BertOptions { LowerCaseBeforeTokenization = false, ApplyBasicTokenization = false };
|
||||||
|
var tokenizer = BertTokenizer.Create(vocabPath, bertOptions);
|
||||||
|
var text = "வாடிக்கையாளர் ராஜேஷ் குமார் அழைத்தார்.";
|
||||||
|
var tokens = tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true);
|
||||||
|
Console.WriteLine($"count={tokens.Count}");
|
||||||
|
foreach (var t in tokens) Console.WriteLine($"{t.Id}\t{t.Value}");
|
||||||
|
|
||||||
|
var runnerOptions = Options.Create(new PiiRedactionOptions { TamilOnnxModelPath = Path.Combine(modelDir, "model.onnx") });
|
||||||
|
using var runner = new TamilOnnxNerRunner(runnerOptions, NullLogger<TamilOnnxNerRunner>.Instance);
|
||||||
|
Console.WriteLine($"Available: {runner.IsModelAvailable}");
|
||||||
|
foreach (var e in runner.PredictEntities(text).Entities)
|
||||||
|
{
|
||||||
|
Console.WriteLine($"Entity: '{e.Value}' [{e.StartIndex},{e.Length}]");
|
||||||
|
}
|
||||||
14
scripts/tamil-ner-diagnostic/tamil-ner-diagnostic.csproj
Normal file
14
scripts/tamil-ner-diagnostic/tamil-ner-diagnostic.csproj
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
</PropertyGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<ProjectReference Include="..\..\src\PiiRedaction.Infrastructure\PiiRedaction.Infrastructure.csproj" />
|
||||||
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="Microsoft.ML.Tokenizers" Version="2.0.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
</Project>
|
||||||
@@ -31,8 +31,9 @@ public static class ServiceCollectionExtensions
|
|||||||
services.AddSingleton<IPiiRedactor, PlaceholderPiiRedactor>();
|
services.AddSingleton<IPiiRedactor, PlaceholderPiiRedactor>();
|
||||||
services.AddSingleton<IPromptSanitizer, PromptSanitizer>();
|
services.AddSingleton<IPromptSanitizer, PromptSanitizer>();
|
||||||
|
|
||||||
services.AddSingleton<OnnxNerModelRunner>();
|
services.AddSingleton<EnglishOnnxNerRunner>();
|
||||||
services.AddSingleton<IOnnxNerModelRunner>(provider => provider.GetRequiredService<OnnxNerModelRunner>());
|
services.AddSingleton<TamilOnnxNerRunner>();
|
||||||
|
services.AddSingleton<IOnnxNerModelRunner, RoutingOnnxNerModelRunner>();
|
||||||
services.AddSingleton<IChatClient, MockChatClient>();
|
services.AddSingleton<IChatClient, MockChatClient>();
|
||||||
services.AddSingleton<ILlmPromptService, MockLlmPromptService>();
|
services.AddSingleton<ILlmPromptService, MockLlmPromptService>();
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,10 @@ using PiiRedaction.ConsoleApp.Samples;
|
|||||||
using PiiRedaction.Core.Abstractions;
|
using PiiRedaction.Core.Abstractions;
|
||||||
using PiiRedaction.Core.Models;
|
using PiiRedaction.Core.Models;
|
||||||
|
|
||||||
|
System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
|
||||||
|
Console.InputEncoding = System.Text.Encoding.UTF8;
|
||||||
|
Console.OutputEncoding = System.Text.Encoding.UTF8;
|
||||||
|
|
||||||
var interactive = args.Contains("--interactive", StringComparer.OrdinalIgnoreCase);
|
var interactive = args.Contains("--interactive", StringComparer.OrdinalIgnoreCase);
|
||||||
var listSamples = args.Contains("--list", StringComparer.OrdinalIgnoreCase);
|
var listSamples = args.Contains("--list", StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
|||||||
@@ -59,7 +59,7 @@ public sealed class PromptDemoRunner
|
|||||||
Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --sample 2");
|
Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --sample 2");
|
||||||
Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --name MrTitlePerson");
|
Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --name MrTitlePerson");
|
||||||
Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --list");
|
Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --list");
|
||||||
Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --interactive");
|
Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --interactive # UTF-8 input recommended for non-ASCII text");
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void DisplayDetectedEntities(IReadOnlyList<PiiEntity> entities)
|
private static void DisplayDetectedEntities(IReadOnlyList<PiiEntity> entities)
|
||||||
|
|||||||
@@ -68,6 +68,36 @@ public static class SamplePromptCatalog
|
|||||||
"Loan number, customer ID, and account number together.",
|
"Loan number, customer ID, and account number together.",
|
||||||
"Please verify LN-100200 for CustomerId CID-3000 on AccountNumber ACC-400500."),
|
"Please verify LN-100200 for CustomerId CID-3000 on AccountNumber ACC-400500."),
|
||||||
|
|
||||||
|
new(
|
||||||
|
"TamilCustomerNameOnly",
|
||||||
|
"NER (Tamil)",
|
||||||
|
"Tamil script person name detected via Tamil ONNX NER.",
|
||||||
|
"வாடிக்கையாளர் ராஜேஷ் குமார் சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்."),
|
||||||
|
|
||||||
|
new(
|
||||||
|
"TamilWithPhonePan",
|
||||||
|
"NER (Tamil) + Regex",
|
||||||
|
"Tamil script person plus phone and PAN (regex).",
|
||||||
|
"வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210 PAN ABCDE1234F."),
|
||||||
|
|
||||||
|
new(
|
||||||
|
"TanglishCustomer",
|
||||||
|
"NER (English/Tanglish)",
|
||||||
|
"Latin-script Tanglish person name via English ONNX NER.",
|
||||||
|
"Customer Senthil phone 9876543210 reported a failed UPI transfer."),
|
||||||
|
|
||||||
|
new(
|
||||||
|
"MixedTamilEnglish",
|
||||||
|
"NER (Mixed)",
|
||||||
|
"Code-mixed Tamil and English — both script routers may contribute person spans.",
|
||||||
|
"வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge."),
|
||||||
|
|
||||||
|
new(
|
||||||
|
"TamilFullFinancial",
|
||||||
|
"NER (Tamil) + Regex + Domain",
|
||||||
|
"Tamil person with email, phone, loan number, and PAN (canonical demo in Tamil).",
|
||||||
|
"வாடிக்கையாளர் ராஜேஷ் குமார் மின்னஞ்சல் ravi.kumar@gmail.com தொலைபேசி 9876543210 LoanNumber LN-456789 PAN ABCDE1234F. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்."),
|
||||||
|
|
||||||
new(
|
new(
|
||||||
"NoPiiCleanTicket",
|
"NoPiiCleanTicket",
|
||||||
"Negative",
|
"Negative",
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
{
|
{
|
||||||
"PiiRedaction": {
|
"PiiRedaction": {
|
||||||
"OnnxModelPath": "models/ner-model.onnx"
|
"OnnxModelPath": "models/ner-model.onnx",
|
||||||
|
"EnglishOnnxModelPath": "models/en/ner-model.onnx",
|
||||||
|
"TamilOnnxModelPath": "models/ta/model.onnx",
|
||||||
|
"EnableTamilNer": true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,4 +5,10 @@ public sealed class PiiRedactionOptions
|
|||||||
public const string SectionName = "PiiRedaction";
|
public const string SectionName = "PiiRedaction";
|
||||||
|
|
||||||
public string OnnxModelPath { get; set; } = "models/ner-model.onnx";
|
public string OnnxModelPath { get; set; } = "models/ner-model.onnx";
|
||||||
|
|
||||||
|
public string EnglishOnnxModelPath { get; set; } = "models/en/ner-model.onnx";
|
||||||
|
|
||||||
|
public string TamilOnnxModelPath { get; set; } = "models/ta/model.onnx";
|
||||||
|
|
||||||
|
public bool EnableTamilNer { get; set; } = true;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,15 +7,20 @@ namespace PiiRedaction.Core.Detection;
|
|||||||
/// Aggregates multiple PII detectors and merges overlapping spans.
|
/// Aggregates multiple PII detectors and merges overlapping spans.
|
||||||
/// Detectors are applied in registration order; earlier detectors win on overlap.
|
/// Detectors are applied in registration order; earlier detectors win on overlap.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class CompositePiiDetector : IPiiDetector
|
public sealed class CompositePiiDetector : IPiiDetector, INerRoutingSource
|
||||||
{
|
{
|
||||||
private readonly IReadOnlyList<IPiiDetector> _detectors;
|
private readonly IReadOnlyList<IPiiDetector> _detectors;
|
||||||
|
private readonly INerRoutingSource? _nerRoutingSource;
|
||||||
|
|
||||||
public CompositePiiDetector(IEnumerable<IPiiDetector> detectors)
|
public CompositePiiDetector(IEnumerable<IPiiDetector> detectors)
|
||||||
{
|
{
|
||||||
_detectors = detectors.ToList();
|
_detectors = detectors.ToList();
|
||||||
|
_nerRoutingSource = _detectors.OfType<INerRoutingSource>().FirstOrDefault();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public IReadOnlyList<NerModelOrigin> LastInvokedModels =>
|
||||||
|
_nerRoutingSource?.LastInvokedModels ?? [];
|
||||||
|
|
||||||
public IReadOnlyList<PiiEntity> Detect(string text)
|
public IReadOnlyList<PiiEntity> Detect(string text)
|
||||||
{
|
{
|
||||||
ArgumentException.ThrowIfNullOrWhiteSpace(text);
|
ArgumentException.ThrowIfNullOrWhiteSpace(text);
|
||||||
|
|||||||
8
src/PiiRedaction.Core/Detection/INerRoutingSource.cs
Normal file
8
src/PiiRedaction.Core/Detection/INerRoutingSource.cs
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
using PiiRedaction.Core.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Core.Detection;
|
||||||
|
|
||||||
|
public interface INerRoutingSource
|
||||||
|
{
|
||||||
|
IReadOnlyList<NerModelOrigin> LastInvokedModels { get; }
|
||||||
|
}
|
||||||
@@ -7,5 +7,5 @@ public interface IOnnxNerModelRunner
|
|||||||
{
|
{
|
||||||
bool IsModelAvailable { get; }
|
bool IsModelAvailable { get; }
|
||||||
|
|
||||||
IReadOnlyList<PiiEntity> PredictEntities(string text);
|
NerPredictionResult PredictEntities(string text);
|
||||||
}
|
}
|
||||||
|
|||||||
11
src/PiiRedaction.Core/Detection/NerEntityTagging.cs
Normal file
11
src/PiiRedaction.Core/Detection/NerEntityTagging.cs
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
using PiiRedaction.Core.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Core.Detection;
|
||||||
|
|
||||||
|
public static class NerEntityTagging
|
||||||
|
{
|
||||||
|
public static IReadOnlyList<PiiEntity> WithOrigin(
|
||||||
|
IReadOnlyList<PiiEntity> entities,
|
||||||
|
NerModelOrigin origin) =>
|
||||||
|
entities.Select(entity => entity with { ModelOrigin = origin }).ToList();
|
||||||
|
}
|
||||||
10
src/PiiRedaction.Core/Detection/NerPredictionResult.cs
Normal file
10
src/PiiRedaction.Core/Detection/NerPredictionResult.cs
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
using PiiRedaction.Core.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Core.Detection;
|
||||||
|
|
||||||
|
public sealed record NerPredictionResult(
|
||||||
|
IReadOnlyList<PiiEntity> Entities,
|
||||||
|
IReadOnlyList<NerModelOrigin> InvokedModels)
|
||||||
|
{
|
||||||
|
public static NerPredictionResult Empty { get; } = new([], []);
|
||||||
|
}
|
||||||
@@ -8,7 +8,7 @@ namespace PiiRedaction.Core.Detection;
|
|||||||
/// NER is used for entities that lack rigid formats and vary in surface form across prompts.
|
/// NER is used for entities that lack rigid formats and vary in surface form across prompts.
|
||||||
/// Requires a loaded ONNX model; returns no person entities when the model is unavailable.
|
/// Requires a loaded ONNX model; returns no person entities when the model is unavailable.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class OnnxNerPiiDetector : IPiiDetector
|
public sealed class OnnxNerPiiDetector : IPiiDetector, INerRoutingSource
|
||||||
{
|
{
|
||||||
private readonly IOnnxNerModelRunner _modelRunner;
|
private readonly IOnnxNerModelRunner _modelRunner;
|
||||||
|
|
||||||
@@ -17,15 +17,20 @@ public sealed class OnnxNerPiiDetector : IPiiDetector
|
|||||||
_modelRunner = modelRunner;
|
_modelRunner = modelRunner;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public IReadOnlyList<NerModelOrigin> LastInvokedModels { get; private set; } = [];
|
||||||
|
|
||||||
public IReadOnlyList<PiiEntity> Detect(string text)
|
public IReadOnlyList<PiiEntity> Detect(string text)
|
||||||
{
|
{
|
||||||
ArgumentException.ThrowIfNullOrWhiteSpace(text);
|
ArgumentException.ThrowIfNullOrWhiteSpace(text);
|
||||||
|
|
||||||
if (!_modelRunner.IsModelAvailable)
|
if (!_modelRunner.IsModelAvailable)
|
||||||
{
|
{
|
||||||
|
LastInvokedModels = [];
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
return _modelRunner.PredictEntities(text);
|
var result = _modelRunner.PredictEntities(text);
|
||||||
|
LastInvokedModels = result.InvokedModels;
|
||||||
|
return result.Entities;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
9
src/PiiRedaction.Core/Detection/ScriptComposition.cs
Normal file
9
src/PiiRedaction.Core/Detection/ScriptComposition.cs
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
namespace PiiRedaction.Core.Detection;
|
||||||
|
|
||||||
|
public enum ScriptComposition
|
||||||
|
{
|
||||||
|
LatinOnly,
|
||||||
|
TamilOnly,
|
||||||
|
Mixed,
|
||||||
|
NoLetters
|
||||||
|
}
|
||||||
45
src/PiiRedaction.Core/Detection/ScriptRouter.cs
Normal file
45
src/PiiRedaction.Core/Detection/ScriptRouter.cs
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
namespace PiiRedaction.Core.Detection;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Classifies prompt text by script composition to route NER inference.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class ScriptRouter
|
||||||
|
{
|
||||||
|
private const char TamilRangeStart = '\u0B80';
|
||||||
|
private const char TamilRangeEnd = '\u0BFF';
|
||||||
|
|
||||||
|
public ScriptComposition GetComposition(string text)
|
||||||
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(text);
|
||||||
|
|
||||||
|
var hasLatin = false;
|
||||||
|
var hasTamil = false;
|
||||||
|
|
||||||
|
foreach (var character in text)
|
||||||
|
{
|
||||||
|
if (IsTamilLetter(character))
|
||||||
|
{
|
||||||
|
hasTamil = true;
|
||||||
|
}
|
||||||
|
else if (char.IsAsciiLetter(character))
|
||||||
|
{
|
||||||
|
hasLatin = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (hasLatin && hasTamil)
|
||||||
|
{
|
||||||
|
return ScriptComposition.Mixed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!hasLatin && !hasTamil)
|
||||||
|
{
|
||||||
|
return ScriptComposition.NoLetters;
|
||||||
|
}
|
||||||
|
|
||||||
|
return hasTamil ? ScriptComposition.TamilOnly : ScriptComposition.LatinOnly;
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static bool IsTamilLetter(char character) =>
|
||||||
|
character is >= TamilRangeStart and <= TamilRangeEnd;
|
||||||
|
}
|
||||||
7
src/PiiRedaction.Core/Models/NerModelOrigin.cs
Normal file
7
src/PiiRedaction.Core/Models/NerModelOrigin.cs
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
namespace PiiRedaction.Core.Models;
|
||||||
|
|
||||||
|
public enum NerModelOrigin
|
||||||
|
{
|
||||||
|
English,
|
||||||
|
Tamil
|
||||||
|
}
|
||||||
@@ -6,7 +6,8 @@ public sealed record PiiEntity(
|
|||||||
int StartIndex,
|
int StartIndex,
|
||||||
int Length,
|
int Length,
|
||||||
PiiDetectionSource Source,
|
PiiDetectionSource Source,
|
||||||
double? Confidence = null)
|
double? Confidence = null,
|
||||||
|
NerModelOrigin? ModelOrigin = null)
|
||||||
{
|
{
|
||||||
public int EndIndex => StartIndex + Length;
|
public int EndIndex => StartIndex + Length;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,4 +4,5 @@ public sealed record SanitizationResult(
|
|||||||
string OriginalPrompt,
|
string OriginalPrompt,
|
||||||
string SanitizedPrompt,
|
string SanitizedPrompt,
|
||||||
IReadOnlyList<PiiEntity> DetectedEntities,
|
IReadOnlyList<PiiEntity> DetectedEntities,
|
||||||
RedactionResult Redaction);
|
RedactionResult Redaction,
|
||||||
|
IReadOnlyList<NerModelOrigin> NerModelsInvoked);
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
using PiiRedaction.Core.Abstractions;
|
using PiiRedaction.Core.Abstractions;
|
||||||
|
using PiiRedaction.Core.Detection;
|
||||||
using PiiRedaction.Core.Models;
|
using PiiRedaction.Core.Models;
|
||||||
|
|
||||||
namespace PiiRedaction.Core.Sanitization;
|
namespace PiiRedaction.Core.Sanitization;
|
||||||
@@ -21,11 +22,15 @@ public sealed class PromptSanitizer : IPromptSanitizer
|
|||||||
|
|
||||||
var entities = _detector.Detect(request.OriginalPrompt);
|
var entities = _detector.Detect(request.OriginalPrompt);
|
||||||
var redaction = _redactor.Redact(request.OriginalPrompt, entities);
|
var redaction = _redactor.Redact(request.OriginalPrompt, entities);
|
||||||
|
var nerModelsInvoked = _detector is INerRoutingSource routingSource
|
||||||
|
? routingSource.LastInvokedModels
|
||||||
|
: [];
|
||||||
|
|
||||||
return new SanitizationResult(
|
return new SanitizationResult(
|
||||||
request.OriginalPrompt,
|
request.OriginalPrompt,
|
||||||
redaction.SanitizedText,
|
redaction.SanitizedText,
|
||||||
entities,
|
entities,
|
||||||
redaction);
|
redaction,
|
||||||
|
nerModelsInvoked);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
107
src/PiiRedaction.Infrastructure/Onnx/BertWordPieceEncoder.cs
Normal file
107
src/PiiRedaction.Infrastructure/Onnx/BertWordPieceEncoder.cs
Normal file
@@ -0,0 +1,107 @@
|
|||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.ML.Tokenizers;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
public sealed class BertWordPieceEncoder : ITokenClassifierEncoder
|
||||||
|
{
|
||||||
|
private readonly BertTokenizer? _tokenizer;
|
||||||
|
private readonly ILogger _logger;
|
||||||
|
|
||||||
|
public BertWordPieceEncoder(string modelDirectory, ILogger logger)
|
||||||
|
{
|
||||||
|
_logger = logger;
|
||||||
|
_tokenizer = TryLoadTokenizer(modelDirectory);
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool IsAvailable => _tokenizer is not null;
|
||||||
|
|
||||||
|
public EncodedSequence? Encode(string text, int maxSequenceLength)
|
||||||
|
{
|
||||||
|
if (_tokenizer is null)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var encodedTokens = _tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true);
|
||||||
|
var wordTokens = encodedTokens.Take(Math.Max(0, maxSequenceLength - 2)).ToList();
|
||||||
|
if (wordTokens.Count == 0)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var sequenceLength = wordTokens.Count + 2;
|
||||||
|
var inputIds = new long[sequenceLength];
|
||||||
|
var attentionMask = new long[sequenceLength];
|
||||||
|
var tokenTypeIds = new long[sequenceLength];
|
||||||
|
var offsets = new (int Start, int End)[sequenceLength];
|
||||||
|
var tokenIds = new int[sequenceLength];
|
||||||
|
|
||||||
|
inputIds[0] = _tokenizer.ClassificationTokenId;
|
||||||
|
attentionMask[0] = 1;
|
||||||
|
tokenIds[0] = _tokenizer.ClassificationTokenId;
|
||||||
|
offsets[0] = (0, 0);
|
||||||
|
|
||||||
|
for (var i = 0; i < wordTokens.Count; i++)
|
||||||
|
{
|
||||||
|
var token = wordTokens[i];
|
||||||
|
var index = i + 1;
|
||||||
|
inputIds[index] = token.Id;
|
||||||
|
attentionMask[index] = 1;
|
||||||
|
tokenIds[index] = token.Id;
|
||||||
|
offsets[index] = ToCharOffsets(token.Offset, text.Length);
|
||||||
|
}
|
||||||
|
|
||||||
|
inputIds[sequenceLength - 1] = _tokenizer.SeparatorTokenId;
|
||||||
|
attentionMask[sequenceLength - 1] = 1;
|
||||||
|
tokenIds[sequenceLength - 1] = _tokenizer.SeparatorTokenId;
|
||||||
|
offsets[sequenceLength - 1] = (0, 0);
|
||||||
|
|
||||||
|
return new EncodedSequence(inputIds, attentionMask, tokenTypeIds, offsets, tokenIds, sequenceLength);
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool IsSpecialToken(int tokenId) =>
|
||||||
|
_tokenizer is not null &&
|
||||||
|
(tokenId == _tokenizer.ClassificationTokenId ||
|
||||||
|
tokenId == _tokenizer.SeparatorTokenId ||
|
||||||
|
tokenId == _tokenizer.PaddingTokenId);
|
||||||
|
|
||||||
|
private BertTokenizer? TryLoadTokenizer(string modelDirectory)
|
||||||
|
{
|
||||||
|
var vocabPath = OnnxAssetPathResolver.ResolveAssetPath(Path.Combine(modelDirectory, "vocab.txt"));
|
||||||
|
if (!File.Exists(vocabPath))
|
||||||
|
{
|
||||||
|
_logger.LogWarning("Tokenizer vocabulary not found at {VocabPath}.", vocabPath);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
return BertTokenizer.Create(vocabPath, CreateBertOptions(modelDirectory));
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Failed to load BERT tokenizer from {VocabPath}.", vocabPath);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static BertOptions CreateBertOptions(string modelDirectory)
|
||||||
|
{
|
||||||
|
var tokenizerJsonPath = OnnxAssetPathResolver.ResolveAssetPath(
|
||||||
|
Path.Combine(modelDirectory, "tokenizer.json"));
|
||||||
|
var whitespaceOnlyPretokenization = File.Exists(tokenizerJsonPath);
|
||||||
|
|
||||||
|
return new BertOptions
|
||||||
|
{
|
||||||
|
LowerCaseBeforeTokenization = false,
|
||||||
|
ApplyBasicTokenization = !whitespaceOnlyPretokenization
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static (int Start, int End) ToCharOffsets(Range offset, int textLength)
|
||||||
|
{
|
||||||
|
var (start, length) = offset.GetOffsetAndLength(textLength);
|
||||||
|
return (start, start + length);
|
||||||
|
}
|
||||||
|
}
|
||||||
46
src/PiiRedaction.Infrastructure/Onnx/EnglishOnnxNerRunner.cs
Normal file
46
src/PiiRedaction.Infrastructure/Onnx/EnglishOnnxNerRunner.cs
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
using PiiRedaction.Core.Configuration;
|
||||||
|
using PiiRedaction.Core.Detection;
|
||||||
|
using PiiRedaction.Core.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
public class EnglishOnnxNerRunner : IOnnxNerModelRunner, IDisposable
|
||||||
|
{
|
||||||
|
private readonly OnnxTokenClassifierRunner _runner;
|
||||||
|
|
||||||
|
protected EnglishOnnxNerRunner(IOptions<PiiRedactionOptions> options, ILogger logger)
|
||||||
|
{
|
||||||
|
var modelPath = OnnxAssetPathResolver.ResolveModelPath(
|
||||||
|
options.Value.EnglishOnnxModelPath,
|
||||||
|
options.Value.OnnxModelPath);
|
||||||
|
|
||||||
|
var modelDirectory = Path.GetDirectoryName(modelPath) ?? Environment.CurrentDirectory;
|
||||||
|
var labels = OnnxAssetPathResolver.LoadLabels(modelDirectory);
|
||||||
|
var encoder = new BertWordPieceEncoder(modelDirectory, logger);
|
||||||
|
_runner = new OnnxTokenClassifierRunner(modelPath, encoder, NerLabelConfig.English, labels, logger);
|
||||||
|
}
|
||||||
|
|
||||||
|
public EnglishOnnxNerRunner(IOptions<PiiRedactionOptions> options, ILogger<EnglishOnnxNerRunner> logger)
|
||||||
|
: this(options, (ILogger)logger)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool IsModelAvailable => _runner.IsAvailable;
|
||||||
|
|
||||||
|
public NerPredictionResult PredictEntities(string text)
|
||||||
|
{
|
||||||
|
if (!IsModelAvailable)
|
||||||
|
{
|
||||||
|
return NerPredictionResult.Empty;
|
||||||
|
}
|
||||||
|
|
||||||
|
var entities = _runner.PredictEntities(text);
|
||||||
|
return new NerPredictionResult(
|
||||||
|
NerEntityTagging.WithOrigin(entities, NerModelOrigin.English),
|
||||||
|
[NerModelOrigin.English]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose() => _runner.Dispose();
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
namespace PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
public sealed record EncodedSequence(
|
||||||
|
long[] InputIds,
|
||||||
|
long[] AttentionMask,
|
||||||
|
long[] TokenTypeIds,
|
||||||
|
(int Start, int End)[] Offsets,
|
||||||
|
int[] TokenIds,
|
||||||
|
int SequenceLength);
|
||||||
|
|
||||||
|
public interface ITokenClassifierEncoder
|
||||||
|
{
|
||||||
|
bool IsAvailable { get; }
|
||||||
|
|
||||||
|
EncodedSequence? Encode(string text, int maxSequenceLength);
|
||||||
|
|
||||||
|
bool IsSpecialToken(int tokenId);
|
||||||
|
}
|
||||||
26
src/PiiRedaction.Infrastructure/Onnx/NerLabelConfig.cs
Normal file
26
src/PiiRedaction.Infrastructure/Onnx/NerLabelConfig.cs
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
namespace PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
public sealed class NerLabelConfig
|
||||||
|
{
|
||||||
|
private readonly Func<string, bool> _isPersonLabel;
|
||||||
|
|
||||||
|
private NerLabelConfig(Func<string, bool> isPersonLabel) => _isPersonLabel = isPersonLabel;
|
||||||
|
|
||||||
|
public static NerLabelConfig English { get; } = new(IsEnglishPersonLabel);
|
||||||
|
|
||||||
|
public static NerLabelConfig Tamil { get; } = new(IsTamilPersonLabel);
|
||||||
|
|
||||||
|
public bool IsPersonLabel(string label) => _isPersonLabel(label);
|
||||||
|
|
||||||
|
public bool IsBeginLabel(string label) => label.StartsWith("B-", StringComparison.Ordinal);
|
||||||
|
|
||||||
|
public bool IsInsideLabel(string label) => label.StartsWith("I-", StringComparison.Ordinal);
|
||||||
|
|
||||||
|
private static bool IsEnglishPersonLabel(string label) =>
|
||||||
|
label is "B-PER" or "I-PER" or "B-PERSON" or "I-PERSON"
|
||||||
|
|| (label.EndsWith("-PER", StringComparison.Ordinal) &&
|
||||||
|
(label.StartsWith("B-", StringComparison.Ordinal) || label.StartsWith("I-", StringComparison.Ordinal)));
|
||||||
|
|
||||||
|
private static bool IsTamilPersonLabel(string label) =>
|
||||||
|
label.Contains("person", StringComparison.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
namespace PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
public static class OnnxAssetPathResolver
|
||||||
|
{
|
||||||
|
public static string ResolveAssetPath(string configuredPath)
|
||||||
|
{
|
||||||
|
if (Path.IsPathRooted(configuredPath) && File.Exists(configuredPath))
|
||||||
|
{
|
||||||
|
return configuredPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
var directory = new DirectoryInfo(Environment.CurrentDirectory);
|
||||||
|
while (directory is not null)
|
||||||
|
{
|
||||||
|
var candidate = Path.GetFullPath(Path.Combine(directory.FullName, configuredPath));
|
||||||
|
if (File.Exists(candidate))
|
||||||
|
{
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
|
||||||
|
directory = directory.Parent;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, configuredPath));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string ResolveModelPath(string primaryPath, params string[] fallbackPaths)
|
||||||
|
{
|
||||||
|
foreach (var candidatePath in new[] { primaryPath }.Concat(fallbackPaths))
|
||||||
|
{
|
||||||
|
var resolved = ResolveAssetPath(candidatePath);
|
||||||
|
if (File.Exists(resolved))
|
||||||
|
{
|
||||||
|
return resolved;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ResolveAssetPath(primaryPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string[] LoadLabels(string modelDirectory)
|
||||||
|
{
|
||||||
|
var labelsPath = ResolveAssetPath(Path.Combine(modelDirectory, "ner-labels.txt"));
|
||||||
|
if (!File.Exists(labelsPath))
|
||||||
|
{
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return File.ReadAllLines(labelsPath)
|
||||||
|
.Where(line => !string.IsNullOrWhiteSpace(line))
|
||||||
|
.ToArray();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,325 +1,16 @@
|
|||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using Microsoft.Extensions.Options;
|
using Microsoft.Extensions.Options;
|
||||||
using Microsoft.ML.OnnxRuntime;
|
|
||||||
using Microsoft.ML.OnnxRuntime.Tensors;
|
|
||||||
using Microsoft.ML.Tokenizers;
|
|
||||||
using PiiRedaction.Core.Configuration;
|
using PiiRedaction.Core.Configuration;
|
||||||
using PiiRedaction.Core.Detection;
|
|
||||||
using PiiRedaction.Core.Models;
|
|
||||||
|
|
||||||
namespace PiiRedaction.Infrastructure.Onnx;
|
namespace PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Wraps ONNX Runtime inference for NER models.
|
/// Backward-compatible alias for <see cref="EnglishOnnxNerRunner"/>.
|
||||||
/// Tokenization and tensor preparation are isolated here so detectors remain model-agnostic.
|
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class OnnxNerModelRunner : IOnnxNerModelRunner, IDisposable
|
public sealed class OnnxNerModelRunner : EnglishOnnxNerRunner
|
||||||
{
|
{
|
||||||
private const int MaxSequenceLength = 128;
|
|
||||||
|
|
||||||
private readonly ILogger<OnnxNerModelRunner> _logger;
|
|
||||||
private readonly string _modelPath;
|
|
||||||
private readonly BertTokenizer? _tokenizer;
|
|
||||||
private readonly string[] _labels;
|
|
||||||
private InferenceSession? _session;
|
|
||||||
|
|
||||||
public OnnxNerModelRunner(IOptions<PiiRedactionOptions> options, ILogger<OnnxNerModelRunner> logger)
|
public OnnxNerModelRunner(IOptions<PiiRedactionOptions> options, ILogger<OnnxNerModelRunner> logger)
|
||||||
|
: base(options, logger)
|
||||||
{
|
{
|
||||||
_logger = logger;
|
|
||||||
_modelPath = ResolveAssetPath(options.Value.OnnxModelPath);
|
|
||||||
var modelDirectory = Path.GetDirectoryName(_modelPath) ?? Environment.CurrentDirectory;
|
|
||||||
_labels = LoadLabels(modelDirectory);
|
|
||||||
_tokenizer = TryLoadTokenizer(modelDirectory);
|
|
||||||
_session = TryCreateSession();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public bool IsModelAvailable => _session is not null && _tokenizer is not null && _labels.Length > 0;
|
|
||||||
|
|
||||||
public IReadOnlyList<PiiEntity> PredictEntities(string text)
|
|
||||||
{
|
|
||||||
if (!IsModelAvailable || _session is null || _tokenizer is null)
|
|
||||||
{
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
var encodedTokens = _tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true);
|
|
||||||
var wordTokens = encodedTokens.Take(Math.Max(0, MaxSequenceLength - 2)).ToList();
|
|
||||||
if (wordTokens.Count == 0)
|
|
||||||
{
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
var sequenceLength = wordTokens.Count + 2;
|
|
||||||
var inputIds = new long[sequenceLength];
|
|
||||||
var attentionMask = new long[sequenceLength];
|
|
||||||
var tokenTypeIds = new long[sequenceLength];
|
|
||||||
var offsets = new (int Start, int End)[sequenceLength];
|
|
||||||
var tokenIds = new int[sequenceLength];
|
|
||||||
|
|
||||||
inputIds[0] = _tokenizer.ClassificationTokenId;
|
|
||||||
attentionMask[0] = 1;
|
|
||||||
tokenIds[0] = _tokenizer.ClassificationTokenId;
|
|
||||||
offsets[0] = (0, 0);
|
|
||||||
|
|
||||||
for (var i = 0; i < wordTokens.Count; i++)
|
|
||||||
{
|
|
||||||
var token = wordTokens[i];
|
|
||||||
var index = i + 1;
|
|
||||||
inputIds[index] = token.Id;
|
|
||||||
attentionMask[index] = 1;
|
|
||||||
tokenIds[index] = token.Id;
|
|
||||||
offsets[index] = ToCharOffsets(token.Offset, text.Length);
|
|
||||||
}
|
|
||||||
|
|
||||||
inputIds[sequenceLength - 1] = _tokenizer.SeparatorTokenId;
|
|
||||||
attentionMask[sequenceLength - 1] = 1;
|
|
||||||
tokenIds[sequenceLength - 1] = _tokenizer.SeparatorTokenId;
|
|
||||||
offsets[sequenceLength - 1] = (0, 0);
|
|
||||||
|
|
||||||
var predictedLabelIds = RunInference(inputIds, attentionMask, tokenTypeIds, sequenceLength);
|
|
||||||
return DecodePersonEntities(text, predictedLabelIds, offsets, tokenIds, sequenceLength);
|
|
||||||
}
|
|
||||||
|
|
||||||
private int[] RunInference(long[] inputIds, long[] attentionMask, long[] tokenTypeIds, int sequenceLength)
|
|
||||||
{
|
|
||||||
var inputIdsTensor = CreateTensor(inputIds, sequenceLength);
|
|
||||||
var attentionMaskTensor = CreateTensor(attentionMask, sequenceLength);
|
|
||||||
var inputs = new List<NamedOnnxValue>
|
|
||||||
{
|
|
||||||
NamedOnnxValue.CreateFromTensor(_session!.InputMetadata.Keys.First(key => key.Contains("input_ids", StringComparison.OrdinalIgnoreCase)), inputIdsTensor),
|
|
||||||
NamedOnnxValue.CreateFromTensor(_session.InputMetadata.Keys.First(key => key.Contains("attention_mask", StringComparison.OrdinalIgnoreCase)), attentionMaskTensor)
|
|
||||||
};
|
|
||||||
|
|
||||||
var tokenTypeInputName = _session.InputMetadata.Keys.FirstOrDefault(key => key.Contains("token_type", StringComparison.OrdinalIgnoreCase));
|
|
||||||
if (tokenTypeInputName is not null)
|
|
||||||
{
|
|
||||||
inputs.Add(NamedOnnxValue.CreateFromTensor(tokenTypeInputName, CreateTensor(tokenTypeIds, sequenceLength)));
|
|
||||||
}
|
|
||||||
|
|
||||||
using var results = _session.Run(inputs);
|
|
||||||
var outputName = _session.OutputMetadata.Keys.FirstOrDefault(key =>
|
|
||||||
key.Contains("logits", StringComparison.OrdinalIgnoreCase))
|
|
||||||
?? results.First().Name;
|
|
||||||
var logits = results.First(result => result.Name == outputName).AsTensor<float>();
|
|
||||||
var numLabels = _labels.Length;
|
|
||||||
var predictions = new int[sequenceLength];
|
|
||||||
|
|
||||||
for (var tokenIndex = 0; tokenIndex < sequenceLength; tokenIndex++)
|
|
||||||
{
|
|
||||||
var bestLabel = 0;
|
|
||||||
var bestScore = float.MinValue;
|
|
||||||
|
|
||||||
for (var labelIndex = 0; labelIndex < numLabels; labelIndex++)
|
|
||||||
{
|
|
||||||
var score = logits[0, tokenIndex, labelIndex];
|
|
||||||
if (score > bestScore)
|
|
||||||
{
|
|
||||||
bestScore = score;
|
|
||||||
bestLabel = labelIndex;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
predictions[tokenIndex] = bestLabel;
|
|
||||||
}
|
|
||||||
|
|
||||||
return predictions;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static DenseTensor<long> CreateTensor(long[] values, int sequenceLength)
|
|
||||||
{
|
|
||||||
var tensor = new DenseTensor<long>([1, sequenceLength]);
|
|
||||||
for (var i = 0; i < sequenceLength; i++)
|
|
||||||
{
|
|
||||||
tensor[0, i] = values[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
return tensor;
|
|
||||||
}
|
|
||||||
|
|
||||||
private IReadOnlyList<PiiEntity> DecodePersonEntities(
|
|
||||||
string text,
|
|
||||||
int[] predictedLabelIds,
|
|
||||||
(int Start, int End)[] offsets,
|
|
||||||
int[] tokenIds,
|
|
||||||
int sequenceLength)
|
|
||||||
{
|
|
||||||
var entities = new List<PiiEntity>();
|
|
||||||
int? entityStart = null;
|
|
||||||
int? entityEnd = null;
|
|
||||||
|
|
||||||
void FlushEntity()
|
|
||||||
{
|
|
||||||
if (!entityStart.HasValue || !entityEnd.HasValue || entityEnd.Value <= entityStart.Value)
|
|
||||||
{
|
|
||||||
entityStart = null;
|
|
||||||
entityEnd = null;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
var value = text[entityStart.Value..entityEnd.Value];
|
|
||||||
if (!string.IsNullOrWhiteSpace(value))
|
|
||||||
{
|
|
||||||
entities.Add(new PiiEntity(
|
|
||||||
PiiEntityType.Person,
|
|
||||||
value,
|
|
||||||
entityStart.Value,
|
|
||||||
entityEnd.Value - entityStart.Value,
|
|
||||||
PiiDetectionSource.Ner));
|
|
||||||
}
|
|
||||||
|
|
||||||
entityStart = null;
|
|
||||||
entityEnd = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (var i = 0; i < sequenceLength; i++)
|
|
||||||
{
|
|
||||||
if (IsSpecialToken(tokenIds[i]))
|
|
||||||
{
|
|
||||||
FlushEntity();
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
var label = _labels[predictedLabelIds[i]];
|
|
||||||
var (start, end) = offsets[i];
|
|
||||||
var hasOffset = end > start;
|
|
||||||
|
|
||||||
if (!IsPersonLabel(label))
|
|
||||||
{
|
|
||||||
FlushEntity();
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (label.StartsWith("B-", StringComparison.Ordinal))
|
|
||||||
{
|
|
||||||
FlushEntity();
|
|
||||||
if (hasOffset)
|
|
||||||
{
|
|
||||||
entityStart = start;
|
|
||||||
entityEnd = end;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else if (label.StartsWith("I-", StringComparison.Ordinal))
|
|
||||||
{
|
|
||||||
if (!entityStart.HasValue && hasOffset)
|
|
||||||
{
|
|
||||||
entityStart = start;
|
|
||||||
entityEnd = end;
|
|
||||||
}
|
|
||||||
else if (hasOffset)
|
|
||||||
{
|
|
||||||
entityEnd = Math.Max(entityEnd ?? end, end);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
FlushEntity();
|
|
||||||
return entities;
|
|
||||||
}
|
|
||||||
|
|
||||||
private bool IsSpecialToken(int tokenId) =>
|
|
||||||
tokenId == _tokenizer!.ClassificationTokenId ||
|
|
||||||
tokenId == _tokenizer.SeparatorTokenId ||
|
|
||||||
tokenId == _tokenizer.PaddingTokenId;
|
|
||||||
|
|
||||||
private static (int Start, int End) ToCharOffsets(Range offset, int textLength)
|
|
||||||
{
|
|
||||||
var (start, length) = offset.GetOffsetAndLength(textLength);
|
|
||||||
return (start, start + length);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static bool IsPersonLabel(string label) =>
|
|
||||||
label is "B-PER" or "I-PER" or "B-PERSON" or "I-PERSON"
|
|
||||||
|| (label.EndsWith("-PER", StringComparison.Ordinal) &&
|
|
||||||
(label.StartsWith("B-", StringComparison.Ordinal) || label.StartsWith("I-", StringComparison.Ordinal)));
|
|
||||||
|
|
||||||
private InferenceSession? TryCreateSession()
|
|
||||||
{
|
|
||||||
if (!File.Exists(_modelPath))
|
|
||||||
{
|
|
||||||
_logger.LogWarning(
|
|
||||||
"ONNX NER model not found at {ModelPath}. Person-name detection will return no results.",
|
|
||||||
_modelPath);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (_tokenizer is null || _labels.Length == 0)
|
|
||||||
{
|
|
||||||
_logger.LogWarning(
|
|
||||||
"Tokenizer vocabulary or label map missing for ONNX NER model at {ModelPath}.",
|
|
||||||
_modelPath);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
var session = new InferenceSession(_modelPath);
|
|
||||||
_logger.LogInformation("ONNX NER model loaded from {ModelPath}.", _modelPath);
|
|
||||||
return session;
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
_logger.LogError(ex, "Failed to load ONNX NER model from {ModelPath}.", _modelPath);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private BertTokenizer? TryLoadTokenizer(string modelDirectory)
|
|
||||||
{
|
|
||||||
var vocabPath = ResolveAssetPath(Path.Combine(modelDirectory, "vocab.txt"));
|
|
||||||
if (!File.Exists(vocabPath))
|
|
||||||
{
|
|
||||||
_logger.LogWarning("Tokenizer vocabulary not found at {VocabPath}.", vocabPath);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
return BertTokenizer.Create(vocabPath, new BertOptions
|
|
||||||
{
|
|
||||||
LowerCaseBeforeTokenization = false
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
_logger.LogError(ex, "Failed to load BERT tokenizer from {VocabPath}.", vocabPath);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static string[] LoadLabels(string modelDirectory)
|
|
||||||
{
|
|
||||||
var labelsPath = ResolveAssetPath(Path.Combine(modelDirectory, "ner-labels.txt"));
|
|
||||||
if (!File.Exists(labelsPath))
|
|
||||||
{
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
return File.ReadAllLines(labelsPath)
|
|
||||||
.Where(line => !string.IsNullOrWhiteSpace(line))
|
|
||||||
.ToArray();
|
|
||||||
}
|
|
||||||
|
|
||||||
internal static string ResolveAssetPath(string configuredPath)
|
|
||||||
{
|
|
||||||
if (Path.IsPathRooted(configuredPath) && File.Exists(configuredPath))
|
|
||||||
{
|
|
||||||
return configuredPath;
|
|
||||||
}
|
|
||||||
|
|
||||||
var directory = new DirectoryInfo(Environment.CurrentDirectory);
|
|
||||||
while (directory is not null)
|
|
||||||
{
|
|
||||||
var candidate = Path.GetFullPath(Path.Combine(directory.FullName, configuredPath));
|
|
||||||
if (File.Exists(candidate))
|
|
||||||
{
|
|
||||||
return candidate;
|
|
||||||
}
|
|
||||||
|
|
||||||
directory = directory.Parent;
|
|
||||||
}
|
|
||||||
|
|
||||||
return Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, configuredPath));
|
|
||||||
}
|
|
||||||
|
|
||||||
public void Dispose() => _session?.Dispose();
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,245 @@
|
|||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.ML.OnnxRuntime;
|
||||||
|
using Microsoft.ML.OnnxRuntime.Tensors;
|
||||||
|
using PiiRedaction.Core.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Shared ONNX token-classification inference and BIO decoding for NER models.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class OnnxTokenClassifierRunner : IDisposable
|
||||||
|
{
|
||||||
|
private const int MaxSequenceLength = 128;
|
||||||
|
|
||||||
|
private readonly ITokenClassifierEncoder _encoder;
|
||||||
|
private readonly NerLabelConfig _labelConfig;
|
||||||
|
private readonly string[] _labels;
|
||||||
|
private readonly ILogger _logger;
|
||||||
|
private readonly string _modelPath;
|
||||||
|
private InferenceSession? _session;
|
||||||
|
|
||||||
|
public OnnxTokenClassifierRunner(
|
||||||
|
string modelPath,
|
||||||
|
ITokenClassifierEncoder encoder,
|
||||||
|
NerLabelConfig labelConfig,
|
||||||
|
string[] labels,
|
||||||
|
ILogger logger)
|
||||||
|
{
|
||||||
|
_modelPath = modelPath;
|
||||||
|
_encoder = encoder;
|
||||||
|
_labelConfig = labelConfig;
|
||||||
|
_labels = labels;
|
||||||
|
_logger = logger;
|
||||||
|
_session = TryCreateSession();
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool IsAvailable => _session is not null && _encoder.IsAvailable && _labels.Length > 0;
|
||||||
|
|
||||||
|
public IReadOnlyList<PiiEntity> PredictEntities(string text)
|
||||||
|
{
|
||||||
|
if (!IsAvailable || _session is null)
|
||||||
|
{
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
var encoded = _encoder.Encode(text, MaxSequenceLength);
|
||||||
|
if (encoded is null)
|
||||||
|
{
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
var predictedLabelIds = RunInference(encoded);
|
||||||
|
var entities = DecodePersonEntities(text, predictedLabelIds, encoded);
|
||||||
|
|
||||||
|
if (entities.Count == 0)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("Decoded 0 person span(s) from {ModelPath}.", _modelPath);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
var spanSummary = string.Join(", ", entities.Select(entity => $"\"{entity.Value}\"@{entity.StartIndex}"));
|
||||||
|
_logger.LogDebug(
|
||||||
|
"Decoded {Count} person span(s) from {ModelPath}: {Spans}",
|
||||||
|
entities.Count,
|
||||||
|
_modelPath,
|
||||||
|
spanSummary);
|
||||||
|
}
|
||||||
|
|
||||||
|
return entities;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int[] RunInference(EncodedSequence encoded)
|
||||||
|
{
|
||||||
|
var inputIdsTensor = CreateTensor(encoded.InputIds, encoded.SequenceLength);
|
||||||
|
var attentionMaskTensor = CreateTensor(encoded.AttentionMask, encoded.SequenceLength);
|
||||||
|
var inputs = new List<NamedOnnxValue>
|
||||||
|
{
|
||||||
|
NamedOnnxValue.CreateFromTensor(
|
||||||
|
_session!.InputMetadata.Keys.First(key => key.Contains("input_ids", StringComparison.OrdinalIgnoreCase)),
|
||||||
|
inputIdsTensor),
|
||||||
|
NamedOnnxValue.CreateFromTensor(
|
||||||
|
_session.InputMetadata.Keys.First(key => key.Contains("attention_mask", StringComparison.OrdinalIgnoreCase)),
|
||||||
|
attentionMaskTensor)
|
||||||
|
};
|
||||||
|
|
||||||
|
var tokenTypeInputName = _session.InputMetadata.Keys.FirstOrDefault(key =>
|
||||||
|
key.Contains("token_type", StringComparison.OrdinalIgnoreCase));
|
||||||
|
if (tokenTypeInputName is not null)
|
||||||
|
{
|
||||||
|
inputs.Add(NamedOnnxValue.CreateFromTensor(
|
||||||
|
tokenTypeInputName,
|
||||||
|
CreateTensor(encoded.TokenTypeIds, encoded.SequenceLength)));
|
||||||
|
}
|
||||||
|
|
||||||
|
using var results = _session.Run(inputs);
|
||||||
|
var outputName = _session.OutputMetadata.Keys.FirstOrDefault(key =>
|
||||||
|
key.Contains("logits", StringComparison.OrdinalIgnoreCase))
|
||||||
|
?? results.First().Name;
|
||||||
|
var logits = results.First(result => result.Name == outputName).AsTensor<float>();
|
||||||
|
var numLabels = _labels.Length;
|
||||||
|
var predictions = new int[encoded.SequenceLength];
|
||||||
|
|
||||||
|
for (var tokenIndex = 0; tokenIndex < encoded.SequenceLength; tokenIndex++)
|
||||||
|
{
|
||||||
|
var bestLabel = 0;
|
||||||
|
var bestScore = float.MinValue;
|
||||||
|
|
||||||
|
for (var labelIndex = 0; labelIndex < numLabels; labelIndex++)
|
||||||
|
{
|
||||||
|
var score = logits[0, tokenIndex, labelIndex];
|
||||||
|
if (score > bestScore)
|
||||||
|
{
|
||||||
|
bestScore = score;
|
||||||
|
bestLabel = labelIndex;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
predictions[tokenIndex] = bestLabel;
|
||||||
|
}
|
||||||
|
|
||||||
|
return predictions;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static DenseTensor<long> CreateTensor(long[] values, int sequenceLength)
|
||||||
|
{
|
||||||
|
var tensor = new DenseTensor<long>([1, sequenceLength]);
|
||||||
|
for (var i = 0; i < sequenceLength; i++)
|
||||||
|
{
|
||||||
|
tensor[0, i] = values[i];
|
||||||
|
}
|
||||||
|
|
||||||
|
return tensor;
|
||||||
|
}
|
||||||
|
|
||||||
|
private IReadOnlyList<PiiEntity> DecodePersonEntities(
|
||||||
|
string text,
|
||||||
|
int[] predictedLabelIds,
|
||||||
|
EncodedSequence encoded)
|
||||||
|
{
|
||||||
|
var entities = new List<PiiEntity>();
|
||||||
|
int? entityStart = null;
|
||||||
|
int? entityEnd = null;
|
||||||
|
|
||||||
|
void FlushEntity()
|
||||||
|
{
|
||||||
|
if (!entityStart.HasValue || !entityEnd.HasValue || entityEnd.Value <= entityStart.Value)
|
||||||
|
{
|
||||||
|
entityStart = null;
|
||||||
|
entityEnd = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var value = text[entityStart.Value..entityEnd.Value];
|
||||||
|
if (!string.IsNullOrWhiteSpace(value))
|
||||||
|
{
|
||||||
|
entities.Add(new PiiEntity(
|
||||||
|
PiiEntityType.Person,
|
||||||
|
value,
|
||||||
|
entityStart.Value,
|
||||||
|
entityEnd.Value - entityStart.Value,
|
||||||
|
PiiDetectionSource.Ner));
|
||||||
|
}
|
||||||
|
|
||||||
|
entityStart = null;
|
||||||
|
entityEnd = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (var i = 0; i < encoded.SequenceLength; i++)
|
||||||
|
{
|
||||||
|
if (_encoder.IsSpecialToken(encoded.TokenIds[i]))
|
||||||
|
{
|
||||||
|
FlushEntity();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var label = _labels[predictedLabelIds[i]];
|
||||||
|
var (start, end) = encoded.Offsets[i];
|
||||||
|
var hasOffset = end > start;
|
||||||
|
|
||||||
|
if (!_labelConfig.IsPersonLabel(label))
|
||||||
|
{
|
||||||
|
FlushEntity();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (_labelConfig.IsBeginLabel(label))
|
||||||
|
{
|
||||||
|
FlushEntity();
|
||||||
|
if (hasOffset)
|
||||||
|
{
|
||||||
|
entityStart = start;
|
||||||
|
entityEnd = end;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (_labelConfig.IsInsideLabel(label))
|
||||||
|
{
|
||||||
|
if (!entityStart.HasValue && hasOffset)
|
||||||
|
{
|
||||||
|
entityStart = start;
|
||||||
|
entityEnd = end;
|
||||||
|
}
|
||||||
|
else if (hasOffset)
|
||||||
|
{
|
||||||
|
entityEnd = Math.Max(entityEnd ?? end, end);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
FlushEntity();
|
||||||
|
return entities;
|
||||||
|
}
|
||||||
|
|
||||||
|
private InferenceSession? TryCreateSession()
|
||||||
|
{
|
||||||
|
if (!File.Exists(_modelPath))
|
||||||
|
{
|
||||||
|
_logger.LogWarning(
|
||||||
|
"ONNX NER model not found at {ModelPath}. Person-name detection will return no results.",
|
||||||
|
_modelPath);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!_encoder.IsAvailable || _labels.Length == 0)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(
|
||||||
|
"Tokenizer or label map missing for ONNX NER model at {ModelPath}.",
|
||||||
|
_modelPath);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var session = new InferenceSession(_modelPath);
|
||||||
|
_logger.LogInformation("ONNX NER model loaded from {ModelPath}.", _modelPath);
|
||||||
|
return session;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Failed to load ONNX NER model from {ModelPath}.", _modelPath);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose() => _session?.Dispose();
|
||||||
|
}
|
||||||
@@ -0,0 +1,157 @@
|
|||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
using PiiRedaction.Core.Configuration;
|
||||||
|
using PiiRedaction.Core.Detection;
|
||||||
|
using PiiRedaction.Core.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Routes NER inference to English and/or Tamil ONNX models based on script composition.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class RoutingOnnxNerModelRunner : IOnnxNerModelRunner
|
||||||
|
{
|
||||||
|
private readonly ScriptRouter _scriptRouter = new();
|
||||||
|
private readonly IOnnxNerModelRunner _englishRunner;
|
||||||
|
private readonly IOnnxNerModelRunner _tamilRunner;
|
||||||
|
private readonly bool _enableTamilNer;
|
||||||
|
private readonly ILogger<RoutingOnnxNerModelRunner> _logger;
|
||||||
|
|
||||||
|
public RoutingOnnxNerModelRunner(
|
||||||
|
EnglishOnnxNerRunner englishRunner,
|
||||||
|
TamilOnnxNerRunner tamilRunner,
|
||||||
|
IOptions<PiiRedactionOptions> options,
|
||||||
|
ILogger<RoutingOnnxNerModelRunner> logger)
|
||||||
|
: this(englishRunner, tamilRunner, options.Value.EnableTamilNer, logger)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
internal RoutingOnnxNerModelRunner(
|
||||||
|
IOnnxNerModelRunner englishRunner,
|
||||||
|
IOnnxNerModelRunner tamilRunner,
|
||||||
|
bool enableTamilNer,
|
||||||
|
ILogger<RoutingOnnxNerModelRunner>? logger = null)
|
||||||
|
{
|
||||||
|
_englishRunner = englishRunner;
|
||||||
|
_tamilRunner = tamilRunner;
|
||||||
|
_enableTamilNer = enableTamilNer;
|
||||||
|
_logger = logger ?? NullLogger<RoutingOnnxNerModelRunner>.Instance;
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool IsModelAvailable =>
|
||||||
|
_englishRunner.IsModelAvailable || (_enableTamilNer && _tamilRunner.IsModelAvailable);
|
||||||
|
|
||||||
|
public NerPredictionResult PredictEntities(string text)
|
||||||
|
{
|
||||||
|
var composition = _scriptRouter.GetComposition(text);
|
||||||
|
_logger.LogDebug("Script composition: {Composition}", composition);
|
||||||
|
|
||||||
|
var entities = new List<PiiEntity>();
|
||||||
|
var invoked = new List<NerModelOrigin>();
|
||||||
|
|
||||||
|
switch (composition)
|
||||||
|
{
|
||||||
|
case ScriptComposition.LatinOnly:
|
||||||
|
if (_englishRunner.IsModelAvailable)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("Invoking English NER model.");
|
||||||
|
var englishResult = _englishRunner.PredictEntities(text);
|
||||||
|
entities.AddRange(englishResult.Entities);
|
||||||
|
invoked.AddRange(englishResult.InvokedModels);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogDebug("Skipping English NER model (not available).");
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
|
||||||
|
case ScriptComposition.TamilOnly:
|
||||||
|
if (_enableTamilNer && _tamilRunner.IsModelAvailable)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("Invoking Tamil NER model.");
|
||||||
|
var tamilResult = _tamilRunner.PredictEntities(text);
|
||||||
|
entities.AddRange(tamilResult.Entities);
|
||||||
|
invoked.AddRange(tamilResult.InvokedModels);
|
||||||
|
}
|
||||||
|
else if (!_enableTamilNer)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("Skipping Tamil NER model (disabled in options).");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogDebug("Skipping Tamil NER model (not available).");
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
|
||||||
|
case ScriptComposition.Mixed:
|
||||||
|
if (_englishRunner.IsModelAvailable)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("Invoking English NER model (mixed script).");
|
||||||
|
var englishResult = _englishRunner.PredictEntities(text);
|
||||||
|
entities.AddRange(englishResult.Entities);
|
||||||
|
invoked.AddRange(englishResult.InvokedModels);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogDebug("Skipping English NER model (not available, mixed script).");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (_enableTamilNer && _tamilRunner.IsModelAvailable)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("Invoking Tamil NER model (mixed script).");
|
||||||
|
var tamilResult = _tamilRunner.PredictEntities(text);
|
||||||
|
entities.AddRange(tamilResult.Entities);
|
||||||
|
invoked.AddRange(tamilResult.InvokedModels);
|
||||||
|
}
|
||||||
|
else if (!_enableTamilNer)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("Skipping Tamil NER model (disabled in options, mixed script).");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogDebug("Skipping Tamil NER model (not available, mixed script).");
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
|
||||||
|
case ScriptComposition.NoLetters:
|
||||||
|
_logger.LogDebug("No letters detected; skipping all NER models.");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
var merged = MergePersonSpans(entities);
|
||||||
|
_logger.LogInformation(
|
||||||
|
"NER routing complete: {EntityCount} person span(s) from [{InvokedModels}].",
|
||||||
|
merged.Count,
|
||||||
|
invoked.Count == 0 ? "none" : string.Join(", ", invoked));
|
||||||
|
|
||||||
|
return new NerPredictionResult(merged, invoked);
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static IReadOnlyList<PiiEntity> MergePersonSpans(IReadOnlyList<PiiEntity> entities)
|
||||||
|
{
|
||||||
|
if (entities.Count <= 1)
|
||||||
|
{
|
||||||
|
return entities;
|
||||||
|
}
|
||||||
|
|
||||||
|
var accepted = new List<PiiEntity>();
|
||||||
|
foreach (var candidate in entities.OrderByDescending(entity => entity.Length).ThenBy(entity => entity.StartIndex))
|
||||||
|
{
|
||||||
|
if (accepted.Any(existing => Overlaps(existing, candidate)))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
accepted.Add(candidate);
|
||||||
|
}
|
||||||
|
|
||||||
|
return accepted.OrderBy(entity => entity.StartIndex).ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool Overlaps(PiiEntity left, PiiEntity right) =>
|
||||||
|
left.StartIndex < right.EndIndex && right.StartIndex < left.EndIndex;
|
||||||
|
}
|
||||||
102
src/PiiRedaction.Infrastructure/Onnx/SentencePieceEncoder.cs
Normal file
102
src/PiiRedaction.Infrastructure/Onnx/SentencePieceEncoder.cs
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.ML.Tokenizers;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
public sealed class SentencePieceEncoder : ITokenClassifierEncoder
|
||||||
|
{
|
||||||
|
private readonly SentencePieceTokenizer? _tokenizer;
|
||||||
|
private readonly ILogger _logger;
|
||||||
|
|
||||||
|
public SentencePieceEncoder(string modelDirectory, ILogger logger)
|
||||||
|
{
|
||||||
|
_logger = logger;
|
||||||
|
_tokenizer = TryLoadTokenizer(modelDirectory);
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool IsAvailable => _tokenizer is not null;
|
||||||
|
|
||||||
|
public EncodedSequence? Encode(string text, int maxSequenceLength)
|
||||||
|
{
|
||||||
|
if (_tokenizer is null)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var encodedTokens = _tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true);
|
||||||
|
var wordTokens = encodedTokens.Take(Math.Max(0, maxSequenceLength - 2)).ToList();
|
||||||
|
if (wordTokens.Count == 0)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var sequenceLength = wordTokens.Count + 2;
|
||||||
|
var inputIds = new long[sequenceLength];
|
||||||
|
var attentionMask = new long[sequenceLength];
|
||||||
|
var tokenTypeIds = new long[sequenceLength];
|
||||||
|
var offsets = new (int Start, int End)[sequenceLength];
|
||||||
|
var tokenIds = new int[sequenceLength];
|
||||||
|
|
||||||
|
inputIds[0] = _tokenizer.BeginningOfSentenceId;
|
||||||
|
attentionMask[0] = 1;
|
||||||
|
tokenIds[0] = _tokenizer.BeginningOfSentenceId;
|
||||||
|
offsets[0] = (0, 0);
|
||||||
|
|
||||||
|
for (var i = 0; i < wordTokens.Count; i++)
|
||||||
|
{
|
||||||
|
var token = wordTokens[i];
|
||||||
|
var index = i + 1;
|
||||||
|
inputIds[index] = token.Id;
|
||||||
|
attentionMask[index] = 1;
|
||||||
|
tokenIds[index] = token.Id;
|
||||||
|
offsets[index] = ToCharOffsets(token.Offset, text.Length);
|
||||||
|
}
|
||||||
|
|
||||||
|
inputIds[sequenceLength - 1] = _tokenizer.EndOfSentenceId;
|
||||||
|
attentionMask[sequenceLength - 1] = 1;
|
||||||
|
tokenIds[sequenceLength - 1] = _tokenizer.EndOfSentenceId;
|
||||||
|
offsets[sequenceLength - 1] = (0, 0);
|
||||||
|
|
||||||
|
return new EncodedSequence(inputIds, attentionMask, tokenTypeIds, offsets, tokenIds, sequenceLength);
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool IsSpecialToken(int tokenId) =>
|
||||||
|
_tokenizer is not null &&
|
||||||
|
(tokenId == _tokenizer.BeginningOfSentenceId ||
|
||||||
|
tokenId == _tokenizer.EndOfSentenceId ||
|
||||||
|
tokenId == _tokenizer.UnknownId);
|
||||||
|
|
||||||
|
private SentencePieceTokenizer? TryLoadTokenizer(string modelDirectory)
|
||||||
|
{
|
||||||
|
foreach (var fileName in new[] { "sentencepiece.bpe.model", "spiece.model", "tokenizer.model" })
|
||||||
|
{
|
||||||
|
var modelPath = OnnxAssetPathResolver.ResolveAssetPath(Path.Combine(modelDirectory, fileName));
|
||||||
|
if (!File.Exists(modelPath))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var stream = File.OpenRead(modelPath);
|
||||||
|
return SentencePieceTokenizer.Create(stream, addBeginningOfSentence: false, addEndOfSentence: false);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Failed to load SentencePiece tokenizer from {ModelPath}.", modelPath);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogWarning(
|
||||||
|
"SentencePiece model not found in {ModelDirectory}. Expected sentencepiece.bpe.model or spiece.model.",
|
||||||
|
modelDirectory);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static (int Start, int End) ToCharOffsets(Range offset, int textLength)
|
||||||
|
{
|
||||||
|
var (start, length) = offset.GetOffsetAndLength(textLength);
|
||||||
|
return (start, start + length);
|
||||||
|
}
|
||||||
|
}
|
||||||
38
src/PiiRedaction.Infrastructure/Onnx/TamilOnnxNerRunner.cs
Normal file
38
src/PiiRedaction.Infrastructure/Onnx/TamilOnnxNerRunner.cs
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
using PiiRedaction.Core.Configuration;
|
||||||
|
using PiiRedaction.Core.Detection;
|
||||||
|
using PiiRedaction.Core.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
public sealed class TamilOnnxNerRunner : IOnnxNerModelRunner, IDisposable
|
||||||
|
{
|
||||||
|
private readonly OnnxTokenClassifierRunner _runner;
|
||||||
|
|
||||||
|
public TamilOnnxNerRunner(IOptions<PiiRedactionOptions> options, ILogger<TamilOnnxNerRunner> logger)
|
||||||
|
{
|
||||||
|
var modelPath = OnnxAssetPathResolver.ResolveModelPath(options.Value.TamilOnnxModelPath);
|
||||||
|
var modelDirectory = Path.GetDirectoryName(modelPath) ?? Environment.CurrentDirectory;
|
||||||
|
var labels = OnnxAssetPathResolver.LoadLabels(modelDirectory);
|
||||||
|
var encoder = TokenClassifierEncoderFactory.Create(modelDirectory, logger);
|
||||||
|
_runner = new OnnxTokenClassifierRunner(modelPath, encoder, NerLabelConfig.Tamil, labels, logger);
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool IsModelAvailable => _runner.IsAvailable;
|
||||||
|
|
||||||
|
public NerPredictionResult PredictEntities(string text)
|
||||||
|
{
|
||||||
|
if (!IsModelAvailable)
|
||||||
|
{
|
||||||
|
return NerPredictionResult.Empty;
|
||||||
|
}
|
||||||
|
|
||||||
|
var entities = _runner.PredictEntities(text);
|
||||||
|
return new NerPredictionResult(
|
||||||
|
NerEntityTagging.WithOrigin(entities, NerModelOrigin.Tamil),
|
||||||
|
[NerModelOrigin.Tamil]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose() => _runner.Dispose();
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
public static class TokenClassifierEncoderFactory
|
||||||
|
{
|
||||||
|
private static readonly string[] SentencePieceFileNames =
|
||||||
|
["sentencepiece.bpe.model", "spiece.model", "tokenizer.model"];
|
||||||
|
|
||||||
|
public static ITokenClassifierEncoder Create(string modelDirectory, ILogger logger)
|
||||||
|
{
|
||||||
|
var vocabPath = OnnxAssetPathResolver.ResolveAssetPath(Path.Combine(modelDirectory, "vocab.txt"));
|
||||||
|
if (File.Exists(vocabPath))
|
||||||
|
{
|
||||||
|
logger.LogInformation(
|
||||||
|
"Using WordPiece tokenizer (vocab.txt) from {ModelDirectory}.",
|
||||||
|
modelDirectory);
|
||||||
|
return new BertWordPieceEncoder(modelDirectory, logger);
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var fileName in SentencePieceFileNames)
|
||||||
|
{
|
||||||
|
var sentencePiecePath = OnnxAssetPathResolver.ResolveAssetPath(
|
||||||
|
Path.Combine(modelDirectory, fileName));
|
||||||
|
if (File.Exists(sentencePiecePath))
|
||||||
|
{
|
||||||
|
logger.LogInformation(
|
||||||
|
"Using SentencePiece tokenizer ({FileName}) from {ModelDirectory}.",
|
||||||
|
fileName,
|
||||||
|
modelDirectory);
|
||||||
|
return new SentencePieceEncoder(modelDirectory, logger);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.LogWarning(
|
||||||
|
"No tokenizer assets found in {ModelDirectory}. Expected vocab.txt or a SentencePiece model file.",
|
||||||
|
modelDirectory);
|
||||||
|
return new BertWordPieceEncoder(modelDirectory, logger);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -19,4 +19,8 @@
|
|||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<InternalsVisibleTo Include="PiiRedaction.Infrastructure.Tests" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
18
src/PiiRedaction.TestHarness.Wpf/App.xaml
Normal file
18
src/PiiRedaction.TestHarness.Wpf/App.xaml
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
<Application x:Class="PiiRedaction.TestHarness.Wpf.App"
|
||||||
|
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
|
||||||
|
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
|
||||||
|
xmlns:converters="clr-namespace:PiiRedaction.TestHarness.Wpf.Converters">
|
||||||
|
<Application.Resources>
|
||||||
|
<ResourceDictionary>
|
||||||
|
<ResourceDictionary.MergedDictionaries>
|
||||||
|
<ResourceDictionary Source="Resources/Styles.xaml" />
|
||||||
|
</ResourceDictionary.MergedDictionaries>
|
||||||
|
|
||||||
|
<converters:ScriptCompositionToBrushConverter x:Key="ScriptCompositionToBrushConverter" />
|
||||||
|
<converters:NerInvokedToBrushConverter x:Key="NerInvokedToBrushConverter" />
|
||||||
|
<converters:BoolToVisibilityConverter x:Key="BoolToVisibilityConverter" />
|
||||||
|
<converters:PassFailBrushConverter x:Key="PassFailBrushConverter" />
|
||||||
|
<converters:StringNotEmptyToVisibilityConverter x:Key="StringNotEmptyToVisibilityConverter" />
|
||||||
|
</ResourceDictionary>
|
||||||
|
</Application.Resources>
|
||||||
|
</Application>
|
||||||
67
src/PiiRedaction.TestHarness.Wpf/App.xaml.cs
Normal file
67
src/PiiRedaction.TestHarness.Wpf/App.xaml.cs
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
using System.IO;
|
||||||
|
using System.Windows;
|
||||||
|
using Microsoft.Extensions.Configuration;
|
||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
using Microsoft.Extensions.Hosting;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using PiiRedaction.TestHarness.Wpf.DependencyInjection;
|
||||||
|
using PiiRedaction.TestHarness.Wpf.Logging;
|
||||||
|
using PiiRedaction.TestHarness.Wpf.Services;
|
||||||
|
using PiiRedaction.TestHarness.Wpf.ViewModels;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf;
|
||||||
|
|
||||||
|
public partial class App : Application
|
||||||
|
{
|
||||||
|
private IHost? _host;
|
||||||
|
|
||||||
|
protected override async void OnStartup(StartupEventArgs e)
|
||||||
|
{
|
||||||
|
base.OnStartup(e);
|
||||||
|
|
||||||
|
_host = Host.CreateDefaultBuilder()
|
||||||
|
.ConfigureAppConfiguration((_, configuration) =>
|
||||||
|
{
|
||||||
|
configuration.SetBasePath(AppContext.BaseDirectory);
|
||||||
|
configuration.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true);
|
||||||
|
configuration.AddEnvironmentVariables();
|
||||||
|
})
|
||||||
|
.ConfigureLogging((context, logging) =>
|
||||||
|
{
|
||||||
|
logging.AddConfiguration(context.Configuration.GetSection("Logging"));
|
||||||
|
logging.AddFilter("PiiRedaction.Infrastructure.Onnx", LogLevel.Debug);
|
||||||
|
})
|
||||||
|
.ConfigureServices((context, services) =>
|
||||||
|
{
|
||||||
|
services.AddPiiRedactionServices(context.Configuration);
|
||||||
|
services.AddSingleton<ITestPromptCatalog, TestPromptCatalog>();
|
||||||
|
services.AddSingleton<IRedactionAppService, RedactionAppService>();
|
||||||
|
services.AddSingleton<IScriptAnalysisService, ScriptAnalysisService>();
|
||||||
|
services.AddSingleton<IModelStatusService, ModelStatusService>();
|
||||||
|
services.AddSingleton<MainViewModel>();
|
||||||
|
services.AddSingleton<MainWindow>();
|
||||||
|
})
|
||||||
|
.Build();
|
||||||
|
|
||||||
|
Directory.SetCurrentDirectory(AppContext.BaseDirectory);
|
||||||
|
|
||||||
|
var loggerFactory = _host.Services.GetRequiredService<ILoggerFactory>();
|
||||||
|
loggerFactory.AddProvider(new UiLoggerProvider(_host.Services.GetRequiredService<INerLogService>()));
|
||||||
|
|
||||||
|
await _host.StartAsync().ConfigureAwait(true);
|
||||||
|
|
||||||
|
var mainWindow = _host.Services.GetRequiredService<MainWindow>();
|
||||||
|
mainWindow.Show();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async void OnExit(ExitEventArgs e)
|
||||||
|
{
|
||||||
|
if (_host is not null)
|
||||||
|
{
|
||||||
|
await _host.StopAsync().ConfigureAwait(true);
|
||||||
|
_host.Dispose();
|
||||||
|
}
|
||||||
|
|
||||||
|
base.OnExit(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
using System.Globalization;
|
||||||
|
using System.Windows;
|
||||||
|
using System.Windows.Data;
|
||||||
|
using System.Windows.Media;
|
||||||
|
using PiiRedaction.Core.Detection;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.Converters;
|
||||||
|
|
||||||
|
public sealed class ScriptCompositionToBrushConverter : IValueConverter
|
||||||
|
{
|
||||||
|
public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture)
|
||||||
|
{
|
||||||
|
if (value is not ScriptComposition composition)
|
||||||
|
{
|
||||||
|
return Brushes.Gray;
|
||||||
|
}
|
||||||
|
|
||||||
|
return composition switch
|
||||||
|
{
|
||||||
|
ScriptComposition.LatinOnly => new SolidColorBrush(Color.FromRgb(37, 99, 235)),
|
||||||
|
ScriptComposition.TamilOnly => new SolidColorBrush(Color.FromRgb(124, 58, 237)),
|
||||||
|
ScriptComposition.Mixed => new SolidColorBrush(Color.FromRgb(217, 119, 6)),
|
||||||
|
ScriptComposition.NoLetters => new SolidColorBrush(Color.FromRgb(107, 114, 128)),
|
||||||
|
_ => Brushes.Gray
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) =>
|
||||||
|
throw new NotSupportedException();
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class BoolToVisibilityConverter : IValueConverter
|
||||||
|
{
|
||||||
|
public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture) =>
|
||||||
|
value is true ? Visibility.Visible : Visibility.Collapsed;
|
||||||
|
|
||||||
|
public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) =>
|
||||||
|
throw new NotSupportedException();
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class StringNotEmptyToVisibilityConverter : IValueConverter
|
||||||
|
{
|
||||||
|
public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture) =>
|
||||||
|
value is string text && !string.IsNullOrWhiteSpace(text)
|
||||||
|
? Visibility.Visible
|
||||||
|
: Visibility.Collapsed;
|
||||||
|
|
||||||
|
public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) =>
|
||||||
|
throw new NotSupportedException();
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class NerInvokedToBrushConverter : IValueConverter
|
||||||
|
{
|
||||||
|
public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture)
|
||||||
|
{
|
||||||
|
if (value is not string summary)
|
||||||
|
{
|
||||||
|
return new SolidColorBrush(Color.FromRgb(107, 114, 128));
|
||||||
|
}
|
||||||
|
|
||||||
|
return summary switch
|
||||||
|
{
|
||||||
|
"—" or "None" => new SolidColorBrush(Color.FromRgb(107, 114, 128)),
|
||||||
|
"English" => new SolidColorBrush(Color.FromRgb(37, 99, 235)),
|
||||||
|
"Tamil" => new SolidColorBrush(Color.FromRgb(124, 58, 237)),
|
||||||
|
_ when summary.Contains("English", StringComparison.Ordinal)
|
||||||
|
&& summary.Contains("Tamil", StringComparison.Ordinal) =>
|
||||||
|
new SolidColorBrush(Color.FromRgb(217, 119, 6)),
|
||||||
|
_ => new SolidColorBrush(Color.FromRgb(107, 114, 128))
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) =>
|
||||||
|
throw new NotSupportedException();
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class PassFailBrushConverter : IValueConverter
|
||||||
|
{
|
||||||
|
public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture)
|
||||||
|
{
|
||||||
|
if (value is true)
|
||||||
|
{
|
||||||
|
return new SolidColorBrush(Color.FromRgb(22, 163, 74));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (value is false)
|
||||||
|
{
|
||||||
|
return new SolidColorBrush(Color.FromRgb(220, 38, 38));
|
||||||
|
}
|
||||||
|
|
||||||
|
return Brushes.Gray;
|
||||||
|
}
|
||||||
|
|
||||||
|
public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) =>
|
||||||
|
throw new NotSupportedException();
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
using Microsoft.Extensions.Configuration;
|
||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
using Microsoft.Extensions.AI;
|
||||||
|
using PiiRedaction.TestHarness.Wpf.Services;
|
||||||
|
using PiiRedaction.Core.Abstractions;
|
||||||
|
using PiiRedaction.Core.Configuration;
|
||||||
|
using PiiRedaction.Core.Detection;
|
||||||
|
using PiiRedaction.Core.Redaction;
|
||||||
|
using PiiRedaction.Core.Sanitization;
|
||||||
|
using PiiRedaction.Infrastructure.Llm;
|
||||||
|
using PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.DependencyInjection;
|
||||||
|
|
||||||
|
public static class ServiceCollectionExtensions
|
||||||
|
{
|
||||||
|
public static IServiceCollection AddPiiRedactionServices(this IServiceCollection services, IConfiguration configuration)
|
||||||
|
{
|
||||||
|
services.AddSingleton<INerLogService, NerLogService>();
|
||||||
|
services.Configure<PiiRedactionOptions>(configuration.GetSection(PiiRedactionOptions.SectionName));
|
||||||
|
|
||||||
|
services.AddSingleton<DomainRulePiiDetector>();
|
||||||
|
services.AddSingleton<RegexPiiDetector>();
|
||||||
|
services.AddSingleton<OnnxNerPiiDetector>();
|
||||||
|
|
||||||
|
services.AddSingleton<IPiiDetector>(provider => new CompositePiiDetector(
|
||||||
|
[
|
||||||
|
provider.GetRequiredService<DomainRulePiiDetector>(),
|
||||||
|
provider.GetRequiredService<RegexPiiDetector>(),
|
||||||
|
provider.GetRequiredService<OnnxNerPiiDetector>()
|
||||||
|
]));
|
||||||
|
|
||||||
|
services.AddSingleton<IPiiRedactor, PlaceholderPiiRedactor>();
|
||||||
|
services.AddSingleton<IPromptSanitizer, PromptSanitizer>();
|
||||||
|
|
||||||
|
services.AddSingleton<EnglishOnnxNerRunner>();
|
||||||
|
services.AddSingleton<TamilOnnxNerRunner>();
|
||||||
|
services.AddSingleton<IOnnxNerModelRunner, RoutingOnnxNerModelRunner>();
|
||||||
|
services.AddSingleton<IChatClient, MockChatClient>();
|
||||||
|
services.AddSingleton<ILlmPromptService, MockLlmPromptService>();
|
||||||
|
|
||||||
|
return services;
|
||||||
|
}
|
||||||
|
}
|
||||||
43
src/PiiRedaction.TestHarness.Wpf/Logging/UiLogger.cs
Normal file
43
src/PiiRedaction.TestHarness.Wpf/Logging/UiLogger.cs
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using PiiRedaction.TestHarness.Wpf.Services;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.Logging;
|
||||||
|
|
||||||
|
public sealed class UiLogger : ILogger
|
||||||
|
{
|
||||||
|
private readonly string _category;
|
||||||
|
private readonly INerLogService _logService;
|
||||||
|
|
||||||
|
public UiLogger(string category, INerLogService logService)
|
||||||
|
{
|
||||||
|
_category = category;
|
||||||
|
_logService = logService;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
||||||
|
|
||||||
|
public bool IsEnabled(LogLevel logLevel) => logLevel != LogLevel.None;
|
||||||
|
|
||||||
|
public void Log<TState>(
|
||||||
|
LogLevel logLevel,
|
||||||
|
EventId eventId,
|
||||||
|
TState state,
|
||||||
|
Exception? exception,
|
||||||
|
Func<TState, Exception?, string> formatter)
|
||||||
|
{
|
||||||
|
if (!IsEnabled(logLevel))
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var message = formatter(state, exception);
|
||||||
|
if (exception is not null)
|
||||||
|
{
|
||||||
|
message = $"{message} ({exception.Message})";
|
||||||
|
}
|
||||||
|
|
||||||
|
var timestamp = DateTime.Now.ToString("HH:mm:ss.fff");
|
||||||
|
var line = $"[{timestamp}] [{logLevel}] {_category}: {message}";
|
||||||
|
_logService.Append(line);
|
||||||
|
}
|
||||||
|
}
|
||||||
26
src/PiiRedaction.TestHarness.Wpf/Logging/UiLoggerProvider.cs
Normal file
26
src/PiiRedaction.TestHarness.Wpf/Logging/UiLoggerProvider.cs
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using PiiRedaction.TestHarness.Wpf.Services;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.Logging;
|
||||||
|
|
||||||
|
public sealed class UiLoggerProvider : ILoggerProvider
|
||||||
|
{
|
||||||
|
private const string NerCategoryPrefix = "PiiRedaction.Infrastructure.Onnx";
|
||||||
|
|
||||||
|
private readonly INerLogService _logService;
|
||||||
|
|
||||||
|
public UiLoggerProvider(INerLogService logService) => _logService = logService;
|
||||||
|
|
||||||
|
public ILogger CreateLogger(string categoryName) =>
|
||||||
|
IsNerCategory(categoryName)
|
||||||
|
? new UiLogger(categoryName, _logService)
|
||||||
|
: NullLogger.Instance;
|
||||||
|
|
||||||
|
internal static bool IsNerCategory(string categoryName) =>
|
||||||
|
categoryName.StartsWith(NerCategoryPrefix, StringComparison.Ordinal);
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
}
|
||||||
350
src/PiiRedaction.TestHarness.Wpf/MainWindow.xaml
Normal file
350
src/PiiRedaction.TestHarness.Wpf/MainWindow.xaml
Normal file
@@ -0,0 +1,350 @@
|
|||||||
|
<Window x:Class="PiiRedaction.TestHarness.Wpf.MainWindow"
|
||||||
|
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
|
||||||
|
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
|
||||||
|
Title="PII Redaction Test Harness"
|
||||||
|
Height="920"
|
||||||
|
Width="1520"
|
||||||
|
MinHeight="720"
|
||||||
|
MinWidth="1200"
|
||||||
|
Background="{StaticResource AppBackgroundBrush}">
|
||||||
|
<Grid Margin="12">
|
||||||
|
<Grid.RowDefinitions>
|
||||||
|
<RowDefinition Height="Auto" />
|
||||||
|
<RowDefinition Height="*" MinHeight="320" />
|
||||||
|
<RowDefinition Height="Auto" />
|
||||||
|
</Grid.RowDefinitions>
|
||||||
|
|
||||||
|
<!-- Status bar -->
|
||||||
|
<Border Grid.Row="0"
|
||||||
|
Style="{StaticResource PanelBorderStyle}"
|
||||||
|
Margin="0,0,0,8">
|
||||||
|
<Grid>
|
||||||
|
<Grid.ColumnDefinitions>
|
||||||
|
<ColumnDefinition Width="*" />
|
||||||
|
<ColumnDefinition Width="Auto" />
|
||||||
|
</Grid.ColumnDefinitions>
|
||||||
|
|
||||||
|
<StackPanel Orientation="Horizontal" VerticalAlignment="Center">
|
||||||
|
<TextBlock Text="Models:" FontWeight="SemiBold" Margin="0,0,8,0" />
|
||||||
|
<TextBlock Text="{Binding ModelStatus}" Margin="0,0,24,0" />
|
||||||
|
<TextBlock Text="Script:" FontWeight="SemiBold" Margin="0,0,8,0" />
|
||||||
|
<Border Padding="6,2"
|
||||||
|
CornerRadius="4"
|
||||||
|
Background="{Binding ScriptComposition, Converter={StaticResource ScriptCompositionToBrushConverter}}">
|
||||||
|
<TextBlock Text="{Binding ScriptComposition}"
|
||||||
|
Foreground="White"
|
||||||
|
FontWeight="SemiBold" />
|
||||||
|
</Border>
|
||||||
|
<TextBlock Text="NER invoked:" FontWeight="SemiBold" Margin="24,0,8,0" />
|
||||||
|
<Border Padding="6,2"
|
||||||
|
CornerRadius="4"
|
||||||
|
Background="{Binding NerModelsInvokedSummary, Converter={StaticResource NerInvokedToBrushConverter}}">
|
||||||
|
<TextBlock Text="{Binding NerModelsInvokedSummary}"
|
||||||
|
Foreground="White"
|
||||||
|
FontWeight="SemiBold" />
|
||||||
|
</Border>
|
||||||
|
<TextBlock Text="Last run:" FontWeight="SemiBold" Margin="24,0,8,0" />
|
||||||
|
<TextBlock>
|
||||||
|
<Run Text="{Binding ElapsedMilliseconds, Mode=OneWay}" />
|
||||||
|
<Run Text=" ms" />
|
||||||
|
</TextBlock>
|
||||||
|
<TextBlock Text="Entities:" FontWeight="SemiBold" Margin="24,0,8,0" />
|
||||||
|
<TextBlock Text="{Binding EntityCount}" />
|
||||||
|
</StackPanel>
|
||||||
|
|
||||||
|
<TextBlock Grid.Column="1"
|
||||||
|
Text="{Binding StatusMessage}"
|
||||||
|
VerticalAlignment="Center"
|
||||||
|
Foreground="#4B5563" />
|
||||||
|
</Grid>
|
||||||
|
</Border>
|
||||||
|
|
||||||
|
<!-- Main resizable workspace -->
|
||||||
|
<Grid Grid.Row="1">
|
||||||
|
<Grid.ColumnDefinitions>
|
||||||
|
<ColumnDefinition Width="260" MinWidth="180" />
|
||||||
|
<ColumnDefinition Width="Auto" />
|
||||||
|
<ColumnDefinition Width="*" MinWidth="500" />
|
||||||
|
</Grid.ColumnDefinitions>
|
||||||
|
|
||||||
|
<!-- Test prompts -->
|
||||||
|
<Border Grid.Column="0" Style="{StaticResource PanelBorderStyle}">
|
||||||
|
<DockPanel>
|
||||||
|
<TextBlock DockPanel.Dock="Top"
|
||||||
|
Text="Test Prompts"
|
||||||
|
FontSize="16"
|
||||||
|
FontWeight="SemiBold"
|
||||||
|
Margin="0,0,0,8" />
|
||||||
|
<ComboBox DockPanel.Dock="Top"
|
||||||
|
Margin="0,0,0,8"
|
||||||
|
ItemsSource="{Binding TopicCategories}"
|
||||||
|
SelectedItem="{Binding SelectedTopicCategory}"
|
||||||
|
ToolTip="Filter prompts by category" />
|
||||||
|
<TextBox DockPanel.Dock="Top"
|
||||||
|
Margin="0,0,0,8"
|
||||||
|
Text="{Binding PromptFilter, UpdateSourceTrigger=PropertyChanged}"
|
||||||
|
ToolTip="Filter by name, topic, category, language, or description" />
|
||||||
|
<StackPanel DockPanel.Dock="Bottom" Orientation="Horizontal" Margin="0,8,0,0">
|
||||||
|
<Button Content="Run All"
|
||||||
|
Command="{Binding RunAllScenariosCommand}" />
|
||||||
|
</StackPanel>
|
||||||
|
<ListBox ItemsSource="{Binding PromptsView}"
|
||||||
|
SelectedItem="{Binding SelectedPrompt}"
|
||||||
|
DisplayMemberPath="DisplayLabel">
|
||||||
|
<ListBox.GroupStyle>
|
||||||
|
<GroupStyle>
|
||||||
|
<GroupStyle.HeaderTemplate>
|
||||||
|
<DataTemplate>
|
||||||
|
<TextBlock Text="{Binding Name}"
|
||||||
|
FontWeight="Bold"
|
||||||
|
Margin="0,8,0,4" />
|
||||||
|
</DataTemplate>
|
||||||
|
</GroupStyle.HeaderTemplate>
|
||||||
|
</GroupStyle>
|
||||||
|
</ListBox.GroupStyle>
|
||||||
|
<ListBox.ItemContainerStyle>
|
||||||
|
<Style TargetType="ListBoxItem">
|
||||||
|
<Setter Property="ToolTip" Value="{Binding Description}" />
|
||||||
|
</Style>
|
||||||
|
</ListBox.ItemContainerStyle>
|
||||||
|
</ListBox>
|
||||||
|
</DockPanel>
|
||||||
|
</Border>
|
||||||
|
|
||||||
|
<GridSplitter Grid.Column="1"
|
||||||
|
Style="{StaticResource GridSplitterStyle}"
|
||||||
|
Width="6"
|
||||||
|
HorizontalAlignment="Center"
|
||||||
|
VerticalAlignment="Stretch" />
|
||||||
|
|
||||||
|
<!-- Input + detection + sanitized (nested splitters) -->
|
||||||
|
<Grid Grid.Column="2" Margin="8,0,0,0">
|
||||||
|
<Grid.RowDefinitions>
|
||||||
|
<RowDefinition Height="*" MinHeight="220" />
|
||||||
|
<RowDefinition Height="Auto" />
|
||||||
|
<RowDefinition Height="200" MinHeight="120" />
|
||||||
|
</Grid.RowDefinitions>
|
||||||
|
|
||||||
|
<!-- Input prompt | Detection details -->
|
||||||
|
<Grid Grid.Row="0">
|
||||||
|
<Grid.ColumnDefinitions>
|
||||||
|
<ColumnDefinition Width="*" MinWidth="280" />
|
||||||
|
<ColumnDefinition Width="Auto" />
|
||||||
|
<ColumnDefinition Width="1.15*" MinWidth="420" />
|
||||||
|
</Grid.ColumnDefinitions>
|
||||||
|
|
||||||
|
<Border Grid.Column="0" Style="{StaticResource PanelBorderStyle}">
|
||||||
|
<DockPanel>
|
||||||
|
<TextBlock DockPanel.Dock="Top"
|
||||||
|
Text="Input Prompt"
|
||||||
|
FontSize="16"
|
||||||
|
FontWeight="SemiBold"
|
||||||
|
Margin="0,0,0,8" />
|
||||||
|
<StackPanel DockPanel.Dock="Bottom" Orientation="Horizontal" Margin="0,8,0,0">
|
||||||
|
<Button Content="Redact"
|
||||||
|
Command="{Binding RedactCommand}" />
|
||||||
|
<Button Content="Clear"
|
||||||
|
Style="{StaticResource SecondaryButtonStyle}"
|
||||||
|
Command="{Binding ClearCommand}" />
|
||||||
|
</StackPanel>
|
||||||
|
<TextBox Text="{Binding InputPrompt, UpdateSourceTrigger=PropertyChanged}"
|
||||||
|
AcceptsReturn="True"
|
||||||
|
TextWrapping="Wrap"
|
||||||
|
VerticalScrollBarVisibility="Auto"
|
||||||
|
FontSize="14" />
|
||||||
|
</DockPanel>
|
||||||
|
</Border>
|
||||||
|
|
||||||
|
<GridSplitter Grid.Column="1"
|
||||||
|
Style="{StaticResource GridSplitterStyle}"
|
||||||
|
Width="6"
|
||||||
|
HorizontalAlignment="Center"
|
||||||
|
VerticalAlignment="Stretch" />
|
||||||
|
|
||||||
|
<!-- Detection details: entities + placeholders stacked with splitter -->
|
||||||
|
<Border Grid.Column="2" Style="{StaticResource PanelBorderStyle}">
|
||||||
|
<Grid>
|
||||||
|
<Grid.RowDefinitions>
|
||||||
|
<RowDefinition Height="*" MinHeight="100" />
|
||||||
|
<RowDefinition Height="Auto" />
|
||||||
|
<RowDefinition Height="*" MinHeight="80" />
|
||||||
|
<RowDefinition Height="Auto" />
|
||||||
|
</Grid.RowDefinitions>
|
||||||
|
|
||||||
|
<DockPanel Grid.Row="0">
|
||||||
|
<TextBlock DockPanel.Dock="Top"
|
||||||
|
Text="Detected Entities"
|
||||||
|
FontSize="15"
|
||||||
|
FontWeight="SemiBold"
|
||||||
|
Margin="0,0,0,6" />
|
||||||
|
<DataGrid ItemsSource="{Binding DetectedEntities}">
|
||||||
|
<DataGrid.Columns>
|
||||||
|
<DataGridTextColumn Header="Type" Binding="{Binding Type}" Width="100" />
|
||||||
|
<DataGridTextColumn Header="Value" Binding="{Binding Value}" Width="2*" MinWidth="120" />
|
||||||
|
<DataGridTextColumn Header="Source" Binding="{Binding Source}" Width="80" />
|
||||||
|
<DataGridTextColumn Header="NER Model" Binding="{Binding NerModel}" Width="90" />
|
||||||
|
<DataGridTextColumn Header="Start" Binding="{Binding StartIndex}" Width="60" />
|
||||||
|
<DataGridTextColumn Header="Length" Binding="{Binding Length}" Width="65" />
|
||||||
|
<DataGridTextColumn Header="Confidence" Binding="{Binding Confidence}" Width="80" />
|
||||||
|
</DataGrid.Columns>
|
||||||
|
</DataGrid>
|
||||||
|
</DockPanel>
|
||||||
|
|
||||||
|
<GridSplitter Grid.Row="1"
|
||||||
|
Style="{StaticResource GridSplitterStyle}"
|
||||||
|
Height="6"
|
||||||
|
HorizontalAlignment="Stretch"
|
||||||
|
VerticalAlignment="Center" />
|
||||||
|
|
||||||
|
<DockPanel Grid.Row="2">
|
||||||
|
<TextBlock DockPanel.Dock="Top"
|
||||||
|
Text="Placeholder Map"
|
||||||
|
FontSize="15"
|
||||||
|
FontWeight="SemiBold"
|
||||||
|
Margin="0,6,0,6" />
|
||||||
|
<DataGrid ItemsSource="{Binding PlaceholderMap}">
|
||||||
|
<DataGrid.Columns>
|
||||||
|
<DataGridTextColumn Header="Placeholder" Binding="{Binding Placeholder}" Width="140" />
|
||||||
|
<DataGridTextColumn Header="Original Value" Binding="{Binding OriginalValue}" Width="*" MinWidth="160" />
|
||||||
|
</DataGrid.Columns>
|
||||||
|
</DataGrid>
|
||||||
|
</DockPanel>
|
||||||
|
|
||||||
|
<Border Grid.Row="3"
|
||||||
|
Margin="0,8,0,0"
|
||||||
|
Padding="8"
|
||||||
|
Background="#FEF2F2"
|
||||||
|
BorderBrush="{StaticResource WarningBrush}"
|
||||||
|
BorderThickness="1"
|
||||||
|
Visibility="{Binding LeakWarning, Converter={StaticResource BoolToVisibilityConverter}}">
|
||||||
|
<TextBlock Text="Leak warning: a detected PII value still appears in the sanitized output."
|
||||||
|
Foreground="{StaticResource WarningBrush}"
|
||||||
|
TextWrapping="Wrap" />
|
||||||
|
</Border>
|
||||||
|
</Grid>
|
||||||
|
</Border>
|
||||||
|
</Grid>
|
||||||
|
|
||||||
|
<GridSplitter Grid.Row="1"
|
||||||
|
Style="{StaticResource GridSplitterStyle}"
|
||||||
|
Height="6"
|
||||||
|
HorizontalAlignment="Stretch"
|
||||||
|
VerticalAlignment="Center" />
|
||||||
|
|
||||||
|
<!-- Sanitized output -->
|
||||||
|
<Border Grid.Row="2" Style="{StaticResource PanelBorderStyle}">
|
||||||
|
<DockPanel>
|
||||||
|
<StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Margin="0,0,0,8">
|
||||||
|
<TextBlock Text="Sanitized Output"
|
||||||
|
FontSize="16"
|
||||||
|
FontWeight="SemiBold"
|
||||||
|
VerticalAlignment="Center" />
|
||||||
|
<Button Content="Copy"
|
||||||
|
Margin="16,0,0,0"
|
||||||
|
Style="{StaticResource SecondaryButtonStyle}"
|
||||||
|
Command="{Binding CopySanitizedCommand}" />
|
||||||
|
<Button Content="Send Mock LLM"
|
||||||
|
Command="{Binding SendToMockLlmCommand}" />
|
||||||
|
</StackPanel>
|
||||||
|
<Grid>
|
||||||
|
<Grid.RowDefinitions>
|
||||||
|
<RowDefinition Height="*" />
|
||||||
|
<RowDefinition Height="Auto" />
|
||||||
|
</Grid.RowDefinitions>
|
||||||
|
<TextBox Grid.Row="0"
|
||||||
|
Text="{Binding SanitizedOutput, Mode=OneWay}"
|
||||||
|
IsReadOnly="True"
|
||||||
|
AcceptsReturn="True"
|
||||||
|
TextWrapping="Wrap"
|
||||||
|
VerticalScrollBarVisibility="Auto"
|
||||||
|
FontSize="14" />
|
||||||
|
<TextBox Grid.Row="1"
|
||||||
|
Margin="0,8,0,0"
|
||||||
|
Text="{Binding MockLlmResponse, Mode=OneWay}"
|
||||||
|
IsReadOnly="True"
|
||||||
|
AcceptsReturn="True"
|
||||||
|
TextWrapping="Wrap"
|
||||||
|
VerticalScrollBarVisibility="Auto"
|
||||||
|
MinHeight="60"
|
||||||
|
FontSize="13"
|
||||||
|
Visibility="{Binding MockLlmResponse, Converter={StaticResource StringNotEmptyToVisibilityConverter}}" />
|
||||||
|
</Grid>
|
||||||
|
</DockPanel>
|
||||||
|
</Border>
|
||||||
|
</Grid>
|
||||||
|
</Grid>
|
||||||
|
|
||||||
|
<!-- Batch results + NER logs -->
|
||||||
|
<StackPanel Grid.Row="2" Margin="0,8,0,0">
|
||||||
|
<Expander Header="NER Logs"
|
||||||
|
IsExpanded="True"
|
||||||
|
Margin="0,0,0,8"
|
||||||
|
Background="{StaticResource PanelBrush}"
|
||||||
|
BorderBrush="{StaticResource BorderBrushColor}"
|
||||||
|
BorderThickness="1"
|
||||||
|
Padding="8">
|
||||||
|
<DockPanel MinHeight="100" MaxHeight="220">
|
||||||
|
<StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Margin="0,0,0,8">
|
||||||
|
<Button Content="Clear Logs"
|
||||||
|
Style="{StaticResource SecondaryButtonStyle}"
|
||||||
|
Command="{Binding ClearNerLogsCommand}" />
|
||||||
|
</StackPanel>
|
||||||
|
<ListBox x:Name="NerLogListBox"
|
||||||
|
ItemsSource="{Binding NerLogLines}"
|
||||||
|
FontFamily="Consolas"
|
||||||
|
FontSize="12"
|
||||||
|
ScrollViewer.HorizontalScrollBarVisibility="Auto"
|
||||||
|
ScrollViewer.VerticalScrollBarVisibility="Auto" />
|
||||||
|
</DockPanel>
|
||||||
|
</Expander>
|
||||||
|
|
||||||
|
<Expander Header="Batch Results"
|
||||||
|
IsExpanded="{Binding IsBatchExpanded}"
|
||||||
|
Margin="0,8,0,0"
|
||||||
|
Background="{StaticResource PanelBrush}"
|
||||||
|
BorderBrush="{StaticResource BorderBrushColor}"
|
||||||
|
BorderThickness="1"
|
||||||
|
Padding="8">
|
||||||
|
<DockPanel MinHeight="120" MaxHeight="320">
|
||||||
|
<TextBlock DockPanel.Dock="Top"
|
||||||
|
Text="{Binding BatchSummary}"
|
||||||
|
FontWeight="SemiBold"
|
||||||
|
Margin="0,0,0,8" />
|
||||||
|
<DataGrid ItemsSource="{Binding BatchResults}"
|
||||||
|
VerticalScrollBarVisibility="Auto"
|
||||||
|
HorizontalScrollBarVisibility="Auto">
|
||||||
|
<DataGrid.Columns>
|
||||||
|
<DataGridTextColumn Header="Scenario" Binding="{Binding Scenario.Name}" Width="180" />
|
||||||
|
<DataGridTextColumn Header="Language" Binding="{Binding Scenario.Language}" Width="80" />
|
||||||
|
<DataGridTextColumn Header="Category" Binding="{Binding Scenario.Category}" Width="140" />
|
||||||
|
<DataGridTextColumn Header="Entities" Binding="{Binding EntityCount}" Width="70" />
|
||||||
|
<DataGridTextColumn Header="ms" Binding="{Binding ElapsedMilliseconds}" Width="60" />
|
||||||
|
<DataGridTemplateColumn Header="Result" Width="70">
|
||||||
|
<DataGridTemplateColumn.CellTemplate>
|
||||||
|
<DataTemplate>
|
||||||
|
<TextBlock FontWeight="SemiBold"
|
||||||
|
Foreground="{Binding Passed, Converter={StaticResource PassFailBrushConverter}}">
|
||||||
|
<TextBlock.Style>
|
||||||
|
<Style TargetType="TextBlock">
|
||||||
|
<Style.Triggers>
|
||||||
|
<DataTrigger Binding="{Binding Passed}" Value="True">
|
||||||
|
<Setter Property="Text" Value="PASS" />
|
||||||
|
</DataTrigger>
|
||||||
|
<DataTrigger Binding="{Binding Passed}" Value="False">
|
||||||
|
<Setter Property="Text" Value="FAIL" />
|
||||||
|
</DataTrigger>
|
||||||
|
</Style.Triggers>
|
||||||
|
</Style>
|
||||||
|
</TextBlock.Style>
|
||||||
|
</TextBlock>
|
||||||
|
</DataTemplate>
|
||||||
|
</DataGridTemplateColumn.CellTemplate>
|
||||||
|
</DataGridTemplateColumn>
|
||||||
|
<DataGridTextColumn Header="Reason" Binding="{Binding FailureReason}" Width="*" />
|
||||||
|
</DataGrid.Columns>
|
||||||
|
</DataGrid>
|
||||||
|
</DockPanel>
|
||||||
|
</Expander>
|
||||||
|
</StackPanel>
|
||||||
|
</Grid>
|
||||||
|
</Window>
|
||||||
27
src/PiiRedaction.TestHarness.Wpf/MainWindow.xaml.cs
Normal file
27
src/PiiRedaction.TestHarness.Wpf/MainWindow.xaml.cs
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
using System.Collections.Specialized;
|
||||||
|
using System.Windows;
|
||||||
|
using PiiRedaction.TestHarness.Wpf.Services;
|
||||||
|
using PiiRedaction.TestHarness.Wpf.ViewModels;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf;
|
||||||
|
|
||||||
|
public partial class MainWindow : Window
|
||||||
|
{
|
||||||
|
public MainWindow(MainViewModel viewModel, INerLogService nerLogService)
|
||||||
|
{
|
||||||
|
InitializeComponent();
|
||||||
|
DataContext = viewModel;
|
||||||
|
|
||||||
|
nerLogService.Lines.CollectionChanged += OnNerLogLinesChanged;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void OnNerLogLinesChanged(object? sender, NotifyCollectionChangedEventArgs e)
|
||||||
|
{
|
||||||
|
if (NerLogListBox.Items.Count == 0)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
NerLogListBox.ScrollIntoView(NerLogListBox.Items[NerLogListBox.Items.Count - 1]);
|
||||||
|
}
|
||||||
|
}
|
||||||
27
src/PiiRedaction.TestHarness.Wpf/Models/BatchRunResult.cs
Normal file
27
src/PiiRedaction.TestHarness.Wpf/Models/BatchRunResult.cs
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
using PiiRedaction.Core.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.Models;
|
||||||
|
|
||||||
|
public sealed record RedactionOutcome(
|
||||||
|
string OriginalPrompt,
|
||||||
|
string SanitizedPrompt,
|
||||||
|
IReadOnlyList<RedactionDisplayModel> DetectedEntities,
|
||||||
|
IReadOnlyList<PlaceholderDisplayModel> Placeholders,
|
||||||
|
long ElapsedMilliseconds,
|
||||||
|
bool HasLeak,
|
||||||
|
IReadOnlyList<NerModelOrigin> NerModelsInvoked,
|
||||||
|
string NerModelsInvokedSummary);
|
||||||
|
|
||||||
|
public sealed record BatchScenarioResult(
|
||||||
|
TestPromptScenario Scenario,
|
||||||
|
bool Passed,
|
||||||
|
string? FailureReason,
|
||||||
|
int EntityCount,
|
||||||
|
long ElapsedMilliseconds);
|
||||||
|
|
||||||
|
public sealed record BatchRunSummary(
|
||||||
|
int Total,
|
||||||
|
int Passed,
|
||||||
|
int Failed,
|
||||||
|
IReadOnlyList<BatchScenarioResult> Results,
|
||||||
|
long TotalElapsedMilliseconds);
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
namespace PiiRedaction.TestHarness.Wpf.Models;
|
||||||
|
|
||||||
|
public enum ModelAvailability
|
||||||
|
{
|
||||||
|
Ready,
|
||||||
|
Missing,
|
||||||
|
Disabled
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed record NerModelStatus(
|
||||||
|
string ModelName,
|
||||||
|
ModelAvailability Availability,
|
||||||
|
string Path);
|
||||||
|
|
||||||
|
public sealed record ModelStatusSnapshot(
|
||||||
|
NerModelStatus English,
|
||||||
|
NerModelStatus Tamil)
|
||||||
|
{
|
||||||
|
public string Summary =>
|
||||||
|
$"English NER: {English.Availability} | Tamil NER: {Tamil.Availability}";
|
||||||
|
}
|
||||||
20
src/PiiRedaction.TestHarness.Wpf/Models/NerRoutingDisplay.cs
Normal file
20
src/PiiRedaction.TestHarness.Wpf/Models/NerRoutingDisplay.cs
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
using PiiRedaction.Core.Detection;
|
||||||
|
using PiiRedaction.Core.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.Models;
|
||||||
|
|
||||||
|
public static class NerRoutingDisplay
|
||||||
|
{
|
||||||
|
public static string FormatInvokedModels(IReadOnlyList<NerModelOrigin> invoked) =>
|
||||||
|
invoked.Count switch
|
||||||
|
{
|
||||||
|
0 => "None",
|
||||||
|
1 => invoked[0].ToString(),
|
||||||
|
_ => string.Join(" + ", invoked)
|
||||||
|
};
|
||||||
|
|
||||||
|
public static string FormatEntityModelOrigin(PiiEntity entity) =>
|
||||||
|
entity.Source == PiiDetectionSource.Ner && entity.ModelOrigin.HasValue
|
||||||
|
? entity.ModelOrigin.Value.ToString()
|
||||||
|
: "—";
|
||||||
|
}
|
||||||
35
src/PiiRedaction.TestHarness.Wpf/Models/PromptTopics.cs
Normal file
35
src/PiiRedaction.TestHarness.Wpf/Models/PromptTopics.cs
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
namespace PiiRedaction.TestHarness.Wpf.Models;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Top-level prompt groupings for the WPF harness category filter.
|
||||||
|
/// </summary>
|
||||||
|
public static class PromptTopics
|
||||||
|
{
|
||||||
|
public const string All = "All";
|
||||||
|
|
||||||
|
public const string CareerGuidance = "Career Guidance";
|
||||||
|
|
||||||
|
public const string BankingFinancial = "Banking & Financial";
|
||||||
|
|
||||||
|
public const string Negative = "Negative";
|
||||||
|
|
||||||
|
public const string EdgeHarness = "Edge & Harness";
|
||||||
|
|
||||||
|
public static IReadOnlyList<string> FilterOptions { get; } =
|
||||||
|
[
|
||||||
|
All,
|
||||||
|
CareerGuidance,
|
||||||
|
BankingFinancial,
|
||||||
|
Negative,
|
||||||
|
EdgeHarness
|
||||||
|
];
|
||||||
|
|
||||||
|
public static int SortOrder(string topic) => topic switch
|
||||||
|
{
|
||||||
|
CareerGuidance => 0,
|
||||||
|
BankingFinancial => 1,
|
||||||
|
EdgeHarness => 2,
|
||||||
|
Negative => 3,
|
||||||
|
_ => 99
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
using PiiRedaction.Core.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.Models;
|
||||||
|
|
||||||
|
public sealed class RedactionDisplayModel
|
||||||
|
{
|
||||||
|
public required string Type { get; init; }
|
||||||
|
public required string Value { get; init; }
|
||||||
|
public required string Source { get; init; }
|
||||||
|
public required string NerModel { get; init; }
|
||||||
|
public int StartIndex { get; init; }
|
||||||
|
public int Length { get; init; }
|
||||||
|
public string? Confidence { get; init; }
|
||||||
|
|
||||||
|
public static RedactionDisplayModel FromEntity(PiiEntity entity) => new()
|
||||||
|
{
|
||||||
|
Type = entity.Type.ToString(),
|
||||||
|
Value = entity.Value,
|
||||||
|
Source = entity.Source.ToString(),
|
||||||
|
NerModel = NerRoutingDisplay.FormatEntityModelOrigin(entity),
|
||||||
|
StartIndex = entity.StartIndex,
|
||||||
|
Length = entity.Length,
|
||||||
|
Confidence = entity.Confidence?.ToString("F2")
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class PlaceholderDisplayModel
|
||||||
|
{
|
||||||
|
public required string Placeholder { get; init; }
|
||||||
|
public required string OriginalValue { get; init; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
namespace PiiRedaction.TestHarness.Wpf.Models;
|
||||||
|
|
||||||
|
public enum PromptLanguage
|
||||||
|
{
|
||||||
|
English,
|
||||||
|
Tamil,
|
||||||
|
Mixed,
|
||||||
|
Tanglish
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed record TestPromptScenario(
|
||||||
|
string Id,
|
||||||
|
string Name,
|
||||||
|
string Topic,
|
||||||
|
PromptLanguage Language,
|
||||||
|
string Category,
|
||||||
|
string Description,
|
||||||
|
string Prompt,
|
||||||
|
bool ExpectDetections);
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>WinExe</OutputType>
|
||||||
|
<TargetFramework>net10.0-windows</TargetFramework>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<UseWPF>true</UseWPF>
|
||||||
|
<ApplicationIcon />
|
||||||
|
<RootNamespace>PiiRedaction.TestHarness.Wpf</RootNamespace>
|
||||||
|
<AssemblyName>PiiRedaction.TestHarness.Wpf</AssemblyName>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<ProjectReference Include="..\PiiRedaction.Core\PiiRedaction.Core.csproj" />
|
||||||
|
<ProjectReference Include="..\PiiRedaction.Infrastructure\PiiRedaction.Infrastructure.csproj" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="CommunityToolkit.Mvvm" Version="8.4.0" />
|
||||||
|
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="10.0.9" />
|
||||||
|
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="10.0.9" />
|
||||||
|
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="10.0.9" />
|
||||||
|
<PackageReference Include="Microsoft.Extensions.Hosting" Version="10.0.9" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<None Update="appsettings.json">
|
||||||
|
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||||
|
</None>
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
68
src/PiiRedaction.TestHarness.Wpf/Resources/Styles.xaml
Normal file
68
src/PiiRedaction.TestHarness.Wpf/Resources/Styles.xaml
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
<ResourceDictionary xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
|
||||||
|
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml">
|
||||||
|
<SolidColorBrush x:Key="AppBackgroundBrush" Color="#F3F4F6" />
|
||||||
|
<SolidColorBrush x:Key="PanelBrush" Color="White" />
|
||||||
|
<SolidColorBrush x:Key="BorderBrushColor" Color="#D1D5DB" />
|
||||||
|
<SolidColorBrush x:Key="AccentBrush" Color="#2563EB" />
|
||||||
|
<SolidColorBrush x:Key="WarningBrush" Color="#DC2626" />
|
||||||
|
|
||||||
|
<Style TargetType="TextBlock">
|
||||||
|
<Setter Property="FontFamily" Value="Segoe UI" />
|
||||||
|
<Setter Property="Foreground" Value="#111827" />
|
||||||
|
</Style>
|
||||||
|
|
||||||
|
<Style TargetType="TextBox">
|
||||||
|
<Setter Property="FontFamily" Value="Segoe UI" />
|
||||||
|
<Setter Property="Padding" Value="8" />
|
||||||
|
<Setter Property="BorderBrush" Value="{StaticResource BorderBrushColor}" />
|
||||||
|
<Setter Property="BorderThickness" Value="1" />
|
||||||
|
</Style>
|
||||||
|
|
||||||
|
<Style TargetType="Button">
|
||||||
|
<Setter Property="FontFamily" Value="Segoe UI" />
|
||||||
|
<Setter Property="Padding" Value="12,6" />
|
||||||
|
<Setter Property="Margin" Value="0,0,8,0" />
|
||||||
|
<Setter Property="Background" Value="{StaticResource AccentBrush}" />
|
||||||
|
<Setter Property="Foreground" Value="White" />
|
||||||
|
<Setter Property="BorderThickness" Value="0" />
|
||||||
|
<Setter Property="Cursor" Value="Hand" />
|
||||||
|
</Style>
|
||||||
|
|
||||||
|
<Style x:Key="SecondaryButtonStyle" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
|
||||||
|
<Setter Property="Background" Value="#E5E7EB" />
|
||||||
|
<Setter Property="Foreground" Value="#111827" />
|
||||||
|
</Style>
|
||||||
|
|
||||||
|
<Style TargetType="GroupBox">
|
||||||
|
<Setter Property="Margin" Value="0,0,0,8" />
|
||||||
|
<Setter Property="Padding" Value="8" />
|
||||||
|
<Setter Property="BorderBrush" Value="{StaticResource BorderBrushColor}" />
|
||||||
|
</Style>
|
||||||
|
|
||||||
|
<Style TargetType="DataGrid">
|
||||||
|
<Setter Property="FontFamily" Value="Segoe UI" />
|
||||||
|
<Setter Property="AutoGenerateColumns" Value="False" />
|
||||||
|
<Setter Property="IsReadOnly" Value="True" />
|
||||||
|
<Setter Property="HeadersVisibility" Value="Column" />
|
||||||
|
<Setter Property="GridLinesVisibility" Value="Horizontal" />
|
||||||
|
<Setter Property="BorderBrush" Value="{StaticResource BorderBrushColor}" />
|
||||||
|
<Setter Property="BorderThickness" Value="1" />
|
||||||
|
<Setter Property="CanUserResizeColumns" Value="True" />
|
||||||
|
<Setter Property="CanUserReorderColumns" Value="True" />
|
||||||
|
<Setter Property="HorizontalScrollBarVisibility" Value="Auto" />
|
||||||
|
<Setter Property="RowHeaderWidth" Value="0" />
|
||||||
|
</Style>
|
||||||
|
|
||||||
|
<Style x:Key="GridSplitterStyle" TargetType="GridSplitter">
|
||||||
|
<Setter Property="Background" Value="#E5E7EB" />
|
||||||
|
<Setter Property="ShowsPreview" Value="True" />
|
||||||
|
<Setter Property="ResizeBehavior" Value="PreviousAndNext" />
|
||||||
|
</Style>
|
||||||
|
|
||||||
|
<Style x:Key="PanelBorderStyle" TargetType="Border">
|
||||||
|
<Setter Property="Background" Value="{StaticResource PanelBrush}" />
|
||||||
|
<Setter Property="BorderBrush" Value="{StaticResource BorderBrushColor}" />
|
||||||
|
<Setter Property="BorderThickness" Value="1" />
|
||||||
|
<Setter Property="Padding" Value="8" />
|
||||||
|
</Style>
|
||||||
|
</ResourceDictionary>
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
using PiiRedaction.TestHarness.Wpf.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.Services;
|
||||||
|
|
||||||
|
public interface IModelStatusService
|
||||||
|
{
|
||||||
|
ModelStatusSnapshot GetStatus();
|
||||||
|
}
|
||||||
12
src/PiiRedaction.TestHarness.Wpf/Services/INerLogService.cs
Normal file
12
src/PiiRedaction.TestHarness.Wpf/Services/INerLogService.cs
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
using System.Collections.ObjectModel;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.Services;
|
||||||
|
|
||||||
|
public interface INerLogService
|
||||||
|
{
|
||||||
|
ObservableCollection<string> Lines { get; }
|
||||||
|
|
||||||
|
void Append(string line);
|
||||||
|
|
||||||
|
void Clear();
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
using PiiRedaction.TestHarness.Wpf.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.Services;
|
||||||
|
|
||||||
|
public interface IRedactionAppService
|
||||||
|
{
|
||||||
|
Task<RedactionOutcome> RedactAsync(string prompt, CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
|
Task<string> SendToMockLlmAsync(string sanitizedPrompt, CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
|
BatchScenarioResult EvaluateScenario(TestPromptScenario scenario, RedactionOutcome outcome);
|
||||||
|
|
||||||
|
Task<BatchRunSummary> RunAllScenariosAsync(
|
||||||
|
IReadOnlyList<TestPromptScenario> scenarios,
|
||||||
|
IProgress<(int Current, int Total, string Name)>? progress = null,
|
||||||
|
CancellationToken cancellationToken = default);
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
using PiiRedaction.Core.Detection;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.Services;
|
||||||
|
|
||||||
|
public interface IScriptAnalysisService
|
||||||
|
{
|
||||||
|
ScriptComposition GetComposition(string text);
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
using PiiRedaction.TestHarness.Wpf.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.Services;
|
||||||
|
|
||||||
|
public interface ITestPromptCatalog
|
||||||
|
{
|
||||||
|
IReadOnlyList<TestPromptScenario> All { get; }
|
||||||
|
|
||||||
|
IReadOnlyList<string> Topics { get; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
using PiiRedaction.Core.Configuration;
|
||||||
|
using PiiRedaction.Infrastructure.Onnx;
|
||||||
|
using PiiRedaction.TestHarness.Wpf.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.Services;
|
||||||
|
|
||||||
|
public sealed class ModelStatusService : IModelStatusService
|
||||||
|
{
|
||||||
|
private readonly EnglishOnnxNerRunner _englishRunner;
|
||||||
|
private readonly TamilOnnxNerRunner _tamilRunner;
|
||||||
|
private readonly PiiRedactionOptions _options;
|
||||||
|
|
||||||
|
public ModelStatusService(
|
||||||
|
EnglishOnnxNerRunner englishRunner,
|
||||||
|
TamilOnnxNerRunner tamilRunner,
|
||||||
|
IOptions<PiiRedactionOptions> options)
|
||||||
|
{
|
||||||
|
_englishRunner = englishRunner;
|
||||||
|
_tamilRunner = tamilRunner;
|
||||||
|
_options = options.Value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ModelStatusSnapshot GetStatus()
|
||||||
|
{
|
||||||
|
var englishPath = OnnxAssetPathResolver.ResolveModelPath(
|
||||||
|
_options.EnglishOnnxModelPath,
|
||||||
|
_options.OnnxModelPath);
|
||||||
|
|
||||||
|
var tamilPath = OnnxAssetPathResolver.ResolveModelPath(_options.TamilOnnxModelPath);
|
||||||
|
|
||||||
|
var englishAvailability = _englishRunner.IsModelAvailable
|
||||||
|
? ModelAvailability.Ready
|
||||||
|
: ModelAvailability.Missing;
|
||||||
|
|
||||||
|
var tamilAvailability = !_options.EnableTamilNer
|
||||||
|
? ModelAvailability.Disabled
|
||||||
|
: _tamilRunner.IsModelAvailable
|
||||||
|
? ModelAvailability.Ready
|
||||||
|
: ModelAvailability.Missing;
|
||||||
|
|
||||||
|
return new ModelStatusSnapshot(
|
||||||
|
new NerModelStatus("English", englishAvailability, englishPath),
|
||||||
|
new NerModelStatus("Tamil", tamilAvailability, tamilPath));
|
||||||
|
}
|
||||||
|
}
|
||||||
31
src/PiiRedaction.TestHarness.Wpf/Services/NerLogService.cs
Normal file
31
src/PiiRedaction.TestHarness.Wpf/Services/NerLogService.cs
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
using System.Collections.ObjectModel;
|
||||||
|
using System.Windows;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.Services;
|
||||||
|
|
||||||
|
public sealed class NerLogService : INerLogService
|
||||||
|
{
|
||||||
|
public ObservableCollection<string> Lines { get; } = [];
|
||||||
|
|
||||||
|
public void Append(string line)
|
||||||
|
{
|
||||||
|
if (Application.Current?.Dispatcher.CheckAccess() == true)
|
||||||
|
{
|
||||||
|
Lines.Add(line);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Application.Current?.Dispatcher.Invoke(() => Lines.Add(line));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Clear()
|
||||||
|
{
|
||||||
|
if (Application.Current?.Dispatcher.CheckAccess() == true)
|
||||||
|
{
|
||||||
|
Lines.Clear();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Application.Current?.Dispatcher.Invoke(() => Lines.Clear());
|
||||||
|
}
|
||||||
|
}
|
||||||
118
src/PiiRedaction.TestHarness.Wpf/Services/RedactionAppService.cs
Normal file
118
src/PiiRedaction.TestHarness.Wpf/Services/RedactionAppService.cs
Normal file
@@ -0,0 +1,118 @@
|
|||||||
|
using System.Diagnostics;
|
||||||
|
using PiiRedaction.Core.Abstractions;
|
||||||
|
using PiiRedaction.Core.Models;
|
||||||
|
using PiiRedaction.TestHarness.Wpf.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.Services;
|
||||||
|
|
||||||
|
public sealed class RedactionAppService : IRedactionAppService
|
||||||
|
{
|
||||||
|
private readonly IPromptSanitizer _sanitizer;
|
||||||
|
private readonly ILlmPromptService _llmPromptService;
|
||||||
|
|
||||||
|
public RedactionAppService(IPromptSanitizer sanitizer, ILlmPromptService llmPromptService)
|
||||||
|
{
|
||||||
|
_sanitizer = sanitizer;
|
||||||
|
_llmPromptService = llmPromptService;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task<RedactionOutcome> RedactAsync(string prompt, CancellationToken cancellationToken = default) =>
|
||||||
|
Task.Run(() =>
|
||||||
|
{
|
||||||
|
cancellationToken.ThrowIfCancellationRequested();
|
||||||
|
|
||||||
|
var stopwatch = Stopwatch.StartNew();
|
||||||
|
var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
|
||||||
|
stopwatch.Stop();
|
||||||
|
|
||||||
|
return MapOutcome(result, stopwatch.ElapsedMilliseconds);
|
||||||
|
}, cancellationToken);
|
||||||
|
|
||||||
|
public Task<string> SendToMockLlmAsync(string sanitizedPrompt, CancellationToken cancellationToken = default) =>
|
||||||
|
_llmPromptService.SendPromptAsync(sanitizedPrompt, cancellationToken);
|
||||||
|
|
||||||
|
public BatchScenarioResult EvaluateScenario(TestPromptScenario scenario, RedactionOutcome outcome)
|
||||||
|
{
|
||||||
|
var entityCount = outcome.DetectedEntities.Count;
|
||||||
|
string? failureReason = null;
|
||||||
|
|
||||||
|
if (scenario.ExpectDetections && entityCount == 0)
|
||||||
|
{
|
||||||
|
failureReason = "Expected at least one PII detection but found none.";
|
||||||
|
}
|
||||||
|
else if (!scenario.ExpectDetections && entityCount > 0)
|
||||||
|
{
|
||||||
|
failureReason = $"Expected no detections but found {entityCount}.";
|
||||||
|
}
|
||||||
|
else if (outcome.HasLeak)
|
||||||
|
{
|
||||||
|
failureReason = "Detected PII value still present in sanitized output.";
|
||||||
|
}
|
||||||
|
|
||||||
|
return new BatchScenarioResult(
|
||||||
|
scenario,
|
||||||
|
failureReason is null,
|
||||||
|
failureReason,
|
||||||
|
entityCount,
|
||||||
|
outcome.ElapsedMilliseconds);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<BatchRunSummary> RunAllScenariosAsync(
|
||||||
|
IReadOnlyList<TestPromptScenario> scenarios,
|
||||||
|
IProgress<(int Current, int Total, string Name)>? progress = null,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
var results = new List<BatchScenarioResult>(scenarios.Count);
|
||||||
|
var totalStopwatch = Stopwatch.StartNew();
|
||||||
|
|
||||||
|
for (var index = 0; index < scenarios.Count; index++)
|
||||||
|
{
|
||||||
|
cancellationToken.ThrowIfCancellationRequested();
|
||||||
|
|
||||||
|
var scenario = scenarios[index];
|
||||||
|
progress?.Report((index + 1, scenarios.Count, scenario.Name));
|
||||||
|
|
||||||
|
var outcome = await RedactAsync(scenario.Prompt, cancellationToken).ConfigureAwait(false);
|
||||||
|
results.Add(EvaluateScenario(scenario, outcome));
|
||||||
|
}
|
||||||
|
|
||||||
|
totalStopwatch.Stop();
|
||||||
|
|
||||||
|
var passed = results.Count(result => result.Passed);
|
||||||
|
return new BatchRunSummary(
|
||||||
|
scenarios.Count,
|
||||||
|
passed,
|
||||||
|
scenarios.Count - passed,
|
||||||
|
results,
|
||||||
|
totalStopwatch.ElapsedMilliseconds);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static RedactionOutcome MapOutcome(SanitizationResult result, long elapsedMilliseconds)
|
||||||
|
{
|
||||||
|
var entities = result.DetectedEntities
|
||||||
|
.Select(RedactionDisplayModel.FromEntity)
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
var placeholders = result.Redaction.PlaceholderMap
|
||||||
|
.Select(pair => new PlaceholderDisplayModel
|
||||||
|
{
|
||||||
|
Placeholder = pair.Key,
|
||||||
|
OriginalValue = pair.Value
|
||||||
|
})
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
var hasLeak = result.DetectedEntities.Any(entity =>
|
||||||
|
!string.IsNullOrWhiteSpace(entity.Value) &&
|
||||||
|
result.SanitizedPrompt.Contains(entity.Value, StringComparison.Ordinal));
|
||||||
|
|
||||||
|
return new RedactionOutcome(
|
||||||
|
result.OriginalPrompt,
|
||||||
|
result.SanitizedPrompt,
|
||||||
|
entities,
|
||||||
|
placeholders,
|
||||||
|
elapsedMilliseconds,
|
||||||
|
hasLeak,
|
||||||
|
result.NerModelsInvoked,
|
||||||
|
NerRoutingDisplay.FormatInvokedModels(result.NerModelsInvoked));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
using PiiRedaction.Core.Detection;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.Services;
|
||||||
|
|
||||||
|
public sealed class ScriptAnalysisService : IScriptAnalysisService
|
||||||
|
{
|
||||||
|
private readonly ScriptRouter _scriptRouter = new();
|
||||||
|
|
||||||
|
public ScriptComposition GetComposition(string text) =>
|
||||||
|
string.IsNullOrWhiteSpace(text)
|
||||||
|
? ScriptComposition.NoLetters
|
||||||
|
: _scriptRouter.GetComposition(text);
|
||||||
|
}
|
||||||
523
src/PiiRedaction.TestHarness.Wpf/Services/TestPromptCatalog.cs
Normal file
523
src/PiiRedaction.TestHarness.Wpf/Services/TestPromptCatalog.cs
Normal file
@@ -0,0 +1,523 @@
|
|||||||
|
using PiiRedaction.TestHarness.Wpf.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.Services;
|
||||||
|
|
||||||
|
public sealed class TestPromptCatalog : ITestPromptCatalog
|
||||||
|
{
|
||||||
|
public IReadOnlyList<string> Topics => PromptTopics.FilterOptions;
|
||||||
|
|
||||||
|
public IReadOnlyList<TestPromptScenario> All { get; } =
|
||||||
|
[
|
||||||
|
// --- Career Guidance (Tamil students — English, Tamil, Tanglish, Mixed) ---
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"CareerEnglishItPath",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Career + NER + Regex",
|
||||||
|
"English: B.Tech graduate asking IT career advice with name, email, and phone.",
|
||||||
|
"Hello, I am Rahul Kumar from Coimbatore. My email is rahul.kumar@college.edu and mobile 9876543210. I finished B.Tech IT. Which software career path is best for me?",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.CareerGuidance),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"CareerEnglishResumeHelp",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Career + NER + Regex",
|
||||||
|
"English: student requesting resume guidance with name and email.",
|
||||||
|
"I am Divya Sharma and my email is divya.sharma@gmail.com. Can you suggest how to improve my resume for data analyst internships?",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.CareerGuidance),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"CareerEnglishPlacementStress",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Career + NER + Regex",
|
||||||
|
"English: final-year student sharing contact details for placement counselling.",
|
||||||
|
"I am Arjun Mehta, phone 9123456780, email arjun.mehta@campus.in. I am in my final year and campus placement offers are very low. What career options should I explore?",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.CareerGuidance),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"CareerTamilEngineeringChoice",
|
||||||
|
PromptLanguage.Tamil,
|
||||||
|
"Career + NER (Tamil) + Regex",
|
||||||
|
"Tamil: engineering student choosing between branches with name and phone.",
|
||||||
|
"நான் முருகன் ராஜா, தொலைபேசி 9845011223. பி.இ முதல் ஆண்டு முடித்தேன். சிவில் பொறியியலா மெக்கானிக்கலா எது நல்ல வேலை வாய்ப்பு தரும்?",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.CareerGuidance),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"CareerTamilMbaGuidance",
|
||||||
|
PromptLanguage.Tamil,
|
||||||
|
"Career + NER (Tamil) + Regex",
|
||||||
|
"Tamil: student asking MBA finance career path with name and email.",
|
||||||
|
"நான் வானதி குமார், மின்னஞ்சல் vanathi.k@univ.in. MBA finance career path பற்றி விளக்குங்கள்.",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.CareerGuidance),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"CareerTamilGovtExamPrep",
|
||||||
|
PromptLanguage.Tamil,
|
||||||
|
"Career + NER (Tamil) + Regex",
|
||||||
|
"Tamil: student preparing for government exams with contact details.",
|
||||||
|
"மாணவர் கார்த்திக் செல்வம், தொலைபேசி 9003214567. TNPSC Group 2 தேர்வுக்கு எப்படி தயாராகுவது? மின்னஞ்சல் karthik.s@prep.in",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.CareerGuidance),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"CareerTanglishAfter12th",
|
||||||
|
PromptLanguage.Tanglish,
|
||||||
|
"Career + NER (Tanglish) + Regex",
|
||||||
|
"Tanglish: student after 12th asking which course for software job.",
|
||||||
|
"Naan Suresh, 12th complete panniten, phone 9003789456. Software job ku enna course padikanum nu sollunga.",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.CareerGuidance),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"CareerTanglishPlacementHelp",
|
||||||
|
PromptLanguage.Tanglish,
|
||||||
|
"Career + NER (Tanglish) + Regex",
|
||||||
|
"Tanglish: student worried about placements with email.",
|
||||||
|
"Hi I am Keerthana from Madurai. Enga college la placement romba kammi. Next enna panrathu? Email keerthana.m@gmail.com",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.CareerGuidance),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"CareerTanglishHigherStudies",
|
||||||
|
PromptLanguage.Tanglish,
|
||||||
|
"Career + NER (Tanglish) + Regex",
|
||||||
|
"Tanglish: student asking about higher studies abroad with phone.",
|
||||||
|
"I am Pradeep, finished BCA. MS ku apply pannanum — guide pannunga. Phone 9840098765.",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.CareerGuidance),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"CareerMixedStudyAbroad",
|
||||||
|
PromptLanguage.Mixed,
|
||||||
|
"Career + NER (Mixed) + Regex",
|
||||||
|
"Mixed Tamil/English: study abroad guidance with name, phone, and email.",
|
||||||
|
"நான் David Thomas, phone 9887766554. Study abroad MS computer science ku guide pannunga. Email david.t@student.in",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.CareerGuidance),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"CareerMixedSkillUpgrade",
|
||||||
|
PromptLanguage.Mixed,
|
||||||
|
"Career + NER (Mixed) + Regex",
|
||||||
|
"Mixed: working professional asking about upskilling with contact info.",
|
||||||
|
"வானக்கம், I am Priya Nair working in BPO. Cloud computing ku switch panna phone 9876012345 and email priya.nair@work.com la details anupunga.",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.CareerGuidance),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"CareerNegativeGeneral",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Career + Negative",
|
||||||
|
"General career question with no student PII.",
|
||||||
|
"What skills are needed for a career in cloud computing after graduation?",
|
||||||
|
expectDetections: false,
|
||||||
|
topic: PromptTopics.CareerGuidance),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"CareerNegativeTamil",
|
||||||
|
PromptLanguage.Tamil,
|
||||||
|
"Career + Negative",
|
||||||
|
"Tamil career market question without personal identifiers.",
|
||||||
|
"இன்றைய job market ல் data science career prospects எப்படி இருக்கும்? பொதுவான விளக்கம் தருங்கள்.",
|
||||||
|
expectDetections: false,
|
||||||
|
topic: PromptTopics.CareerGuidance),
|
||||||
|
|
||||||
|
// --- Banking & Financial ---
|
||||||
|
Scenario(
|
||||||
|
"FullFinancialWithCustomer",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"NER + Regex + Domain",
|
||||||
|
"Canonical demo: person name plus email, phone, loan number, and PAN.",
|
||||||
|
"Customer Ravi Kumar with email ravi.kumar@gmail.com and phone 9876543210 has LoanNumber LN-456789 and PAN ABCDE1234F. Please summarize this customer issue.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"CustomerNameOnly",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"NER",
|
||||||
|
"Person name detected via ONNX NER after 'Customer' keyword.",
|
||||||
|
"Customer Anita Sharma reported unauthorized transactions on her savings account.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"MrTitlePerson",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"NER",
|
||||||
|
"Person detected via ONNX NER (title prefix Mr.).",
|
||||||
|
"Mr. John Smith called about a duplicate debit on 15 March.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"MrsTitlePerson",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"NER",
|
||||||
|
"Person detected via ONNX NER (title prefix Mrs.).",
|
||||||
|
"Mrs. Lakshmi Reddy requested a callback regarding LN-112233.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"DrTitlePerson",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"NER",
|
||||||
|
"Person detected via ONNX NER (title prefix Dr).",
|
||||||
|
"Dr. Jane Doe escalated a complaint about delayed loan disbursement.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"TwoCustomersInOnePrompt",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"NER",
|
||||||
|
"Two distinct person names in the same prompt.",
|
||||||
|
"Customer Ravi Kumar and Customer Priya Nair disputed the same charge.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"PersonWithDomainIds",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"NER + Domain",
|
||||||
|
"Person name combined with business identifiers.",
|
||||||
|
"Customer Meera Iyer holds CID-7070 and account ACC-606060 for verification.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"PersonWithEmailNoPhone",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"NER + Regex",
|
||||||
|
"Person and email without phone number.",
|
||||||
|
"Customer Arjun Mehta wrote from arjun.mehta@company.in about KYC renewal.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"AllRegexTypes",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Regex",
|
||||||
|
"Email, phone, PAN, Aadhaar, and credit card in one prompt.",
|
||||||
|
"Email a@b.co phone 9001234567 PAN ABCDE1234F aadhaar 1234 5678 9012 card 4111-1111-1111-1111.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"AllDomainIds",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Domain",
|
||||||
|
"Loan number, customer ID, and account number together.",
|
||||||
|
"Please verify LN-100200 for CustomerId CID-3000 on AccountNumber ACC-400500.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"TamilCustomerNameOnly",
|
||||||
|
PromptLanguage.Tamil,
|
||||||
|
"NER (Tamil)",
|
||||||
|
"Tamil script person name detected via Tamil ONNX NER.",
|
||||||
|
"வாடிக்கையாளர் ராஜேஷ் குமார் சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"TamilWithPhonePan",
|
||||||
|
PromptLanguage.Tamil,
|
||||||
|
"NER (Tamil) + Regex",
|
||||||
|
"Tamil script person plus phone and PAN (regex).",
|
||||||
|
"வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210 PAN ABCDE1234F.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"TanglishCustomer",
|
||||||
|
PromptLanguage.Tanglish,
|
||||||
|
"NER (English/Tanglish)",
|
||||||
|
"Latin-script Tanglish person name via English ONNX NER.",
|
||||||
|
"Customer Senthil phone 9876543210 reported a failed UPI transfer.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"MixedTamilEnglish",
|
||||||
|
PromptLanguage.Mixed,
|
||||||
|
"NER (Mixed)",
|
||||||
|
"Code-mixed Tamil and English — both script routers may contribute person spans.",
|
||||||
|
"வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"TamilFullFinancial",
|
||||||
|
PromptLanguage.Tamil,
|
||||||
|
"NER (Tamil) + Regex + Domain",
|
||||||
|
"Tamil person with email, phone, loan number, and PAN (canonical demo in Tamil).",
|
||||||
|
"வாடிக்கையாளர் ராஜேஷ் குமார் மின்னஞ்சல் ravi.kumar@gmail.com தொலைபேசி 9876543210 LoanNumber LN-456789 PAN ABCDE1234F. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"NoPiiCleanTicket",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Negative",
|
||||||
|
"No PII — prompt passes through unchanged.",
|
||||||
|
"What is the status of ticket TKT-99887 and when will the API maintenance end?",
|
||||||
|
expectDetections: false,
|
||||||
|
topic: PromptTopics.Negative),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"NegativeWorkflowQuestion",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Negative",
|
||||||
|
"General workflow question with no regulated identifiers.",
|
||||||
|
"Summarize the retail loan approval workflow and typical SLA milestones.",
|
||||||
|
expectDetections: false,
|
||||||
|
topic: PromptTopics.Negative),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"EdgePhoneOnly",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Edge + Regex",
|
||||||
|
"Digits-only phone without a person name.",
|
||||||
|
"Callback requested on 9123456780 regarding branch hours.",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.EdgeHarness),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"EdgeLongMixed",
|
||||||
|
PromptLanguage.Mixed,
|
||||||
|
"Edge + NER (Mixed)",
|
||||||
|
"Longer mixed-language prompt with person and phone.",
|
||||||
|
"வாடிக்கையாளர் Priya Nair called from Chennai about a delayed NEFT transfer. She asked whether LoanNumber LN-909090 is linked to account ACC-808080 and wants an email confirmation sent to priya.nair@example.com on phone 9988776655.",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.EdgeHarness),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"LeakCheckNestedEmail",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"LeakCheck + Regex",
|
||||||
|
"Email embedded in a sentence — placeholders must fully replace the address.",
|
||||||
|
"Please forward the statement for customer.support@banking.example to the operations desk.",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.EdgeHarness),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"TamilEdgePunctuation",
|
||||||
|
PromptLanguage.Tamil,
|
||||||
|
"TamilEdge + NER (Tamil)",
|
||||||
|
"Tamil name surrounded by punctuation and Tamil numerals.",
|
||||||
|
"வாடிக்கையாளர் (ராஜேஷ் குமார்) — தொலைபேசி ௯௮௭௬௫௪௩௨௧௦ — உதவி தேவை.",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.EdgeHarness),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"TanglishLatinInTamilSentence",
|
||||||
|
PromptLanguage.Tanglish,
|
||||||
|
"Tanglish + NER",
|
||||||
|
"Latin person name inside otherwise Tamil context.",
|
||||||
|
"வாடிக்கையாளர் Arun Kumar அவர்களின் KYC ஆவணம் நிலுவையில் உள்ளது.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
// --- English banking & customer-service scenarios ---
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"MsTitlePerson",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"NER",
|
||||||
|
"Person detected via ONNX NER (title prefix Ms.).",
|
||||||
|
"Ms. Kavitha Nair requested a statement for her fixed deposit renewal.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"KycAadhaarUpload",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"NER + Regex",
|
||||||
|
"KYC follow-up with person name and Aadhaar number.",
|
||||||
|
"Customer Deepa Iyer uploaded Aadhaar 2345 6789 0123 for video KYC completion.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"CreditCardDispute",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"NER + Regex",
|
||||||
|
"Card dispute with person, masked card, and email.",
|
||||||
|
"Customer Vikram Singh disputed charge on card 4532-1234-5678-9010 and wrote from vikram.singh@mail.com.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"SavingsAccountClosure",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"NER + Domain",
|
||||||
|
"Account closure request with person name and account number.",
|
||||||
|
"Customer Sanjay Patel wants to close AccountNumber ACC-112233 and transfer the balance.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"LoanStatusByNumber",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Domain",
|
||||||
|
"Loan status lookup using loan number only (no person name).",
|
||||||
|
"Please check disbursement status for LoanNumber LN-778899 and share the expected credit date.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"CustomerIdLookup",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Domain",
|
||||||
|
"CRM lookup using customer ID without a person name.",
|
||||||
|
"Pull interaction history for CustomerId CID-5521 related to the mobile app login failure.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"EmailOnlySupport",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Regex",
|
||||||
|
"Support thread with email address but no detectable person name.",
|
||||||
|
"Reply to the customer at support.user@example.org about the delayed NEFT credit.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"PanAndPhoneNoName",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Regex",
|
||||||
|
"PAN and phone provided for callback without a person name.",
|
||||||
|
"Verify PAN FGHIJ5678K and call back on 9988123456 regarding the EMI bounce.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"RealWorldChargebackNote",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"NER + Regex + Domain",
|
||||||
|
"Realistic call-centre note combining person, phone, loan, and email.",
|
||||||
|
"Customer Meera Iyer called from 9876012345 about a duplicate EMI debit on LoanNumber LN-334455. She can be reached at meera.iyer@bank.in for confirmation.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"EdgeMultiplePhones",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Regex",
|
||||||
|
"Two phone numbers in one prompt (primary and alternate).",
|
||||||
|
"Reach the customer on 9123456780 or alternate 9988776655 for OTP verification.",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.EdgeHarness),
|
||||||
|
|
||||||
|
// --- Tamil scenarios ---
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"TamilLoanDispute",
|
||||||
|
PromptLanguage.Tamil,
|
||||||
|
"NER (Tamil) + Domain",
|
||||||
|
"Tamil person name with loan and account identifiers.",
|
||||||
|
"வாடிக்கையாளர் பிரியா ராமன் LoanNumber LN-220011 கணக்கு ACC-445566 இல் தவறான பற்று வைக்கப்பட்டுள்ளது என புகாரளித்தார்.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"TamilAadhaarKyc",
|
||||||
|
PromptLanguage.Tamil,
|
||||||
|
"NER (Tamil) + Regex",
|
||||||
|
"Tamil script KYC prompt with Aadhaar number.",
|
||||||
|
"வாடிக்கையாளர் சுரேஷ் பாபு ஆதார் 4567 8901 2345 உடன் KYC புதுப்பிப்பை முடிக்க வேண்டும்.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"TamilEmailFollowUp",
|
||||||
|
PromptLanguage.Tamil,
|
||||||
|
"NER (Tamil) + Regex",
|
||||||
|
"Tamil customer follow-up with email address.",
|
||||||
|
"வாடிக்கையாளர் லட்சுமி மின்னஞ்சல் lakshmi.devi@example.com மூலம் சேமிப்பு வட்டி விளக்கம் கேட்டார்.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"TamilNegativeFaq",
|
||||||
|
PromptLanguage.Tamil,
|
||||||
|
"Negative",
|
||||||
|
"Tamil product FAQ with no regulated identifiers.",
|
||||||
|
"சேமிப்பு கணக்கிற்கான வட்டி விகிதம் எப்படி கணக்கிடப்படுகிறது? தயவுசெய்து விளக்கவும்.",
|
||||||
|
expectDetections: false,
|
||||||
|
topic: PromptTopics.Negative),
|
||||||
|
|
||||||
|
// --- Tanglish scenarios ---
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"TanglishUpiRefund",
|
||||||
|
PromptLanguage.Tanglish,
|
||||||
|
"NER (English/Tanglish) + Regex",
|
||||||
|
"Tanglish UPI refund complaint with person and phone.",
|
||||||
|
"Customer Karthik said UPI payment failed, please refund. Phone 9845012345.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"TanglishMsPriyaCallback",
|
||||||
|
PromptLanguage.Tanglish,
|
||||||
|
"NER (English/Tanglish) + Regex",
|
||||||
|
"Tanglish callback request with title and phone.",
|
||||||
|
"Ms Priya called — enna panrathu? Callback 9003123456 before 6 PM.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"TanglishLoanAndCid",
|
||||||
|
PromptLanguage.Tanglish,
|
||||||
|
"NER + Domain",
|
||||||
|
"Tanglish mix of person name, loan number, and customer ID.",
|
||||||
|
"Customer Ganesh holds CID-8812 for LoanNumber LN-660033 and needs disbursement update.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
// --- Mixed-script scenarios ---
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"MixedCallCenterHandoff",
|
||||||
|
PromptLanguage.Mixed,
|
||||||
|
"NER (Mixed) + Regex + Domain",
|
||||||
|
"Call-centre handoff note mixing Tamil, English, phone, and loan ID.",
|
||||||
|
"வாடிக்கையாளர் Anitha Roy phone 9876501234 says EMI debited twice on LoanNumber LN-550077. Email anitha.roy@corp.in for receipt.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"MixedTamilEnglishAccount",
|
||||||
|
PromptLanguage.Mixed,
|
||||||
|
"NER (Mixed) + Domain",
|
||||||
|
"Mixed prompt with English name embedded in Tamil sentence and account number.",
|
||||||
|
"வாடிக்கையாளர் David Thomas அவர்கள் AccountNumber ACC-990011 ஐ மூட விரும்புகிறார்.",
|
||||||
|
expectDetections: true),
|
||||||
|
|
||||||
|
// --- Additional negative / edge scenarios ---
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"NegativeProductFaq",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Negative",
|
||||||
|
"Product FAQ about interest rates — no customer PII.",
|
||||||
|
"What is the current savings account interest rate and how is it credited quarterly?",
|
||||||
|
expectDetections: false,
|
||||||
|
topic: PromptTopics.Negative),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"NegativeBranchLocator",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Negative",
|
||||||
|
"Branch locator query using branch codes only.",
|
||||||
|
"List branches open on Sunday in Chennai zone BR-CHN-04 and BR-CHN-09.",
|
||||||
|
expectDetections: false,
|
||||||
|
topic: PromptTopics.Negative),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"EdgeAadhaarVariants",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Regex",
|
||||||
|
"Aadhaar with spaced digits alongside PAN.",
|
||||||
|
"Documents on file: PAN KLMPN4567Q and Aadhaar 9876 5432 1098 for verification.",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.EdgeHarness),
|
||||||
|
|
||||||
|
Scenario(
|
||||||
|
"EdgeDomainWithoutLabels",
|
||||||
|
PromptLanguage.English,
|
||||||
|
"Domain",
|
||||||
|
"Domain IDs embedded in natural sentence without explicit field labels.",
|
||||||
|
"The case is tied to LN-445566 and ACC-778899 under CRM record CID-4400.",
|
||||||
|
expectDetections: true,
|
||||||
|
topic: PromptTopics.EdgeHarness)
|
||||||
|
];
|
||||||
|
|
||||||
|
private static TestPromptScenario Scenario(
|
||||||
|
string name,
|
||||||
|
PromptLanguage language,
|
||||||
|
string category,
|
||||||
|
string description,
|
||||||
|
string prompt,
|
||||||
|
bool expectDetections,
|
||||||
|
string topic = PromptTopics.BankingFinancial) =>
|
||||||
|
new(name, name, topic, language, category, description, prompt, expectDetections);
|
||||||
|
}
|
||||||
380
src/PiiRedaction.TestHarness.Wpf/ViewModels/MainViewModel.cs
Normal file
380
src/PiiRedaction.TestHarness.Wpf/ViewModels/MainViewModel.cs
Normal file
@@ -0,0 +1,380 @@
|
|||||||
|
using System.Collections.ObjectModel;
|
||||||
|
using System.ComponentModel;
|
||||||
|
using System.Windows;
|
||||||
|
using System.Windows.Data;
|
||||||
|
using CommunityToolkit.Mvvm.ComponentModel;
|
||||||
|
using CommunityToolkit.Mvvm.Input;
|
||||||
|
using PiiRedaction.Core.Detection;
|
||||||
|
using PiiRedaction.TestHarness.Wpf.Models;
|
||||||
|
using PiiRedaction.TestHarness.Wpf.Services;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.ViewModels;
|
||||||
|
|
||||||
|
public partial class MainViewModel : ObservableObject
|
||||||
|
{
|
||||||
|
private readonly IRedactionAppService _redactionAppService;
|
||||||
|
private readonly ITestPromptCatalog _promptCatalog;
|
||||||
|
private readonly IScriptAnalysisService _scriptAnalysisService;
|
||||||
|
private readonly IModelStatusService _modelStatusService;
|
||||||
|
private readonly INerLogService _nerLogService;
|
||||||
|
|
||||||
|
public MainViewModel(
|
||||||
|
IRedactionAppService redactionAppService,
|
||||||
|
ITestPromptCatalog promptCatalog,
|
||||||
|
IScriptAnalysisService scriptAnalysisService,
|
||||||
|
IModelStatusService modelStatusService,
|
||||||
|
INerLogService nerLogService)
|
||||||
|
{
|
||||||
|
_redactionAppService = redactionAppService;
|
||||||
|
_promptCatalog = promptCatalog;
|
||||||
|
_scriptAnalysisService = scriptAnalysisService;
|
||||||
|
_modelStatusService = modelStatusService;
|
||||||
|
_nerLogService = nerLogService;
|
||||||
|
|
||||||
|
PromptItems = new ObservableCollection<TestPromptItemViewModel>(
|
||||||
|
_promptCatalog.All.Select(scenario => new TestPromptItemViewModel(scenario)));
|
||||||
|
|
||||||
|
PromptsView = CollectionViewSource.GetDefaultView(PromptItems);
|
||||||
|
PromptsView.GroupDescriptions.Add(new PropertyGroupDescription(nameof(TestPromptItemViewModel.Topic)));
|
||||||
|
PromptsView.SortDescriptions.Add(new SortDescription(nameof(TestPromptItemViewModel.TopicSortOrder), ListSortDirection.Ascending));
|
||||||
|
PromptsView.SortDescriptions.Add(new SortDescription(nameof(TestPromptItemViewModel.Name), ListSortDirection.Ascending));
|
||||||
|
PromptsView.Filter = FilterPrompt;
|
||||||
|
|
||||||
|
TopicCategories = new ObservableCollection<string>(_promptCatalog.Topics);
|
||||||
|
SelectedTopicCategory = PromptTopics.All;
|
||||||
|
|
||||||
|
DetectedEntities = [];
|
||||||
|
PlaceholderMap = [];
|
||||||
|
BatchResults = [];
|
||||||
|
|
||||||
|
RefreshModelStatus();
|
||||||
|
UpdateScriptComposition();
|
||||||
|
}
|
||||||
|
|
||||||
|
public ICollectionView PromptsView { get; }
|
||||||
|
|
||||||
|
public ObservableCollection<TestPromptItemViewModel> PromptItems { get; }
|
||||||
|
|
||||||
|
public ObservableCollection<RedactionDisplayModel> DetectedEntities { get; }
|
||||||
|
|
||||||
|
public ObservableCollection<PlaceholderDisplayModel> PlaceholderMap { get; }
|
||||||
|
|
||||||
|
public ObservableCollection<BatchScenarioResult> BatchResults { get; }
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private string _inputPrompt = string.Empty;
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private string _sanitizedOutput = string.Empty;
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private string _originalPrompt = string.Empty;
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private string _mockLlmResponse = string.Empty;
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private TestPromptItemViewModel? _selectedPrompt;
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private string _nerModelsInvokedSummary = "—";
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private ScriptComposition _scriptComposition = ScriptComposition.NoLetters;
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private long _elapsedMilliseconds;
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private int _entityCount;
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private string _statusMessage = "Ready";
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private bool _isBusy;
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private string _modelStatus = string.Empty;
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private bool _leakWarning;
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private string _promptFilter = string.Empty;
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private string _selectedTopicCategory = PromptTopics.All;
|
||||||
|
|
||||||
|
public ObservableCollection<string> TopicCategories { get; }
|
||||||
|
|
||||||
|
public ObservableCollection<string> NerLogLines => _nerLogService.Lines;
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private string _batchSummary = string.Empty;
|
||||||
|
|
||||||
|
[ObservableProperty]
|
||||||
|
private bool _isBatchExpanded;
|
||||||
|
|
||||||
|
partial void OnInputPromptChanged(string value)
|
||||||
|
{
|
||||||
|
UpdateScriptComposition();
|
||||||
|
RedactCommand.NotifyCanExecuteChanged();
|
||||||
|
}
|
||||||
|
|
||||||
|
partial void OnSelectedPromptChanged(TestPromptItemViewModel? value)
|
||||||
|
{
|
||||||
|
if (value is null)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
ClearRedactionResults();
|
||||||
|
InputPrompt = value.Prompt;
|
||||||
|
StatusMessage = $"Loaded prompt: {value.Name}";
|
||||||
|
}
|
||||||
|
|
||||||
|
[RelayCommand]
|
||||||
|
private void ClearNerLogs()
|
||||||
|
{
|
||||||
|
_nerLogService.Clear();
|
||||||
|
StatusMessage = "NER logs cleared.";
|
||||||
|
}
|
||||||
|
|
||||||
|
[RelayCommand]
|
||||||
|
private void Clear()
|
||||||
|
{
|
||||||
|
InputPrompt = string.Empty;
|
||||||
|
SelectedPrompt = null;
|
||||||
|
ClearRedactionResults();
|
||||||
|
BatchResults.Clear();
|
||||||
|
BatchSummary = string.Empty;
|
||||||
|
IsBatchExpanded = false;
|
||||||
|
StatusMessage = "Cleared.";
|
||||||
|
UpdateScriptComposition();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ClearRedactionResults()
|
||||||
|
{
|
||||||
|
SanitizedOutput = string.Empty;
|
||||||
|
OriginalPrompt = string.Empty;
|
||||||
|
MockLlmResponse = string.Empty;
|
||||||
|
DetectedEntities.Clear();
|
||||||
|
PlaceholderMap.Clear();
|
||||||
|
LeakWarning = false;
|
||||||
|
EntityCount = 0;
|
||||||
|
ElapsedMilliseconds = 0;
|
||||||
|
NerModelsInvokedSummary = "—";
|
||||||
|
SendToMockLlmCommand.NotifyCanExecuteChanged();
|
||||||
|
}
|
||||||
|
|
||||||
|
[RelayCommand(CanExecute = nameof(CanRedact))]
|
||||||
|
private async Task RedactAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(InputPrompt))
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
IsBusy = true;
|
||||||
|
StatusMessage = "Redacting...";
|
||||||
|
|
||||||
|
var outcome = await _redactionAppService.RedactAsync(InputPrompt, cancellationToken)
|
||||||
|
.ConfigureAwait(true);
|
||||||
|
|
||||||
|
ApplyOutcome(outcome);
|
||||||
|
StatusMessage = $"Redaction complete in {outcome.ElapsedMilliseconds} ms.";
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException)
|
||||||
|
{
|
||||||
|
StatusMessage = "Redaction cancelled.";
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
StatusMessage = $"Redaction failed: {ex.Message}";
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
IsBusy = false;
|
||||||
|
RedactCommand.NotifyCanExecuteChanged();
|
||||||
|
RunAllScenariosCommand.NotifyCanExecuteChanged();
|
||||||
|
SendToMockLlmCommand.NotifyCanExecuteChanged();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool CanRedact() => !IsBusy && !string.IsNullOrWhiteSpace(InputPrompt);
|
||||||
|
|
||||||
|
[RelayCommand]
|
||||||
|
private void CopySanitized()
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(SanitizedOutput))
|
||||||
|
{
|
||||||
|
StatusMessage = "Nothing to copy.";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Clipboard.SetText(SanitizedOutput);
|
||||||
|
StatusMessage = "Sanitized output copied to clipboard.";
|
||||||
|
}
|
||||||
|
|
||||||
|
[RelayCommand(CanExecute = nameof(CanSendToMockLlm))]
|
||||||
|
private async Task SendToMockLlmAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(SanitizedOutput))
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
IsBusy = true;
|
||||||
|
StatusMessage = "Sending to mock LLM...";
|
||||||
|
|
||||||
|
MockLlmResponse = await _redactionAppService
|
||||||
|
.SendToMockLlmAsync(SanitizedOutput, cancellationToken)
|
||||||
|
.ConfigureAwait(true);
|
||||||
|
|
||||||
|
StatusMessage = "Mock LLM response received.";
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException)
|
||||||
|
{
|
||||||
|
StatusMessage = "Mock LLM call cancelled.";
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
StatusMessage = $"Mock LLM call failed: {ex.Message}";
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
IsBusy = false;
|
||||||
|
RedactCommand.NotifyCanExecuteChanged();
|
||||||
|
RunAllScenariosCommand.NotifyCanExecuteChanged();
|
||||||
|
SendToMockLlmCommand.NotifyCanExecuteChanged();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool CanSendToMockLlm() => !IsBusy && !string.IsNullOrWhiteSpace(SanitizedOutput);
|
||||||
|
|
||||||
|
[RelayCommand(CanExecute = nameof(CanRunBatch))]
|
||||||
|
private async Task RunAllScenariosAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
IsBusy = true;
|
||||||
|
IsBatchExpanded = true;
|
||||||
|
BatchResults.Clear();
|
||||||
|
BatchSummary = "Running batch validation...";
|
||||||
|
StatusMessage = "Running all scenarios...";
|
||||||
|
|
||||||
|
var scenarios = GetVisibleScenarios();
|
||||||
|
|
||||||
|
var progress = new Progress<(int Current, int Total, string Name)>(report =>
|
||||||
|
{
|
||||||
|
StatusMessage = $"Batch {report.Current}/{report.Total}: {report.Name}";
|
||||||
|
});
|
||||||
|
|
||||||
|
var summary = await _redactionAppService
|
||||||
|
.RunAllScenariosAsync(scenarios, progress, cancellationToken)
|
||||||
|
.ConfigureAwait(true);
|
||||||
|
|
||||||
|
BatchResults.Clear();
|
||||||
|
foreach (var result in summary.Results)
|
||||||
|
{
|
||||||
|
BatchResults.Add(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
BatchSummary =
|
||||||
|
$"{summary.Passed}/{summary.Total} passed in {summary.TotalElapsedMilliseconds} ms";
|
||||||
|
|
||||||
|
StatusMessage = summary.Failed == 0
|
||||||
|
? $"Batch complete: all {summary.Total} scenarios passed."
|
||||||
|
: $"Batch complete: {summary.Failed} scenario(s) failed.";
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException)
|
||||||
|
{
|
||||||
|
StatusMessage = "Batch run cancelled.";
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
StatusMessage = $"Batch run failed: {ex.Message}";
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
IsBusy = false;
|
||||||
|
RedactCommand.NotifyCanExecuteChanged();
|
||||||
|
RunAllScenariosCommand.NotifyCanExecuteChanged();
|
||||||
|
SendToMockLlmCommand.NotifyCanExecuteChanged();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool CanRunBatch() => !IsBusy;
|
||||||
|
|
||||||
|
partial void OnPromptFilterChanged(string value) => PromptsView.Refresh();
|
||||||
|
|
||||||
|
partial void OnSelectedTopicCategoryChanged(string value) => PromptsView.Refresh();
|
||||||
|
|
||||||
|
private void ApplyOutcome(RedactionOutcome outcome)
|
||||||
|
{
|
||||||
|
OriginalPrompt = outcome.OriginalPrompt;
|
||||||
|
SanitizedOutput = outcome.SanitizedPrompt;
|
||||||
|
ElapsedMilliseconds = outcome.ElapsedMilliseconds;
|
||||||
|
EntityCount = outcome.DetectedEntities.Count;
|
||||||
|
LeakWarning = outcome.HasLeak;
|
||||||
|
NerModelsInvokedSummary = outcome.NerModelsInvokedSummary;
|
||||||
|
|
||||||
|
DetectedEntities.Clear();
|
||||||
|
foreach (var entity in outcome.DetectedEntities)
|
||||||
|
{
|
||||||
|
DetectedEntities.Add(entity);
|
||||||
|
}
|
||||||
|
|
||||||
|
PlaceholderMap.Clear();
|
||||||
|
foreach (var placeholder in outcome.Placeholders)
|
||||||
|
{
|
||||||
|
PlaceholderMap.Add(placeholder);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void RefreshModelStatus()
|
||||||
|
{
|
||||||
|
var snapshot = _modelStatusService.GetStatus();
|
||||||
|
ModelStatus = snapshot.Summary;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateScriptComposition() =>
|
||||||
|
ScriptComposition = _scriptAnalysisService.GetComposition(InputPrompt);
|
||||||
|
|
||||||
|
private bool FilterPrompt(object item)
|
||||||
|
{
|
||||||
|
if (item is not TestPromptItemViewModel promptItem)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!string.Equals(SelectedTopicCategory, PromptTopics.All, StringComparison.Ordinal)
|
||||||
|
&& !promptItem.Topic.Equals(SelectedTopicCategory, StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (string.IsNullOrWhiteSpace(PromptFilter))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
var filter = PromptFilter.Trim();
|
||||||
|
return promptItem.Name.Contains(filter, StringComparison.OrdinalIgnoreCase)
|
||||||
|
|| promptItem.Topic.Contains(filter, StringComparison.OrdinalIgnoreCase)
|
||||||
|
|| promptItem.Category.Contains(filter, StringComparison.OrdinalIgnoreCase)
|
||||||
|
|| promptItem.Description.Contains(filter, StringComparison.OrdinalIgnoreCase)
|
||||||
|
|| promptItem.Language.ToString().Contains(filter, StringComparison.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
|
|
||||||
|
private IReadOnlyList<TestPromptScenario> GetVisibleScenarios() =>
|
||||||
|
PromptItems
|
||||||
|
.Where(item => FilterPrompt(item))
|
||||||
|
.Select(item => item.Scenario)
|
||||||
|
.ToList();
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
using PiiRedaction.TestHarness.Wpf.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.TestHarness.Wpf.ViewModels;
|
||||||
|
|
||||||
|
public sealed class TestPromptItemViewModel
|
||||||
|
{
|
||||||
|
public TestPromptItemViewModel(TestPromptScenario scenario)
|
||||||
|
{
|
||||||
|
Scenario = scenario;
|
||||||
|
}
|
||||||
|
|
||||||
|
public TestPromptScenario Scenario { get; }
|
||||||
|
|
||||||
|
public string Name => Scenario.Name;
|
||||||
|
|
||||||
|
public string Topic => Scenario.Topic;
|
||||||
|
|
||||||
|
public int TopicSortOrder => PromptTopics.SortOrder(Scenario.Topic);
|
||||||
|
|
||||||
|
public string Category => Scenario.Category;
|
||||||
|
|
||||||
|
public PromptLanguage Language => Scenario.Language;
|
||||||
|
|
||||||
|
public string Description => Scenario.Description;
|
||||||
|
|
||||||
|
public string Prompt => Scenario.Prompt;
|
||||||
|
|
||||||
|
public string DisplayLabel => $"{Name} ({Language})";
|
||||||
|
}
|
||||||
14
src/PiiRedaction.TestHarness.Wpf/appsettings.json
Normal file
14
src/PiiRedaction.TestHarness.Wpf/appsettings.json
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"Logging": {
|
||||||
|
"LogLevel": {
|
||||||
|
"Default": "Information",
|
||||||
|
"PiiRedaction.Infrastructure.Onnx": "Debug"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"PiiRedaction": {
|
||||||
|
"OnnxModelPath": "models/ner-model.onnx",
|
||||||
|
"EnglishOnnxModelPath": "models/en/ner-model.onnx",
|
||||||
|
"TamilOnnxModelPath": "models/ta/model.onnx",
|
||||||
|
"EnableTamilNer": true
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -46,6 +46,36 @@ public sealed class OnnxNerPiiDetectorTests
|
|||||||
action.Should().Throw<ArgumentException>();
|
action.Should().Throw<ArgumentException>();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void Detect_ModelAvailable_RecordsInvokedModels()
|
||||||
|
{
|
||||||
|
var runner = new FakeOnnxNerModelRunner
|
||||||
|
{
|
||||||
|
IsModelAvailable = true,
|
||||||
|
InvokedModelsToReturn = [NerModelOrigin.English, NerModelOrigin.Tamil]
|
||||||
|
};
|
||||||
|
|
||||||
|
var detector = new OnnxNerPiiDetector(runner);
|
||||||
|
detector.Detect("Mixed prompt");
|
||||||
|
|
||||||
|
detector.LastInvokedModels.Should().Equal(NerModelOrigin.English, NerModelOrigin.Tamil);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void Detect_ModelUnavailable_ClearsInvokedModels()
|
||||||
|
{
|
||||||
|
var runner = new FakeOnnxNerModelRunner
|
||||||
|
{
|
||||||
|
IsModelAvailable = false,
|
||||||
|
InvokedModelsToReturn = [NerModelOrigin.English]
|
||||||
|
};
|
||||||
|
|
||||||
|
var detector = new OnnxNerPiiDetector(runner);
|
||||||
|
detector.Detect("Any text");
|
||||||
|
|
||||||
|
detector.LastInvokedModels.Should().BeEmpty();
|
||||||
|
}
|
||||||
|
|
||||||
private static OnnxNerPiiDetector CreateDetector(bool modelAvailable)
|
private static OnnxNerPiiDetector CreateDetector(bool modelAvailable)
|
||||||
{
|
{
|
||||||
var runner = new FakeOnnxNerModelRunner { IsModelAvailable = modelAvailable };
|
var runner = new FakeOnnxNerModelRunner { IsModelAvailable = modelAvailable };
|
||||||
|
|||||||
54
tests/PiiRedaction.Core.Tests/Detection/ScriptRouterTests.cs
Normal file
54
tests/PiiRedaction.Core.Tests/Detection/ScriptRouterTests.cs
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
using FluentAssertions;
|
||||||
|
using PiiRedaction.Core.Detection;
|
||||||
|
using PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Core.Tests.Detection;
|
||||||
|
|
||||||
|
[TestFixture]
|
||||||
|
public sealed class ScriptRouterTests
|
||||||
|
{
|
||||||
|
private readonly ScriptRouter _router = new();
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void GetComposition_LatinOnly_ReturnsLatinOnly()
|
||||||
|
{
|
||||||
|
_router.GetComposition("Customer Ravi Kumar called about billing.")
|
||||||
|
.Should().Be(ScriptComposition.LatinOnly);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void GetComposition_TamilOnly_ReturnsTamilOnly()
|
||||||
|
{
|
||||||
|
_router.GetComposition("வாடிக்கையாளர் ராஜேஷ் தொலைபேசி 9876543210")
|
||||||
|
.Should().Be(ScriptComposition.TamilOnly);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void GetComposition_Mixed_ReturnsMixed()
|
||||||
|
{
|
||||||
|
_router.GetComposition("Rajesh மற்றும் Priya disputed the charge.")
|
||||||
|
.Should().Be(ScriptComposition.Mixed);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void GetComposition_MixedTamilEnglishSample_ReturnsMixed()
|
||||||
|
{
|
||||||
|
_router.GetComposition("வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge.")
|
||||||
|
.Should().Be(ScriptComposition.Mixed);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void GetComposition_NoLetters_ReturnsNoLetters()
|
||||||
|
{
|
||||||
|
_router.GetComposition("9876543210 12345")
|
||||||
|
.Should().Be(ScriptComposition.NoLetters);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void GetComposition_TamilBoundaryChars_AreClassifiedAsTamil()
|
||||||
|
{
|
||||||
|
_router.GetComposition("\u0B80").Should().Be(ScriptComposition.TamilOnly);
|
||||||
|
_router.GetComposition("\u0BFF").Should().Be(ScriptComposition.TamilOnly);
|
||||||
|
_router.GetComposition("A").Should().Be(ScriptComposition.LatinOnly);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ namespace PiiRedaction.Core.Tests.Integration;
|
|||||||
public sealed class GoldenPromptTests
|
public sealed class GoldenPromptTests
|
||||||
{
|
{
|
||||||
[TestCaseSource(typeof(PromptScenarioCatalog), nameof(PromptScenarioCatalog.AllScenarios))]
|
[TestCaseSource(typeof(PromptScenarioCatalog), nameof(PromptScenarioCatalog.AllScenarios))]
|
||||||
|
[TestCaseSource(typeof(TamilPromptScenarioCatalog), nameof(TamilPromptScenarioCatalog.AllScenarios))]
|
||||||
public void Sanitize_PromptScenario_ProducesExpectedOutput(PromptScenario scenario)
|
public void Sanitize_PromptScenario_ProducesExpectedOutput(PromptScenario scenario)
|
||||||
{
|
{
|
||||||
var sanitizer = ProductionPipelineFactory.CreateForScenario(scenario);
|
var sanitizer = ProductionPipelineFactory.CreateForScenario(scenario);
|
||||||
|
|||||||
@@ -0,0 +1,131 @@
|
|||||||
|
using FluentAssertions;
|
||||||
|
using PiiRedaction.Core.Abstractions;
|
||||||
|
using PiiRedaction.Core.Models;
|
||||||
|
using PiiRedaction.Core.Tests.TestSupport;
|
||||||
|
using PiiRedaction.Tests.Shared;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Core.Tests.Integration;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// End-to-end pipeline proof using routed English + Tamil ONNX NER models.
|
||||||
|
/// </summary>
|
||||||
|
[TestFixture]
|
||||||
|
[Category("TamilNer")]
|
||||||
|
public sealed class RealTamilPipelineTests : RealRoutingNerModelFixture
|
||||||
|
{
|
||||||
|
private IPromptSanitizer _sanitizer = null!;
|
||||||
|
|
||||||
|
[OneTimeSetUp]
|
||||||
|
public void OneTimeSetUpPipeline()
|
||||||
|
{
|
||||||
|
_sanitizer = ProductionPipelineFactory.CreateWithRoutingRealModels(EnglishRunner, TamilRunner);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void Sanitize_TamilCustomerNameOnly_RedactsPerson()
|
||||||
|
{
|
||||||
|
const string prompt =
|
||||||
|
"வாடிக்கையாளர் ராஜேஷ் குமார் சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்.";
|
||||||
|
|
||||||
|
var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
|
||||||
|
|
||||||
|
result.SanitizedPrompt.Should().Contain("<PERSON_1>");
|
||||||
|
result.SanitizedPrompt.Should().NotContain("ராஜேஷ்");
|
||||||
|
result.DetectedEntities.Should().Contain(entity =>
|
||||||
|
entity.Type == PiiEntityType.Person
|
||||||
|
&& entity.Source == PiiDetectionSource.Ner
|
||||||
|
&& entity.ModelOrigin == NerModelOrigin.Tamil);
|
||||||
|
result.NerModelsInvoked.Should().Equal(NerModelOrigin.Tamil);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void Sanitize_TanglishCustomer_InvokesEnglishNerOnly()
|
||||||
|
{
|
||||||
|
const string prompt = "Customer Senthil phone 9876543210 reported a failed UPI transfer.";
|
||||||
|
|
||||||
|
var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
|
||||||
|
|
||||||
|
result.NerModelsInvoked.Should().Equal(NerModelOrigin.English);
|
||||||
|
result.DetectedEntities.Should().Contain(entity =>
|
||||||
|
entity.Type == PiiEntityType.Person && entity.ModelOrigin == NerModelOrigin.English);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void Sanitize_MixedTamilEnglish_InvokesBothNerModels()
|
||||||
|
{
|
||||||
|
const string prompt = "வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge.";
|
||||||
|
|
||||||
|
var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
|
||||||
|
|
||||||
|
result.NerModelsInvoked.Should().Equal(NerModelOrigin.English, NerModelOrigin.Tamil);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void Sanitize_TamilWithPhonePan_RedactsPersonPhoneAndPan()
|
||||||
|
{
|
||||||
|
const string prompt = "வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210 PAN ABCDE1234F.";
|
||||||
|
|
||||||
|
var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
|
||||||
|
|
||||||
|
result.SanitizedPrompt.Should().Contain("<PERSON_1>");
|
||||||
|
result.SanitizedPrompt.Should().Contain("<PHONE_1>");
|
||||||
|
result.SanitizedPrompt.Should().Contain("<PAN_1>");
|
||||||
|
result.SanitizedPrompt.Should().NotContainAny("ராஜேஷ்", "9876543210", "ABCDE1234F");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void Sanitize_TanglishCustomer_RedactsPersonAndPhone()
|
||||||
|
{
|
||||||
|
const string prompt = "Customer Senthil phone 9876543210 reported a failed UPI transfer.";
|
||||||
|
|
||||||
|
var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
|
||||||
|
|
||||||
|
result.SanitizedPrompt.Should().Contain("<PERSON_1>");
|
||||||
|
result.SanitizedPrompt.Should().Contain("<PHONE_1>");
|
||||||
|
result.SanitizedPrompt.Should().NotContainAny("Senthil", "9876543210");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void Sanitize_MixedTamilEnglish_RedactsPersonAndPhone()
|
||||||
|
{
|
||||||
|
const string prompt = "வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge.";
|
||||||
|
|
||||||
|
var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
|
||||||
|
|
||||||
|
result.SanitizedPrompt.Should().Contain("<PERSON_1>");
|
||||||
|
result.SanitizedPrompt.Should().Contain("<PHONE_1>");
|
||||||
|
result.SanitizedPrompt.Should().NotContainAny("Ravi Kumar", "9876543210");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void Sanitize_TamilFullFinancial_RedactsAllPiiTypes()
|
||||||
|
{
|
||||||
|
const string prompt =
|
||||||
|
"வாடிக்கையாளர் ராஜேஷ் குமார் மின்னஞ்சல் ravi.kumar@gmail.com தொலைபேசி 9876543210 LoanNumber LN-456789 PAN ABCDE1234F. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்.";
|
||||||
|
|
||||||
|
var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
|
||||||
|
|
||||||
|
result.SanitizedPrompt.Should().Contain("<PERSON_1>");
|
||||||
|
result.SanitizedPrompt.Should().Contain("<EMAIL_1>");
|
||||||
|
result.SanitizedPrompt.Should().Contain("<PHONE_1>");
|
||||||
|
result.SanitizedPrompt.Should().Contain("<LOAN_NUMBER_1>");
|
||||||
|
result.SanitizedPrompt.Should().Contain("<PAN_1>");
|
||||||
|
result.SanitizedPrompt.Should().NotContainAny(
|
||||||
|
"ராஜேஷ்",
|
||||||
|
"ravi.kumar@gmail.com",
|
||||||
|
"9876543210",
|
||||||
|
"LN-456789",
|
||||||
|
"ABCDE1234F");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void Sanitize_CleanTamilQuestion_PassesThroughWithoutPersonRedaction()
|
||||||
|
{
|
||||||
|
const string prompt = "பணத்தை திரும்பப் பெறுவது எப்படி?";
|
||||||
|
|
||||||
|
var result = _sanitizer.Sanitize(new SanitizationRequest(prompt));
|
||||||
|
|
||||||
|
result.SanitizedPrompt.Should().Be(prompt);
|
||||||
|
result.DetectedEntities.Should().BeEmpty();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,6 +25,9 @@
|
|||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Compile Include="..\TestSupport.Shared\RealNerModelPaths.cs" Link="TestSupport.Shared\RealNerModelPaths.cs" />
|
<Compile Include="..\TestSupport.Shared\RealNerModelPaths.cs" Link="TestSupport.Shared\RealNerModelPaths.cs" />
|
||||||
<Compile Include="..\TestSupport.Shared\RealNerModelFixture.cs" Link="TestSupport.Shared\RealNerModelFixture.cs" />
|
<Compile Include="..\TestSupport.Shared\RealNerModelFixture.cs" Link="TestSupport.Shared\RealNerModelFixture.cs" />
|
||||||
|
<Compile Include="..\TestSupport.Shared\RealTamilNerModelPaths.cs" Link="TestSupport.Shared\RealTamilNerModelPaths.cs" />
|
||||||
|
<Compile Include="..\TestSupport.Shared\RealTamilNerModelFixture.cs" Link="TestSupport.Shared\RealTamilNerModelFixture.cs" />
|
||||||
|
<Compile Include="..\TestSupport.Shared\RealRoutingNerModelFixture.cs" Link="TestSupport.Shared\RealRoutingNerModelFixture.cs" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
@@ -9,11 +9,13 @@ public sealed class FakeOnnxNerModelRunner : IOnnxNerModelRunner
|
|||||||
|
|
||||||
public IReadOnlyList<PiiEntity> EntitiesToReturn { get; set; } = [];
|
public IReadOnlyList<PiiEntity> EntitiesToReturn { get; set; } = [];
|
||||||
|
|
||||||
|
public IReadOnlyList<NerModelOrigin> InvokedModelsToReturn { get; set; } = [NerModelOrigin.English];
|
||||||
|
|
||||||
public string? LastPredictedText { get; private set; }
|
public string? LastPredictedText { get; private set; }
|
||||||
|
|
||||||
public IReadOnlyList<PiiEntity> PredictEntities(string text)
|
public NerPredictionResult PredictEntities(string text)
|
||||||
{
|
{
|
||||||
LastPredictedText = text;
|
LastPredictedText = text;
|
||||||
return EntitiesToReturn;
|
return new NerPredictionResult(EntitiesToReturn, InvokedModelsToReturn);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,11 @@
|
|||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
using PiiRedaction.Core.Abstractions;
|
using PiiRedaction.Core.Abstractions;
|
||||||
|
using PiiRedaction.Core.Configuration;
|
||||||
using PiiRedaction.Core.Detection;
|
using PiiRedaction.Core.Detection;
|
||||||
using PiiRedaction.Core.Redaction;
|
using PiiRedaction.Core.Redaction;
|
||||||
using PiiRedaction.Core.Sanitization;
|
using PiiRedaction.Core.Sanitization;
|
||||||
|
using PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
namespace PiiRedaction.Core.Tests.TestSupport;
|
namespace PiiRedaction.Core.Tests.TestSupport;
|
||||||
|
|
||||||
@@ -10,6 +14,16 @@ public static class ProductionPipelineFactory
|
|||||||
public static IPromptSanitizer CreateWithRealModel(IOnnxNerModelRunner runner) =>
|
public static IPromptSanitizer CreateWithRealModel(IOnnxNerModelRunner runner) =>
|
||||||
new PromptSanitizer(CreateCompositeDetector(runner), new PlaceholderPiiRedactor());
|
new PromptSanitizer(CreateCompositeDetector(runner), new PlaceholderPiiRedactor());
|
||||||
|
|
||||||
|
public static IPromptSanitizer CreateWithRoutingRealModels(
|
||||||
|
EnglishOnnxNerRunner englishRunner,
|
||||||
|
TamilOnnxNerRunner tamilRunner,
|
||||||
|
IOptions<PiiRedactionOptions>? options = null) =>
|
||||||
|
CreateWithRealModel(new RoutingOnnxNerModelRunner(
|
||||||
|
englishRunner,
|
||||||
|
tamilRunner,
|
||||||
|
options ?? Options.Create(new PiiRedactionOptions { EnableTamilNer = true }),
|
||||||
|
NullLogger<RoutingOnnxNerModelRunner>.Instance));
|
||||||
|
|
||||||
public static IPiiDetector CreateCompositeDetector(IOnnxNerModelRunner runner) =>
|
public static IPiiDetector CreateCompositeDetector(IOnnxNerModelRunner runner) =>
|
||||||
new CompositePiiDetector(
|
new CompositePiiDetector(
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
using NUnit.Framework;
|
||||||
|
using PiiRedaction.Core.Models;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Core.Tests.TestSupport;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Golden end-to-end scenarios for Tamil script, Tanglish, and mixed-script prompts.
|
||||||
|
/// Uses fake NER spans for person names; regex and domain rules run for real.
|
||||||
|
/// </summary>
|
||||||
|
public static class TamilPromptScenarioCatalog
|
||||||
|
{
|
||||||
|
public static IEnumerable<TestCaseData> AllScenarios()
|
||||||
|
{
|
||||||
|
foreach (var scenario in BuildScenarios())
|
||||||
|
{
|
||||||
|
yield return new TestCaseData(scenario).SetName(scenario.Name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static IEnumerable<PromptScenario> BuildScenarios()
|
||||||
|
{
|
||||||
|
yield return TamilCustomerNameOnly();
|
||||||
|
yield return TamilWithPhonePan();
|
||||||
|
yield return TanglishCustomer();
|
||||||
|
yield return MixedTamilEnglish();
|
||||||
|
yield return TamilFullFinancial();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static PromptScenario TamilCustomerNameOnly() => new(
|
||||||
|
"Tamil_CustomerNameOnly",
|
||||||
|
"வாடிக்கையாளர் ராஜேஷ் குமார் சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்.",
|
||||||
|
"வாடிக்கையாளர் <PERSON_1> சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்.",
|
||||||
|
[PiiEntityType.Person],
|
||||||
|
["ராஜேஷ் குமார்"]);
|
||||||
|
|
||||||
|
private static PromptScenario TamilWithPhonePan() => new(
|
||||||
|
"Tamil_WithPhonePan",
|
||||||
|
"வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210 PAN ABCDE1234F.",
|
||||||
|
"வாடிக்கையாளர் <PERSON_1> தொலைபேசி <PHONE_1> PAN <PAN_1>.",
|
||||||
|
[PiiEntityType.Person, PiiEntityType.Phone, PiiEntityType.Pan],
|
||||||
|
["ராஜேஷ் குமார்", "9876543210", "ABCDE1234F"]);
|
||||||
|
|
||||||
|
private static PromptScenario TanglishCustomer() => new(
|
||||||
|
"Tanglish_CustomerPhone",
|
||||||
|
"Customer Senthil phone 9876543210 reported a failed UPI transfer.",
|
||||||
|
"Customer <PERSON_1> phone <PHONE_1> reported a failed UPI transfer.",
|
||||||
|
[PiiEntityType.Person, PiiEntityType.Phone],
|
||||||
|
["Senthil", "9876543210"]);
|
||||||
|
|
||||||
|
private static PromptScenario MixedTamilEnglish() => new(
|
||||||
|
"Mixed_TamilEnglish",
|
||||||
|
"வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge.",
|
||||||
|
"வாடிக்கையாளர் <PERSON_1> phone <PHONE_1> disputed the charge.",
|
||||||
|
[PiiEntityType.Person, PiiEntityType.Phone],
|
||||||
|
["Ravi Kumar", "9876543210"]);
|
||||||
|
|
||||||
|
private static PromptScenario TamilFullFinancial() => new(
|
||||||
|
"Tamil_FullFinancial",
|
||||||
|
"வாடிக்கையாளர் ராஜேஷ் குமார் மின்னஞ்சல் ravi.kumar@gmail.com தொலைபேசி 9876543210 LoanNumber LN-456789 PAN ABCDE1234F. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்.",
|
||||||
|
"வாடிக்கையாளர் <PERSON_1> மின்னஞ்சல் <EMAIL_1> தொலைபேசி <PHONE_1> LoanNumber <LOAN_NUMBER_1> PAN <PAN_1>. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்.",
|
||||||
|
[PiiEntityType.Person, PiiEntityType.Email, PiiEntityType.Phone, PiiEntityType.LoanNumber, PiiEntityType.Pan],
|
||||||
|
["ராஜேஷ் குமார்", "ravi.kumar@gmail.com", "9876543210", "LN-456789", "ABCDE1234F"]);
|
||||||
|
}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
using FluentAssertions;
|
||||||
|
using PiiRedaction.Core.Detection;
|
||||||
|
using PiiRedaction.Core.Models;
|
||||||
|
using PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Infrastructure.Tests.Onnx;
|
||||||
|
|
||||||
|
[TestFixture]
|
||||||
|
public sealed class NerLabelConfigTests
|
||||||
|
{
|
||||||
|
[TestCase("B-PER", true)]
|
||||||
|
[TestCase("I-PER", true)]
|
||||||
|
[TestCase("B-PERSON", true)]
|
||||||
|
[TestCase("B-ORG", false)]
|
||||||
|
public void English_IsPersonLabel_MatchesExpected(string label, bool expected)
|
||||||
|
{
|
||||||
|
NerLabelConfig.English.IsPersonLabel(label).Should().Be(expected);
|
||||||
|
}
|
||||||
|
|
||||||
|
[TestCase("B-person-politician", true)]
|
||||||
|
[TestCase("I-person-artist", true)]
|
||||||
|
[TestCase("B-location", false)]
|
||||||
|
[TestCase("O", false)]
|
||||||
|
public void Tamil_IsPersonLabel_MatchesFineGrainedTags(string label, bool expected)
|
||||||
|
{
|
||||||
|
NerLabelConfig.Tamil.IsPersonLabel(label).Should().Be(expected);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,7 +25,7 @@ public sealed class OnnxNerModelRunnerTests
|
|||||||
var path = Path.Combine(Path.GetTempPath(), $"missing-ner-{Guid.NewGuid():N}.onnx");
|
var path = Path.Combine(Path.GetTempPath(), $"missing-ner-{Guid.NewGuid():N}.onnx");
|
||||||
using var runner = CreateRunner(path);
|
using var runner = CreateRunner(path);
|
||||||
|
|
||||||
runner.PredictEntities("Customer Ravi Kumar").Should().BeEmpty();
|
runner.PredictEntities("Customer Ravi Kumar").Entities.Should().BeEmpty();
|
||||||
}
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
|
|||||||
@@ -26,16 +26,13 @@ public sealed class RealNerModelRunnerTests : RealNerModelFixture
|
|||||||
|
|
||||||
{
|
{
|
||||||
|
|
||||||
var entities = Runner.PredictEntities(prompt);
|
var result = Runner.PredictEntities(prompt);
|
||||||
|
var entities = result.Entities;
|
||||||
|
|
||||||
|
|
||||||
entities.Should().Contain(entity =>
|
entities.Should().Contain(entity =>
|
||||||
|
|
||||||
entity.Type == PiiEntityType.Person &&
|
entity.Type == PiiEntityType.Person &&
|
||||||
|
|
||||||
entity.Source == PiiDetectionSource.Ner &&
|
entity.Source == PiiDetectionSource.Ner &&
|
||||||
|
entity.ModelOrigin == NerModelOrigin.English &&
|
||||||
entity.Value.Contains(expectedNamePart, StringComparison.Ordinal) &&
|
entity.Value.Contains(expectedNamePart, StringComparison.Ordinal) &&
|
||||||
|
|
||||||
prompt.AsSpan(entity.StartIndex, entity.Length).ToString() == entity.Value);
|
prompt.AsSpan(entity.StartIndex, entity.Length).ToString() == entity.Value);
|
||||||
@@ -43,9 +40,6 @@ public sealed class RealNerModelRunnerTests : RealNerModelFixture
|
|||||||
|
|
||||||
|
|
||||||
entities.Should().Contain(entity => entity.Value == expectedValue);
|
entities.Should().Contain(entity => entity.Value == expectedValue);
|
||||||
|
result.InvokedModels.Should().Equal(NerModelOrigin.English);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
using FluentAssertions;
|
||||||
|
using PiiRedaction.Core.Models;
|
||||||
|
using PiiRedaction.Infrastructure.Onnx;
|
||||||
|
using PiiRedaction.Tests.Shared;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Infrastructure.Tests.Onnx;
|
||||||
|
|
||||||
|
[TestFixture]
|
||||||
|
[Category("TamilNer")]
|
||||||
|
public sealed class RealTamilNerModelRunnerTests : RealTamilNerModelFixture
|
||||||
|
{
|
||||||
|
[Test]
|
||||||
|
public void IsModelAvailable_LoadsOnnxAndTokenizer()
|
||||||
|
{
|
||||||
|
Runner.IsModelAvailable.Should().BeTrue();
|
||||||
|
}
|
||||||
|
|
||||||
|
[TestCase("வாடிக்கையாளர் ராஜேஷ் குமார் அழைத்தார்.", "ராஜேஷ்", "ராஜேஷ் குமார்")]
|
||||||
|
public void PredictEntities_TamilScript_DetectsPersonEntity(
|
||||||
|
string prompt,
|
||||||
|
string expectedNamePart,
|
||||||
|
string expectedValue)
|
||||||
|
{
|
||||||
|
var result = Runner.PredictEntities(prompt);
|
||||||
|
var entities = result.Entities;
|
||||||
|
|
||||||
|
entities.Should().Contain(entity =>
|
||||||
|
entity.Type == PiiEntityType.Person &&
|
||||||
|
entity.Source == PiiDetectionSource.Ner &&
|
||||||
|
entity.ModelOrigin == NerModelOrigin.Tamil &&
|
||||||
|
entity.Value.Contains(expectedNamePart, StringComparison.Ordinal) &&
|
||||||
|
prompt.AsSpan(entity.StartIndex, entity.Length).ToString() == entity.Value);
|
||||||
|
|
||||||
|
entities.Should().Contain(entity => entity.Value == expectedValue);
|
||||||
|
result.InvokedModels.Should().Equal(NerModelOrigin.Tamil);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void PredictEntities_TamilWithPhone_DetectsPersonAndLeavesPhoneToRegex()
|
||||||
|
{
|
||||||
|
const string prompt = "வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210";
|
||||||
|
|
||||||
|
var entities = Runner.PredictEntities(prompt).Entities;
|
||||||
|
|
||||||
|
entities.Should().Contain(entity =>
|
||||||
|
entity.Type == PiiEntityType.Person &&
|
||||||
|
entity.Source == PiiDetectionSource.Ner &&
|
||||||
|
entity.ModelOrigin == NerModelOrigin.Tamil &&
|
||||||
|
entity.Value.Contains("ராஜேஷ்", StringComparison.Ordinal));
|
||||||
|
entities.Should().NotContain(entity => entity.Type == PiiEntityType.Phone);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void PredictEntities_CleanTamilQuestion_ReturnsNoEntities()
|
||||||
|
{
|
||||||
|
const string prompt = "பணத்தை திரும்பப் பெறுவது எப்படி?";
|
||||||
|
|
||||||
|
Runner.PredictEntities(prompt).Entities.Should().BeEmpty();
|
||||||
|
}
|
||||||
|
|
||||||
|
[TestCase("Customer Senthil phone 9876543210", "Senthil")]
|
||||||
|
public void PredictEntities_TanglishLatinScript_DoesNotInvokeTamilRunner(
|
||||||
|
string prompt,
|
||||||
|
string expectedNamePart)
|
||||||
|
{
|
||||||
|
// TamilOnnxNerRunner is script-scoped; Tanglish is handled by English routing in pipeline tests.
|
||||||
|
// Direct Tamil runner on Latin-only text should not emit person spans.
|
||||||
|
var entities = Runner.PredictEntities(prompt).Entities;
|
||||||
|
|
||||||
|
entities.Should().NotContain(entity =>
|
||||||
|
entity.Type == PiiEntityType.Person &&
|
||||||
|
entity.Value.Contains(expectedNamePart, StringComparison.OrdinalIgnoreCase));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,143 @@
|
|||||||
|
using FluentAssertions;
|
||||||
|
using PiiRedaction.Core.Detection;
|
||||||
|
using PiiRedaction.Core.Models;
|
||||||
|
using PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Infrastructure.Tests.Onnx;
|
||||||
|
|
||||||
|
[TestFixture]
|
||||||
|
public sealed class RoutingOnnxNerModelRunnerTests
|
||||||
|
{
|
||||||
|
[Test]
|
||||||
|
public void PredictEntities_LatinOnly_UsesEnglishRunnerOnly()
|
||||||
|
{
|
||||||
|
var english = new FakeLanguageNerRunner("Ravi Kumar", NerModelOrigin.English);
|
||||||
|
var tamil = new FakeLanguageNerRunner("தமிழ் பெயர்", NerModelOrigin.Tamil);
|
||||||
|
var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
|
||||||
|
|
||||||
|
var result = router.PredictEntities("Customer Ravi Kumar called.");
|
||||||
|
|
||||||
|
result.Entities.Should().ContainSingle(entity =>
|
||||||
|
entity.Value == "Ravi Kumar" && entity.ModelOrigin == NerModelOrigin.English);
|
||||||
|
result.InvokedModels.Should().Equal(NerModelOrigin.English);
|
||||||
|
english.CallCount.Should().Be(1);
|
||||||
|
tamil.CallCount.Should().Be(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void PredictEntities_TamilOnly_UsesTamilRunnerOnly()
|
||||||
|
{
|
||||||
|
var english = new FakeLanguageNerRunner("Ravi Kumar", NerModelOrigin.English);
|
||||||
|
var tamil = new FakeLanguageNerRunner("ராஜேஷ்", NerModelOrigin.Tamil);
|
||||||
|
var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
|
||||||
|
|
||||||
|
var result = router.PredictEntities("வாடிக்கையாளர் ராஜேஷ்");
|
||||||
|
|
||||||
|
result.Entities.Should().ContainSingle(entity =>
|
||||||
|
entity.Value == "ராஜேஷ்" && entity.ModelOrigin == NerModelOrigin.Tamil);
|
||||||
|
result.InvokedModels.Should().Equal(NerModelOrigin.Tamil);
|
||||||
|
english.CallCount.Should().Be(0);
|
||||||
|
tamil.CallCount.Should().Be(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void PredictEntities_Mixed_InvokesBothRunnersAndTagsOrigins()
|
||||||
|
{
|
||||||
|
var english = new FakeLanguageNerRunner("Priya", NerModelOrigin.English);
|
||||||
|
var tamil = new FakeLanguageNerRunner("மற்றும்", NerModelOrigin.Tamil);
|
||||||
|
var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
|
||||||
|
|
||||||
|
var result = router.PredictEntities("Rajesh மற்றும் Priya");
|
||||||
|
|
||||||
|
english.CallCount.Should().Be(1);
|
||||||
|
tamil.CallCount.Should().Be(1);
|
||||||
|
result.InvokedModels.Should().Equal(NerModelOrigin.English, NerModelOrigin.Tamil);
|
||||||
|
result.Entities.Should().Contain(entity => entity.ModelOrigin == NerModelOrigin.English);
|
||||||
|
result.Entities.Should().Contain(entity => entity.ModelOrigin == NerModelOrigin.Tamil);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void PredictEntities_NoLetters_InvokesNeither()
|
||||||
|
{
|
||||||
|
var english = new FakeLanguageNerRunner("ignored", NerModelOrigin.English);
|
||||||
|
var tamil = new FakeLanguageNerRunner("ignored", NerModelOrigin.Tamil);
|
||||||
|
var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: true);
|
||||||
|
|
||||||
|
var result = router.PredictEntities("9876543210");
|
||||||
|
|
||||||
|
result.Entities.Should().BeEmpty();
|
||||||
|
result.InvokedModels.Should().BeEmpty();
|
||||||
|
english.CallCount.Should().Be(0);
|
||||||
|
tamil.CallCount.Should().Be(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void PredictEntities_TamilDisabled_SkipsTamilRunnerForMixedText()
|
||||||
|
{
|
||||||
|
var english = new FakeLanguageNerRunner("EnglishName", NerModelOrigin.English);
|
||||||
|
var tamil = new FakeLanguageNerRunner("தமிழ்", NerModelOrigin.Tamil);
|
||||||
|
var router = new RoutingOnnxNerModelRunner(english, tamil, enableTamilNer: false);
|
||||||
|
|
||||||
|
var result = router.PredictEntities("Rajesh மற்றும் Priya");
|
||||||
|
|
||||||
|
english.CallCount.Should().Be(1);
|
||||||
|
tamil.CallCount.Should().Be(0);
|
||||||
|
result.InvokedModels.Should().Equal(NerModelOrigin.English);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void MergePersonSpans_PrefersLongerOverlappingSpan()
|
||||||
|
{
|
||||||
|
var entities = new[]
|
||||||
|
{
|
||||||
|
CreatePerson("Raj", 0, 3, NerModelOrigin.English),
|
||||||
|
CreatePerson("Rajesh", 0, 6, NerModelOrigin.Tamil)
|
||||||
|
};
|
||||||
|
|
||||||
|
var merged = RoutingOnnxNerModelRunner.MergePersonSpans(entities);
|
||||||
|
|
||||||
|
merged.Should().ContainSingle(entity =>
|
||||||
|
entity.Value == "Rajesh" && entity.ModelOrigin == NerModelOrigin.Tamil);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static PiiEntity CreatePerson(string value, int start, int length, NerModelOrigin origin) =>
|
||||||
|
new(PiiEntityType.Person, value, start, length, PiiDetectionSource.Ner, ModelOrigin: origin);
|
||||||
|
|
||||||
|
private sealed class FakeLanguageNerRunner : IOnnxNerModelRunner
|
||||||
|
{
|
||||||
|
private readonly string _personValue;
|
||||||
|
private readonly NerModelOrigin _origin;
|
||||||
|
|
||||||
|
public FakeLanguageNerRunner(string personValue, NerModelOrigin origin)
|
||||||
|
{
|
||||||
|
_personValue = personValue;
|
||||||
|
_origin = origin;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int CallCount { get; private set; }
|
||||||
|
|
||||||
|
public bool IsModelAvailable => true;
|
||||||
|
|
||||||
|
public NerPredictionResult PredictEntities(string text)
|
||||||
|
{
|
||||||
|
CallCount++;
|
||||||
|
var start = text.IndexOf(_personValue, StringComparison.Ordinal);
|
||||||
|
if (start < 0)
|
||||||
|
{
|
||||||
|
start = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
return new NerPredictionResult(
|
||||||
|
[
|
||||||
|
new PiiEntity(
|
||||||
|
PiiEntityType.Person,
|
||||||
|
_personValue,
|
||||||
|
start,
|
||||||
|
_personValue.Length,
|
||||||
|
PiiDetectionSource.Ner,
|
||||||
|
ModelOrigin: _origin)
|
||||||
|
],
|
||||||
|
[_origin]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
using FluentAssertions;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Infrastructure.Tests.Onnx;
|
||||||
|
|
||||||
|
[TestFixture]
|
||||||
|
public sealed class TokenClassifierEncoderFactoryTests
|
||||||
|
{
|
||||||
|
[Test]
|
||||||
|
public void Create_PrefersWordPieceWhenVocabExists()
|
||||||
|
{
|
||||||
|
var modelDirectory = ResolveTamilModelDirectory();
|
||||||
|
if (!File.Exists(Path.Combine(modelDirectory, "vocab.txt")))
|
||||||
|
{
|
||||||
|
Assert.Ignore("Tamil vocab.txt not found.");
|
||||||
|
}
|
||||||
|
|
||||||
|
var encoder = TokenClassifierEncoderFactory.Create(
|
||||||
|
modelDirectory,
|
||||||
|
NullLogger.Instance);
|
||||||
|
|
||||||
|
encoder.Should().BeOfType<BertWordPieceEncoder>();
|
||||||
|
encoder.IsAvailable.Should().BeTrue();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string ResolveTamilModelDirectory()
|
||||||
|
{
|
||||||
|
var directory = new DirectoryInfo(AppContext.BaseDirectory);
|
||||||
|
while (directory is not null)
|
||||||
|
{
|
||||||
|
var candidate = Path.Combine(directory.FullName, "models", "ta");
|
||||||
|
if (Directory.Exists(candidate))
|
||||||
|
{
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
|
||||||
|
directory = directory.Parent;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Path.Combine(Environment.CurrentDirectory, "models", "ta");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,6 +24,8 @@
|
|||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Compile Include="..\TestSupport.Shared\RealNerModelPaths.cs" Link="TestSupport.Shared\RealNerModelPaths.cs" />
|
<Compile Include="..\TestSupport.Shared\RealNerModelPaths.cs" Link="TestSupport.Shared\RealNerModelPaths.cs" />
|
||||||
<Compile Include="..\TestSupport.Shared\RealNerModelFixture.cs" Link="TestSupport.Shared\RealNerModelFixture.cs" />
|
<Compile Include="..\TestSupport.Shared\RealNerModelFixture.cs" Link="TestSupport.Shared\RealNerModelFixture.cs" />
|
||||||
|
<Compile Include="..\TestSupport.Shared\RealTamilNerModelPaths.cs" Link="TestSupport.Shared\RealTamilNerModelPaths.cs" />
|
||||||
|
<Compile Include="..\TestSupport.Shared\RealTamilNerModelFixture.cs" Link="TestSupport.Shared\RealTamilNerModelFixture.cs" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
@@ -6,12 +6,12 @@ using PiiRedaction.Infrastructure.Onnx;
|
|||||||
namespace PiiRedaction.Tests.Shared;
|
namespace PiiRedaction.Tests.Shared;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reuses a single <see cref="OnnxNerModelRunner"/> per fixture for performance.
|
/// Reuses a single <see cref="EnglishOnnxNerRunner"/> per fixture for performance.
|
||||||
/// Skips all tests in the class when the ONNX model is missing or cannot be loaded.
|
/// Skips all tests in the class when the ONNX model is missing or cannot be loaded.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public abstract class RealNerModelFixture
|
public abstract class RealNerModelFixture
|
||||||
{
|
{
|
||||||
protected OnnxNerModelRunner Runner { get; private set; } = null!;
|
protected EnglishOnnxNerRunner Runner { get; private set; } = null!;
|
||||||
|
|
||||||
protected string ModelPath { get; private set; } = null!;
|
protected string ModelPath { get; private set; } = null!;
|
||||||
|
|
||||||
@@ -24,8 +24,12 @@ public abstract class RealNerModelFixture
|
|||||||
Assert.Ignore(RealNerModelPaths.ModelMissingMessage);
|
Assert.Ignore(RealNerModelPaths.ModelMissingMessage);
|
||||||
}
|
}
|
||||||
|
|
||||||
var options = Options.Create(new PiiRedactionOptions { OnnxModelPath = ModelPath });
|
var options = Options.Create(new PiiRedactionOptions
|
||||||
Runner = new OnnxNerModelRunner(options, NullLogger<OnnxNerModelRunner>.Instance);
|
{
|
||||||
|
EnglishOnnxModelPath = ModelPath,
|
||||||
|
OnnxModelPath = ModelPath
|
||||||
|
});
|
||||||
|
Runner = new EnglishOnnxNerRunner(options, NullLogger<EnglishOnnxNerRunner>.Instance);
|
||||||
|
|
||||||
if (!Runner.IsModelAvailable)
|
if (!Runner.IsModelAvailable)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -6,11 +6,17 @@ public static class RealNerModelPaths
|
|||||||
"ONNX model not found. Run scripts/download-ner-model.ps1 from the repository root.";
|
"ONNX model not found. Run scripts/download-ner-model.ps1 from the repository root.";
|
||||||
|
|
||||||
public static string ResolveRepoModelPath()
|
public static string ResolveRepoModelPath()
|
||||||
|
{
|
||||||
|
foreach (var relativePath in new[]
|
||||||
|
{
|
||||||
|
Path.Combine("models", "en", "ner-model.onnx"),
|
||||||
|
Path.Combine("models", "ner-model.onnx")
|
||||||
|
})
|
||||||
{
|
{
|
||||||
var directory = new DirectoryInfo(AppContext.BaseDirectory);
|
var directory = new DirectoryInfo(AppContext.BaseDirectory);
|
||||||
while (directory is not null)
|
while (directory is not null)
|
||||||
{
|
{
|
||||||
var candidate = Path.Combine(directory.FullName, "models", "ner-model.onnx");
|
var candidate = Path.Combine(directory.FullName, relativePath);
|
||||||
if (File.Exists(candidate))
|
if (File.Exists(candidate))
|
||||||
{
|
{
|
||||||
return candidate;
|
return candidate;
|
||||||
@@ -18,6 +24,7 @@ public static class RealNerModelPaths
|
|||||||
|
|
||||||
directory = directory.Parent;
|
directory = directory.Parent;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return Path.Combine(Environment.CurrentDirectory, "models", "ner-model.onnx");
|
return Path.Combine(Environment.CurrentDirectory, "models", "ner-model.onnx");
|
||||||
}
|
}
|
||||||
|
|||||||
69
tests/TestSupport.Shared/RealRoutingNerModelFixture.cs
Normal file
69
tests/TestSupport.Shared/RealRoutingNerModelFixture.cs
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
using PiiRedaction.Core.Configuration;
|
||||||
|
using PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Tests.Shared;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Loads English and Tamil ONNX runners and exposes a <see cref="RoutingOnnxNerModelRunner"/>.
|
||||||
|
/// Skips when either model is missing or cannot be loaded.
|
||||||
|
/// </summary>
|
||||||
|
public abstract class RealRoutingNerModelFixture
|
||||||
|
{
|
||||||
|
protected RoutingOnnxNerModelRunner Runner { get; private set; } = null!;
|
||||||
|
|
||||||
|
protected EnglishOnnxNerRunner EnglishRunner { get; private set; } = null!;
|
||||||
|
|
||||||
|
protected TamilOnnxNerRunner TamilRunner { get; private set; } = null!;
|
||||||
|
|
||||||
|
[OneTimeSetUp]
|
||||||
|
public void OneTimeSetUpRoutingModels()
|
||||||
|
{
|
||||||
|
var englishPath = RealNerModelPaths.ResolveRepoModelPath();
|
||||||
|
if (!File.Exists(englishPath))
|
||||||
|
{
|
||||||
|
Assert.Ignore(RealNerModelPaths.ModelMissingMessage);
|
||||||
|
}
|
||||||
|
|
||||||
|
var tamilPath = RealTamilNerModelPaths.ResolveRepoModelPath();
|
||||||
|
if (!File.Exists(tamilPath))
|
||||||
|
{
|
||||||
|
Assert.Ignore(RealTamilNerModelPaths.ModelMissingMessage);
|
||||||
|
}
|
||||||
|
|
||||||
|
var options = Options.Create(new PiiRedactionOptions
|
||||||
|
{
|
||||||
|
EnglishOnnxModelPath = englishPath,
|
||||||
|
OnnxModelPath = englishPath,
|
||||||
|
TamilOnnxModelPath = tamilPath,
|
||||||
|
EnableTamilNer = true
|
||||||
|
});
|
||||||
|
|
||||||
|
EnglishRunner = new EnglishOnnxNerRunner(options, NullLogger<EnglishOnnxNerRunner>.Instance);
|
||||||
|
TamilRunner = new TamilOnnxNerRunner(options, NullLogger<TamilOnnxNerRunner>.Instance);
|
||||||
|
|
||||||
|
if (!EnglishRunner.IsModelAvailable)
|
||||||
|
{
|
||||||
|
EnglishRunner.Dispose();
|
||||||
|
TamilRunner.Dispose();
|
||||||
|
Assert.Ignore(RealNerModelPaths.ModelMissingMessage);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!TamilRunner.IsModelAvailable)
|
||||||
|
{
|
||||||
|
EnglishRunner.Dispose();
|
||||||
|
TamilRunner.Dispose();
|
||||||
|
Assert.Ignore(RealTamilNerModelPaths.ModelMissingMessage);
|
||||||
|
}
|
||||||
|
|
||||||
|
Runner = new RoutingOnnxNerModelRunner(EnglishRunner, TamilRunner, options, NullLogger<RoutingOnnxNerModelRunner>.Instance);
|
||||||
|
}
|
||||||
|
|
||||||
|
[OneTimeTearDown]
|
||||||
|
public void OneTimeTearDownRoutingModels()
|
||||||
|
{
|
||||||
|
EnglishRunner?.Dispose();
|
||||||
|
TamilRunner?.Dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
45
tests/TestSupport.Shared/RealTamilNerModelFixture.cs
Normal file
45
tests/TestSupport.Shared/RealTamilNerModelFixture.cs
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
using PiiRedaction.Core.Configuration;
|
||||||
|
using PiiRedaction.Infrastructure.Onnx;
|
||||||
|
|
||||||
|
namespace PiiRedaction.Tests.Shared;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Reuses a single <see cref="TamilOnnxNerRunner"/> per fixture for performance.
|
||||||
|
/// Skips all tests in the class when the Tamil ONNX model is missing or cannot be loaded.
|
||||||
|
/// </summary>
|
||||||
|
public abstract class RealTamilNerModelFixture
|
||||||
|
{
|
||||||
|
protected TamilOnnxNerRunner Runner { get; private set; } = null!;
|
||||||
|
|
||||||
|
protected string ModelPath { get; private set; } = null!;
|
||||||
|
|
||||||
|
[OneTimeSetUp]
|
||||||
|
public void OneTimeSetUpTamilModel()
|
||||||
|
{
|
||||||
|
ModelPath = RealTamilNerModelPaths.ResolveRepoModelPath();
|
||||||
|
if (!File.Exists(ModelPath))
|
||||||
|
{
|
||||||
|
Assert.Ignore(RealTamilNerModelPaths.ModelMissingMessage);
|
||||||
|
}
|
||||||
|
|
||||||
|
var options = Options.Create(new PiiRedactionOptions
|
||||||
|
{
|
||||||
|
TamilOnnxModelPath = ModelPath
|
||||||
|
});
|
||||||
|
Runner = new TamilOnnxNerRunner(options, NullLogger<TamilOnnxNerRunner>.Instance);
|
||||||
|
|
||||||
|
if (!Runner.IsModelAvailable)
|
||||||
|
{
|
||||||
|
Runner.Dispose();
|
||||||
|
Assert.Ignore(RealTamilNerModelPaths.ModelMissingMessage);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[OneTimeTearDown]
|
||||||
|
public void OneTimeTearDownTamilModel()
|
||||||
|
{
|
||||||
|
Runner?.Dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
25
tests/TestSupport.Shared/RealTamilNerModelPaths.cs
Normal file
25
tests/TestSupport.Shared/RealTamilNerModelPaths.cs
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
namespace PiiRedaction.Tests.Shared;
|
||||||
|
|
||||||
|
public static class RealTamilNerModelPaths
|
||||||
|
{
|
||||||
|
public const string ModelMissingMessage =
|
||||||
|
"Tamil ONNX model not found. Run scripts/download-tamil-ner-model.ps1 from the repository root.";
|
||||||
|
|
||||||
|
public static string ResolveRepoModelPath()
|
||||||
|
{
|
||||||
|
const string relativePath = "models/ta/model.onnx";
|
||||||
|
var directory = new DirectoryInfo(AppContext.BaseDirectory);
|
||||||
|
while (directory is not null)
|
||||||
|
{
|
||||||
|
var candidate = Path.Combine(directory.FullName, relativePath);
|
||||||
|
if (File.Exists(candidate))
|
||||||
|
{
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
|
||||||
|
directory = directory.Parent;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Path.Combine(Environment.CurrentDirectory, relativePath);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user