From cf8f5a7232cbc08acead80ee8c96038633d5a4f3 Mon Sep 17 00:00:00 2001 From: Bilal Nazer Ali Date: Tue, 7 Jul 2026 17:12:38 +0530 Subject: [PATCH] Add Tamil NER routing and WPF test harness for POC validation. Introduce dual-script ONNX NER routing (English/Tamil/mixed), Tamil console samples and integration tests, model download scripts, and a resizable WPF MVVM harness with click-to-load prompts, batch validation, and runtime-adjustable detection panels. --- .gitignore | 4 + PiiRedaction.slnx | 1 + README.md | 84 ++- docs/architecture.md | 166 +++++- docs/ner-models.md | 548 ++++++++++++++++++ docs/tamil-tanglish-ner-plan.md | 307 ++++++++++ models/en/.gitkeep | 0 models/ta/.gitkeep | 0 scripts/download-tamil-ner-model.ps1 | 229 ++++++++ scripts/download-tamil-ner-model.py | 148 +++++ scripts/tamil-ner-diagnostic/Program.cs | 27 + .../tamil-ner-diagnostic.csproj | 14 + .../ServiceCollectionExtensions.cs | 5 +- src/PiiRedaction.ConsoleApp/Program.cs | 4 + .../Samples/PromptDemoRunner.cs | 2 +- .../Samples/SamplePromptCatalog.cs | 30 + src/PiiRedaction.ConsoleApp/appsettings.json | 5 +- .../Configuration/PiiRedactionOptions.cs | 6 + .../Detection/ScriptComposition.cs | 9 + .../Detection/ScriptRouter.cs | 45 ++ .../Onnx/BertWordPieceEncoder.cs | 107 ++++ .../Onnx/EnglishOnnxNerRunner.cs | 35 ++ .../Onnx/ITokenClassifierEncoder.cs | 18 + .../Onnx/NerLabelConfig.cs | 26 + .../Onnx/OnnxAssetPathResolver.cs | 53 ++ .../Onnx/OnnxNerModelRunner.cs | 315 +--------- .../Onnx/OnnxTokenClassifierRunner.cs | 229 ++++++++ .../Onnx/RoutingOnnxNerModelRunner.cs | 105 ++++ .../Onnx/SentencePieceEncoder.cs | 102 ++++ .../Onnx/TamilOnnxNerRunner.cs | 27 + .../Onnx/TokenClassifierEncoderFactory.cs | 40 ++ .../PiiRedaction.Infrastructure.csproj | 4 + src/PiiRedaction.TestHarness.Wpf/App.xaml | 17 + src/PiiRedaction.TestHarness.Wpf/App.xaml.cs | 57 ++ .../Converters/ValueConverters.cs | 71 +++ .../ServiceCollectionExtensions.cs | 42 ++ .../MainWindow.xaml | 311 ++++++++++ .../MainWindow.xaml.cs | 13 + .../Models/BatchRunResult.cs | 23 + .../Models/ModelStatusSnapshot.cs | 21 + .../Models/RedactionDisplayModel.cs | 29 + .../Models/TestPromptScenario.cs | 18 + .../PiiRedaction.TestHarness.Wpf.csproj | 33 ++ .../Resources/Styles.xaml | 68 +++ .../Services/IModelStatusService.cs | 8 + .../Services/IRedactionAppService.cs | 17 + .../Services/IScriptAnalysisService.cs | 8 + .../Services/ITestPromptCatalog.cs | 8 + .../Services/ModelStatusService.cs | 46 ++ .../Services/RedactionAppService.cs | 116 ++++ .../Services/ScriptAnalysisService.cs | 13 + .../Services/TestPromptCatalog.cs | 194 +++++++ .../ViewModels/MainViewModel.cs | 337 +++++++++++ .../ViewModels/TestPromptItemViewModel.cs | 25 + .../appsettings.json | 8 + .../Detection/ScriptRouterTests.cs | 54 ++ .../Integration/GoldenPromptTests.cs | 1 + .../Integration/RealTamilPipelineTests.cs | 106 ++++ .../PiiRedaction.Core.Tests.csproj | 3 + .../TestSupport/ProductionPipelineFactory.cs | 12 + .../TestSupport/TamilPromptScenarioCatalog.cs | 63 ++ .../Onnx/NerLabelConfigTests.cs | 28 + .../Onnx/RealTamilNerModelRunnerTests.cs | 70 +++ .../Onnx/RoutingOnnxNerModelRunnerTests.cs | 118 ++++ .../TokenClassifierEncoderFactoryTests.cs | 43 ++ .../PiiRedaction.Infrastructure.Tests.csproj | 2 + .../TestSupport.Shared/RealNerModelFixture.cs | 12 +- tests/TestSupport.Shared/RealNerModelPaths.cs | 21 +- .../RealRoutingNerModelFixture.cs | 69 +++ .../RealTamilNerModelFixture.cs | 45 ++ .../RealTamilNerModelPaths.cs | 25 + 71 files changed, 4494 insertions(+), 356 deletions(-) create mode 100644 docs/ner-models.md create mode 100644 docs/tamil-tanglish-ner-plan.md create mode 100644 models/en/.gitkeep create mode 100644 models/ta/.gitkeep create mode 100644 scripts/download-tamil-ner-model.ps1 create mode 100644 scripts/download-tamil-ner-model.py create mode 100644 scripts/tamil-ner-diagnostic/Program.cs create mode 100644 scripts/tamil-ner-diagnostic/tamil-ner-diagnostic.csproj create mode 100644 src/PiiRedaction.Core/Detection/ScriptComposition.cs create mode 100644 src/PiiRedaction.Core/Detection/ScriptRouter.cs create mode 100644 src/PiiRedaction.Infrastructure/Onnx/BertWordPieceEncoder.cs create mode 100644 src/PiiRedaction.Infrastructure/Onnx/EnglishOnnxNerRunner.cs create mode 100644 src/PiiRedaction.Infrastructure/Onnx/ITokenClassifierEncoder.cs create mode 100644 src/PiiRedaction.Infrastructure/Onnx/NerLabelConfig.cs create mode 100644 src/PiiRedaction.Infrastructure/Onnx/OnnxAssetPathResolver.cs create mode 100644 src/PiiRedaction.Infrastructure/Onnx/OnnxTokenClassifierRunner.cs create mode 100644 src/PiiRedaction.Infrastructure/Onnx/RoutingOnnxNerModelRunner.cs create mode 100644 src/PiiRedaction.Infrastructure/Onnx/SentencePieceEncoder.cs create mode 100644 src/PiiRedaction.Infrastructure/Onnx/TamilOnnxNerRunner.cs create mode 100644 src/PiiRedaction.Infrastructure/Onnx/TokenClassifierEncoderFactory.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/App.xaml create mode 100644 src/PiiRedaction.TestHarness.Wpf/App.xaml.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/Converters/ValueConverters.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/DependencyInjection/ServiceCollectionExtensions.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/MainWindow.xaml create mode 100644 src/PiiRedaction.TestHarness.Wpf/MainWindow.xaml.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/Models/BatchRunResult.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/Models/ModelStatusSnapshot.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/Models/RedactionDisplayModel.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/Models/TestPromptScenario.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/PiiRedaction.TestHarness.Wpf.csproj create mode 100644 src/PiiRedaction.TestHarness.Wpf/Resources/Styles.xaml create mode 100644 src/PiiRedaction.TestHarness.Wpf/Services/IModelStatusService.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/Services/IRedactionAppService.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/Services/IScriptAnalysisService.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/Services/ITestPromptCatalog.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/Services/ModelStatusService.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/Services/RedactionAppService.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/Services/ScriptAnalysisService.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/Services/TestPromptCatalog.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/ViewModels/MainViewModel.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/ViewModels/TestPromptItemViewModel.cs create mode 100644 src/PiiRedaction.TestHarness.Wpf/appsettings.json create mode 100644 tests/PiiRedaction.Core.Tests/Detection/ScriptRouterTests.cs create mode 100644 tests/PiiRedaction.Core.Tests/Integration/RealTamilPipelineTests.cs create mode 100644 tests/PiiRedaction.Core.Tests/TestSupport/TamilPromptScenarioCatalog.cs create mode 100644 tests/PiiRedaction.Infrastructure.Tests/Onnx/NerLabelConfigTests.cs create mode 100644 tests/PiiRedaction.Infrastructure.Tests/Onnx/RealTamilNerModelRunnerTests.cs create mode 100644 tests/PiiRedaction.Infrastructure.Tests/Onnx/RoutingOnnxNerModelRunnerTests.cs create mode 100644 tests/PiiRedaction.Infrastructure.Tests/Onnx/TokenClassifierEncoderFactoryTests.cs create mode 100644 tests/TestSupport.Shared/RealRoutingNerModelFixture.cs create mode 100644 tests/TestSupport.Shared/RealTamilNerModelFixture.cs create mode 100644 tests/TestSupport.Shared/RealTamilNerModelPaths.cs diff --git a/.gitignore b/.gitignore index 7d13b63..55f9311 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,11 @@ models/*.onnx models/vocab.txt models/ner-labels.txt models/*.json +models/en/* +models/ta/* !models/.gitkeep +!models/en/.gitkeep +!models/ta/.gitkeep ## IDE .idea/ diff --git a/PiiRedaction.slnx b/PiiRedaction.slnx index 1c9bace..eb3f0db 100644 --- a/PiiRedaction.slnx +++ b/PiiRedaction.slnx @@ -3,6 +3,7 @@ + diff --git a/README.md b/README.md index affcdfd..481fbd1 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,10 @@ Financial and customer-service prompts often contain regulated data (names, gove For solution design, data-flow diagrams, trust boundaries, and project responsibilities, see **[docs/architecture.md](docs/architecture.md)**. +For English and Tamil ONNX NER model IDs, assets, routing, and reproduction steps, see **[docs/ner-models.md](docs/ner-models.md)**. + +**Planned:** Tamil / Tanglish person-name support via dual ONNX NER routing — see **[docs/tamil-tanglish-ner-plan.md](docs/tamil-tanglish-ner-plan.md)**. + ## Why Three Detection Strategies? | Strategy | Used For | Rationale | @@ -31,15 +35,17 @@ The placeholder map (`` → original value) is kept **in-process** for ``` src/ -├── PiiRedaction.ConsoleApp/ # Presentation: input/output, DI bootstrap -├── PiiRedaction.Core/ # Business logic: detection, redaction, models -└── PiiRedaction.Infrastructure/ # Technical adapters: ONNX Runtime, mock LLM -models/ # Optional ONNX model files (gitignored) +├── PiiRedaction.ConsoleApp/ # Console demo: input/output, DI bootstrap +├── PiiRedaction.TestHarness.Wpf/ # WPF MVVM test harness for manual POC validation +├── PiiRedaction.Core/ # Business logic: detection, redaction, models +└── PiiRedaction.Infrastructure/ # Technical adapters: ONNX Runtime, mock LLM +models/ # Optional ONNX model files (gitignored) ``` | Project | Responsibility | |---------|----------------| | `PiiRedaction.ConsoleApp` | Read prompt, call sanitizer, display results, call LLM service | +| `PiiRedaction.TestHarness.Wpf` | Desktop test harness: preset prompts, redact UI, batch validation | | `PiiRedaction.Core` | PII detection abstractions, redaction, sanitization orchestration | | `PiiRedaction.Infrastructure` | ONNX model runner, `IChatClient` mock implementation | @@ -63,7 +69,7 @@ dotnet build dotnet run --project src/PiiRedaction.ConsoleApp ``` -By default the console app runs **11 curated sample prompts** covering NER/person names, regex identifiers, domain IDs, combined scenarios, and a clean no-PII ticket. +By default the console app runs **16 curated sample prompts** covering English and Tamil/Tanglish/mixed person names, regex identifiers, domain IDs, combined scenarios, and a clean no-PII ticket. No flags are required for Tamil samples — they run in the default batch alongside English. List available samples: @@ -78,6 +84,27 @@ dotnet run --project src/PiiRedaction.ConsoleApp -- --sample 2 dotnet run --project src/PiiRedaction.ConsoleApp -- --name MrTitlePerson ``` +### WPF Test Harness + +A desktop **MVVM** application for interactive POC validation with English and Tamil prompts. Requires **Windows** (`net10.0-windows`). + +**Prerequisites:** English and Tamil ONNX models downloaded (see [ONNX Model Setup](#onnx-model-setup)). + +```bash +dotnet run --project src/PiiRedaction.TestHarness.Wpf +``` + +**Workflow:** + +1. **Select a test prompt** from the left panel (grouped by language: English, Tamil, Mixed, Tanglish) or type your own prompt in the input box. +2. **Click a test prompt** in the left panel to load it into the input box (previous results are cleared automatically). +3. Click **Redact** to run the full detection pipeline. The status bar shows model availability, script composition (LatinOnly / TamilOnly / Mixed), and elapsed time. +4. Review **Sanitized Output**, detected entities, and the placeholder map in the right panel. A leak warning appears if any detected value remains in the sanitized text. +5. Optionally click **Send Mock LLM** to send only the sanitized prompt to the mock LLM. +6. Click **Run All** to execute all **22 curated scenarios** (16 console samples + 6 harness-only edge cases) and view pass/fail results in the batch panel. + +The harness uses the same DI registrations and `IPromptSanitizer` pipeline as the console app, with thin application services (`IRedactionAppService`, `ITestPromptCatalog`, `IScriptAnalysisService`, `IModelStatusService`) following SOLID principles. + Interactive mode (enter your own prompt): ```bash @@ -100,7 +127,12 @@ Samples are defined in [`SamplePromptCatalog.cs`](src/PiiRedaction.ConsoleApp/Sa | 7 | PersonWithEmailNoPhone | NER + Regex | `Customer Arjun Mehta` + email | | 8 | AllRegexTypes | Regex | email, phone, PAN, Aadhaar, card | | 9 | AllDomainIds | Domain | LN, CID, ACC | -| 10 | NoPiiCleanTicket | Negative | no redaction | +| 10 | TamilCustomerNameOnly | NER (Tamil) | `வாடிக்கையாளர் ராஜேஷ் குமார்` | +| 11 | TamilWithPhonePan | NER (Tamil) + Regex | Tamil person + phone + PAN | +| 12 | TanglishCustomer | NER (English/Tanglish) | `Customer Senthil` + phone | +| 13 | MixedTamilEnglish | NER (Mixed) | `வாடிக்கையாளர் Ravi Kumar` + phone | +| 14 | TamilFullFinancial | NER (Tamil) + Regex + Domain | Tamil canonical demo | +| 15 | NoPiiCleanTicket | Negative | no redaction | Person names are detected via **ONNX NER** using `dslim/bert-base-NER` (or a compatible token-classification export). A real model is **required** for person-name detection; there is no regex or heuristic fallback. @@ -147,36 +179,41 @@ Person-name detection requires a token-classification ONNX model and companion t | File | Purpose | |------|---------| -| `models/ner-model.onnx` | Exported NER model | -| `models/vocab.txt` | BERT WordPiece vocabulary | -| `models/ner-labels.txt` | One BIO label per line (`O`, `B-PER`, `I-PER`, etc.) | +| `models/en/ner-model.onnx` | English BERT NER model (or legacy `models/ner-model.onnx`) | +| `models/en/vocab.txt` | BERT WordPiece vocabulary | +| `models/en/ner-labels.txt` | One BIO label per line (`O`, `B-PER`, `I-PER`, etc.) | +| `models/ta/model.onnx` | Tamil IndicBERT NER model | +| `models/ta/sentencepiece.bpe.model` | SentencePiece tokenizer for Tamil model | +| `models/ta/ner-labels.txt` | Fine-grained Tamil NER labels | -### Download script +### Download scripts From the repository root: ```powershell .\scripts\download-ner-model.ps1 +.\scripts\download-tamil-ner-model.ps1 ``` Or with Python directly: ```bash python scripts/download-ner-model.py +python scripts/download-tamil-ner-model.py ``` -The script exports [`dslim/bert-base-NER`](https://huggingface.co/dslim/bert-base-NER) via Hugging Face Optimum when Python is available. Otherwise it downloads the pre-exported ONNX assets from Hugging Face directly. +The English script exports [`dslim/bert-base-NER`](https://huggingface.co/dslim/bert-base-NER) via Hugging Face Optimum when Python is available. The Tamil script exports [`prachuryyaIITG/SampurNER_Tamil_IndicBERTv2`](https://huggingface.co/prachuryyaIITG/SampurNER_Tamil_IndicBERTv2). Otherwise each script downloads pre-exported ONNX assets from Hugging Face directly. + +Set `EnableTamilNer` to `false` in `appsettings.json` to revert to English-only routing. ### Inference pipeline -`OnnxNerModelRunner` performs the full pipeline: +`RoutingOnnxNerModelRunner` classifies script composition and delegates to: -- BERT WordPiece tokenization (`Microsoft.ML.Tokenizers`) -- ONNX Runtime inference (`input_ids`, `attention_mask`, optional `token_type_ids`) -- BIO label decoding (`B-PER` / `I-PER` → `PiiEntityType.Person`) -- Character-span alignment back to the source text +- **`EnglishOnnxNerRunner`** — BERT WordPiece tokenization for Latin script and Tanglish +- **`TamilOnnxNerRunner`** — SentencePiece tokenization for Tamil script (U+0B80–U+0BFF) -When the model or tokenizer files are missing, person detection returns no results. +Both runners share `OnnxTokenClassifierRunner` for ONNX Runtime inference and BIO label decoding. Overlapping person spans from mixed-script prompts are merged (longer span wins). ## Swapping Mock LLM for Azure OpenAI @@ -251,24 +288,29 @@ The solution includes an **NUnit** test suite across two projects: dotnet test dotnet test --filter "FullyQualifiedName~GoldenPromptTests" dotnet test --filter "Category=RealModel" +dotnet test --filter "Category=TamilNer" dotnet test --logger "console;verbosity=detailed" ``` -Fast CI runs without the ONNX model: fake-based tests always execute; tests marked **`Category=RealModel`** are skipped when `models/ner-model.onnx` is absent. Download the model first: +Fast CI runs without the ONNX model: fake-based tests always execute; tests marked **`Category=RealModel`** or **`Category=TamilNer`** are skipped when the corresponding ONNX models are absent. Download models first: ```powershell .\scripts\download-ner-model.ps1 +.\scripts\download-tamil-ner-model.ps1 ``` ### Test architecture -- **`PromptScenarioCatalog`** — five focused end-to-end scenarios (canonical demo, multi-regex, duplicate people, overlap stress, no-PII negative) -- **`ProductionPipelineFactory`** — builds the same Domain → Regex → OnnxNer composite stack as production DI; `CreateWithRealModel(runner)` wires a real `OnnxNerModelRunner` +- **`PromptScenarioCatalog`** — five focused end-to-end English scenarios (canonical demo, multi-regex, duplicate people, overlap stress, no-PII negative) +- **`TamilPromptScenarioCatalog`** — five Tamil/Tanglish/mixed golden scenarios (fake NER for person spans) +- **`ProductionPipelineFactory`** — builds the same Domain → Regex → OnnxNer composite stack as production DI; `CreateWithRealModel(runner)` wires a real runner; `CreateWithRoutingRealModels` wires English + Tamil routing - **`FakeOnnxNerModelRunner`** — unit-test double for NER; golden tests inject person spans per scenario - **`GoldenPromptTests`** — end-to-end sanitization proof across the catalog (fake NER) - **`RealNerModelFixture`** — shared fixture that loads `models/ner-model.onnx` once per class; skips when model missing - **`RealNerModelRunnerTests`** — direct ONNX inference with span accuracy checks -- **`RealNerPipelineTests`** — full pipeline with real NER (canonical, multi-person, clean-ticket negative) +- **`RealNerPipelineTests`** — full pipeline with real English NER (canonical, multi-person, clean-ticket negative) +- **`RealTamilPipelineTests`** — full pipeline with routed English + Tamil NER (`Category=TamilNer`) +- **`RealTamilNerModelRunnerTests`** — direct Tamil ONNX inference (`Category=TamilNer`) - **`OnnxNerModelRunnerTests`** — unit tests for missing/invalid model paths (no download required) - **`CompositePiiDetectorTests`** — overlap merge and source-priority rules - **`LlmBoundaryTests`** — verifies raw PII never appears in outbound LLM messages diff --git a/docs/architecture.md b/docs/architecture.md index 3118415..b33ea2f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,7 +10,7 @@ The POC validates a compliance-oriented pattern suitable for financial and custo ## Canonical Example -The console application ships with a **sample catalog** (11 prompts). The canonical demo is sample `FullFinancialWithCustomer`. The table below shows the exact strings produced by the production pipeline when the ONNX NER model is loaded (run `scripts/download-ner-model.ps1` first). +The console application ships with a **sample catalog** (16 prompts). The canonical demo is sample `FullFinancialWithCustomer`. Tamil script, Tanglish, and mixed-script samples run in the **default** `dotnet run` batch (no `--interactive` required). The table below shows the exact strings produced by the production pipeline when the ONNX NER models are loaded (run `scripts/download-ner-model.ps1` and `scripts/download-tamil-ner-model.ps1` first). | Stage | Value | |-------|-------| @@ -38,7 +38,7 @@ Running `dotnet run --project src/PiiRedaction.ConsoleApp` executes all samples ### NER / person-name samples -These prompts exercise `OnnxNerPiiDetector` and `OnnxNerModelRunner`. Person names require the ONNX model (`models/ner-model.onnx` plus `vocab.txt` and `ner-labels.txt`). Without the model, person spans are not detected. +These prompts exercise `OnnxNerPiiDetector` and `RoutingOnnxNerModelRunner`. Person names require ONNX models (`models/en/` for English, `models/ta/` for Tamil script). Without models, person spans are not detected. Legacy `models/ner-model.onnx` is still supported for English. | Sample | Input (excerpt) | Detected person | Sanitized (excerpt) | |--------|-----------------|-----------------|---------------------| @@ -50,6 +50,18 @@ These prompts exercise `OnnxNerPiiDetector` and `OnnxNerModelRunner`. Person nam | **PersonWithDomainIds** | Customer Meera Iyer holds CID-7070… | Meera Iyer | Customer `` holds ``… | | **PersonWithEmailNoPhone** | Customer Arjun Mehta wrote from arjun.mehta@company.in… | Arjun Mehta | Customer `` wrote from ``… | +### Tamil / Tanglish / mixed samples + +These prompts exercise `RoutingOnnxNerModelRunner` script routing. Tamil script uses `models/ta/`; Latin Tanglish uses `models/en/`. Mixed prompts may invoke both models. + +| Sample | Input (excerpt) | Detected person | Sanitized (excerpt) | +|--------|-----------------|-----------------|---------------------| +| **TamilCustomerNameOnly** | வாடிக்கையாளர் ராஜேஷ் குமார் சேமிப்பு… | ராஜேஷ் குமார் | வாடிக்கையாளர் `` சேமிப்பு… | +| **TamilWithPhonePan** | வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210 PAN… | ராஜேஷ் குமார் | `` … `` … `` | +| **TanglishCustomer** | Customer Senthil phone 9876543210… | Senthil | Customer `` phone ``… | +| **MixedTamilEnglish** | வாடிக்கையாளர் Ravi Kumar phone 9876543210… | Ravi Kumar | வாடிக்கையாளர் `` phone ``… | +| **TamilFullFinancial** | வாடிக்கையாளர் ராஜேஷ் குமார் மின்னஞ்சல் ravi.kumar@gmail.com… | ராஜேஷ் குமார் | Tamil canonical — all placeholder types | + ### Other sample categories | Category | Sample | Purpose | @@ -84,7 +96,9 @@ flowchart TB end subgraph infra [PiiRedaction.Infrastructure] - onnxRunner["OnnxNerModelRunner"] + onnxRunner["RoutingOnnxNerModelRunner"] + enRunner["EnglishOnnxNerRunner"] + taRunner["TamilOnnxNerRunner"] mockLlm["MockLlmPromptService"] mockChat["MockChatClient"] end @@ -110,6 +124,8 @@ flowchart TB di -.-> mockLlm ``` +`RoutingOnnxNerModelRunner` selects English and/or Tamil ONNX models based on script composition in the prompt. See [Dual-Model NER Routing (Tamil + English)](#dual-model-ner-routing-tamil--english) for the routing decision tree. + --- ## Detection to Redaction Detail @@ -123,7 +139,7 @@ flowchart LR subgraph detectPhase [Detection Phase] domainDet["DomainRulePiiDetector"] regexDet["RegexPiiDetector"] - onnxDet["OnnxNerPiiDetector"] + onnxDet["OnnxNerPiiDetector
(RoutingOnnxNerModelRunner)"] composite["CompositePiiDetector"] merge["Overlap merge and source priority"] entityList["PiiEntity list"] @@ -157,14 +173,144 @@ flowchart LR 2. On overlapping spans, the first registered detector wins. 3. Tie-breaking uses source priority: Domain (3) > Regex (2) > NER (1). +The ONNX NER detector delegates to `RoutingOnnxNerModelRunner`, which routes inference to English and/or Tamil models by script composition. See [Dual-Model NER Routing (Tamil + English)](#dual-model-ner-routing-tamil--english). + **Placeholder assignment** (applied by `PlaceholderPiiRedactor`): + - Format: `<{TYPE}_{n}>` (e.g. ``, ``). - Duplicate values of the same type reuse the same placeholder. - Replacement proceeds from highest `StartIndex` to lowest to avoid index drift. --- +## Dual-Model NER Routing (Tamil + English) + +Person-name detection uses two ONNX token-classifier models: **English** (`models/en/`, BERT WordPiece) and **Tamil** (`models/ta/`, SentencePiece or WordPiece). `OnnxNerPiiDetector` calls `RoutingOnnxNerModelRunner`, which classifies prompt script via `ScriptRouter` and dispatches to `EnglishOnnxNerRunner` and/or `TamilOnnxNerRunner`. Both runners share `OnnxTokenClassifierRunner` for BIO decoding; only **PERSON** spans are emitted. + +The diagram below expands the detection and NER branches summarized in [High-Level Data Flow](#high-level-data-flow) and [Detection to Redaction Detail](#detection-to-redaction-detail). + +### End-to-end pipeline (with NER branch) + +```mermaid +flowchart TB + subgraph Entry["Console entry"] + A["Program.cs
Host + AddPiiRedactionServices()"] + B["PromptDemoRunner.RunAsync()"] + A --> B + end + + B --> C["SanitizationRequest(OriginalPrompt)"] + C --> D["PromptSanitizer.Sanitize()"] + + subgraph Detect["CompositePiiDetector.Detect() — registration order"] + direction TB + E1["DomainRulePiiDetector
LOAN_NUMBER, CUSTOMER_ID, ACCOUNT_NUMBER"] + E2["RegexPiiDetector
EMAIL, PHONE, AADHAAR, PAN, CREDIT_CARD"] + E3["OnnxNerPiiDetector
PERSON (via IOnnxNerModelRunner)"] + E1 --> MERGE + E2 --> MERGE + E3 --> MERGE + MERGE["Merge overlapping spans
sort: StartIndex ↑, Length ↓, Source priority ↓
(Domain=3, Regex=2, Ner=1)
first candidate wins on overlap"] + end + + D --> Detect + MERGE --> F["IReadOnlyList<PiiEntity>"] + + F --> G["PlaceholderPiiRedactor.Redact()
replace spans right-to-left
dedupe by Type|Value → <TYPE_n>"] + G --> H["SanitizationResult
SanitizedPrompt, DetectedEntities, PlaceholderMap"] + + H --> I["MockLlmPromptService.SendPromptAsync(SanitizedPrompt)"] + I --> J["Mock LLM response
(sanitized text only)"] + + subgraph NerBranch["OnnxNerPiiDetector branch"] + E3 --> N1{"RoutingOnnxNerModelRunner
.IsModelAvailable?"} + N1 -->|no| N2["return []"] + N1 -->|yes| N3["RoutingOnnxNerModelRunner
.PredictEntities()"] + end +``` + +### RoutingOnnxNerModelRunner decision tree + +`ScriptRouter.GetComposition` scans each character once. Tamil letters (U+0B80–U+0BFF) and ASCII Latin letters (`char.IsAsciiLetter`) determine the route. When both scripts appear, classification is **Mixed** (early exit). + +```mermaid +flowchart TB + IN["text"] --> SR["ScriptRouter.GetComposition(text)
scan each char"] + + SR --> C1{"LatinOnly?"} + SR --> C2{"TamilOnly?"} + SR --> C3{"Mixed?"} + SR --> C4{"NoLetters?"} + + C1 -->|yes| EN1{"EnglishOnnxNerRunner
.IsModelAvailable?"} + EN1 -->|yes| EN_RUN["EnglishOnnxNerRunner.PredictEntities(text)"] + EN1 -->|no| SKIP1["skip English"] + EN_RUN --> ACC + SKIP1 --> ACC + + C2 -->|yes| TA_GATE{"EnableTamilNer
&& TamilOnnxNerRunner
.IsModelAvailable?"} + TA_GATE -->|yes| TA_RUN["TamilOnnxNerRunner.PredictEntities(text)"] + TA_GATE -->|no| SKIP2["skip Tamil"] + TA_RUN --> ACC + SKIP2 --> ACC + + C3 -->|yes| EN2{"English available?"} + EN2 -->|yes| EN_MIX["EnglishOnnxNerRunner.PredictEntities(text)"] + EN2 -->|no| SKIP3["skip English"] + EN_MIX --> TA_GATE2{"EnableTamilNer
&& Tamil available?"} + SKIP3 --> TA_GATE2 + TA_GATE2 -->|yes| TA_MIX["TamilOnnxNerRunner.PredictEntities(text)"] + TA_GATE2 -->|no| SKIP4["skip Tamil"] + TA_MIX --> ACC + SKIP4 --> ACC + + C4 -->|yes| EMPTY["no NER inference"] + EMPTY --> OUT_EMPTY["return []"] + + subgraph EN_Pipeline["EnglishOnnxNerRunner"] + EN_RUN --> EN_ENC["BertWordPieceEncoder
(model dir vocab.txt)"] + EN_ENC --> EN_OCR["OnnxTokenClassifierRunner
NerLabelConfig.English
B-PER / I-PER / B-PERSON / I-PERSON"] + end + + subgraph TA_Pipeline["TamilOnnxNerRunner"] + TA_RUN --> TA_ENC["TokenClassifierEncoderFactory.Create()
vocab.txt → BertWordPieceEncoder
else SentencePiece (*.bpe.model, spiece.model, tokenizer.model)"] + TA_ENC --> TA_OCR["OnnxTokenClassifierRunner
NerLabelConfig.Tamil
label contains 'person' (case-insensitive)"] + end + + subgraph SharedInference["OnnxTokenClassifierRunner (shared)"] + ENC["Encode(text, max 128 tokens)"] + ONNX["ONNX InferenceSession.Run
input_ids + attention_mask [+ token_type_ids]"] + ARGMAX["Per-token argmax over logits"] + BIO["BIO decode → PiiEntityType.Person
PiiDetectionSource.Ner"] + ENC --> ONNX --> ARGMAX --> BIO + end + + EN_OCR --> SharedInference + TA_OCR --> SharedInference + BIO --> ACC["accumulate entities"] + + ACC --> MERGE["MergePersonSpans()
sort: Length ↓, StartIndex ↑
drop overlapping spans
(longer span wins)"] + MERGE --> OUT["return merged PERSON entities"] +``` + +### Routing rules + +| Rule | Source | Behavior | +|------|--------|----------| +| **Script classification** | `ScriptRouter.GetComposition` | Single pass over characters. Tamil letter = U+0B80–U+0BFF. Latin letter = `char.IsAsciiLetter`. Both seen → `Mixed` (early exit). Neither → `NoLetters`. Tamil only → `TamilOnly`. Latin only → `LatinOnly`. | +| **LatinOnly** | `RoutingOnnxNerModelRunner` | Run **English only** if `englishRunner.IsModelAvailable`. | +| **TamilOnly** | `RoutingOnnxNerModelRunner` | Run **Tamil only** if `EnableTamilNer` (default `true` in `PiiRedactionOptions`) **and** `tamilRunner.IsModelAvailable`. | +| **Mixed** | `RoutingOnnxNerModelRunner` | Run **both** models independently on the **full text** (English if available; Tamil if `EnableTamilNer` and available). | +| **NoLetters** | `RoutingOnnxNerModelRunner` | No NER inference; returns `[]` from routing (before merge). | +| **Model availability gate** | `OnnxNerPiiDetector` | If `RoutingOnnxNerModelRunner.IsModelAvailable` is false, NER detector returns `[]` (English OR Tamil available when Tamil enabled). | +| **Post-route merge** | `MergePersonSpans` | After EN/TA results are concatenated, overlapping PERSON spans are deduped; **longer span wins**, then ordered by `StartIndex`. | +| **Composite merge** | `CompositePiiDetector` | Domain → Regex → NER all run. Overlaps resolved globally: earlier registration order + longer span + higher source priority (Domain > Regex > Ner). | +| **Encoder choice** | `EnglishOnnxNerRunner` vs `TamilOnnxNerRunner` | English always uses `BertWordPieceEncoder`. Tamil uses factory: `vocab.txt` → WordPiece; else first SentencePiece file found; fallback WordPiece with warning. | +| **NER output scope** | `OnnxTokenClassifierRunner` | Only **PERSON** entities decoded from BIO tags; max sequence length **128** tokens. | + +--- + ## Runtime Sequence ```mermaid @@ -255,7 +401,7 @@ In the POC, `MockChatClient` simulates the external provider without network I/O |---------|-------|----------------| | `PiiRedaction.ConsoleApp` | Presentation | Application entry point; reads prompt (sample or interactive); bootstraps `IHost` and DI via `AddPiiRedactionServices`; orchestrates sanitization and LLM invocation; renders audit output (detected entities, sanitized text, placeholder map). | | `PiiRedaction.Core` | Domain / Application | Defines abstractions (`IPiiDetector`, `IPiiRedactor`, `IPromptSanitizer`, `ILlmPromptService`); implements detection strategies (`RegexPiiDetector`, `DomainRulePiiDetector`, `OnnxNerPiiDetector`, `CompositePiiDetector`); implements `PlaceholderPiiRedactor` and `PromptSanitizer`; owns domain models (`PiiEntity`, `SanitizationResult`, `RedactionResult`) and configuration (`PiiRedactionOptions`). Has no dependency on ONNX Runtime or LLM SDKs. | -| `PiiRedaction.Infrastructure` | Infrastructure | Implements technical adapters: `OnnxNerModelRunner` (ONNX Runtime inference), `MockChatClient` and `MockLlmPromptService` (`Microsoft.Extensions.AI`); depends on Core abstractions and is swappable without changing domain logic. | +| `PiiRedaction.Infrastructure` | Infrastructure | Implements technical adapters: `RoutingOnnxNerModelRunner`, `EnglishOnnxNerRunner`, `TamilOnnxNerRunner` (ONNX Runtime inference), `MockChatClient` and `MockLlmPromptService` (`Microsoft.Extensions.AI`); depends on Core abstractions and is swappable without changing domain logic. | | `tests/PiiRedaction.Core.Tests` | Test | Unit and integration tests for detectors, redactor, sanitizer, overlap rules, golden prompt scenarios (`PromptScenarioCatalog`), and LLM boundary assertions. | | `tests/PiiRedaction.Infrastructure.Tests` | Test | Tests for mock LLM behavior and ONNX runner load semantics. | @@ -272,7 +418,7 @@ In the POC, `MockChatClient` simulates the external provider without network I/O | `IPromptSanitizer` | Core | `PromptSanitizer` | Unlikely to change; orchestrates detect + redact | | `ILlmPromptService` | Core | `MockLlmPromptService` | Production adapter with telemetry, retry, policy | | `IChatClient` | Microsoft.Extensions.AI | `MockChatClient` | Azure OpenAI, OpenAI, or other provider SDK | -| `IOnnxNerModelRunner` | Core | `OnnxNerModelRunner` | BERT WordPiece tokenization, ONNX inference, BIO label decoding | +| `IOnnxNerModelRunner` | Core | `RoutingOnnxNerModelRunner` | Script-based routing to English (BERT WordPiece) and Tamil (SentencePiece) ONNX models | --- @@ -282,14 +428,18 @@ Runtime behavior is controlled via `appsettings.json` under the `PiiRedaction` s | Setting | Effect | |---------|--------| -| `OnnxModelPath` | Path to ONNX NER model (`models/ner-model.onnx` by default). Companion files `vocab.txt` and `ner-labels.txt` must live in the same directory. | +| `OnnxModelPath` | Legacy English model path (`models/ner-model.onnx`). Used as fallback when `models/en/` is absent. | +| `EnglishOnnxModelPath` | Primary English ONNX model (`models/en/ner-model.onnx`). | +| `TamilOnnxModelPath` | Tamil ONNX model (`models/ta/model.onnx`). | +| `EnableTamilNer` | When `false`, routing uses English model only. Default `true`. | -Download the model assets with `scripts/download-ner-model.ps1` (exports `dslim/bert-base-NER`). +Download model assets with `scripts/download-ner-model.ps1` and `scripts/download-tamil-ner-model.ps1`. --- ## Related Documentation +- [NER models](ner-models.md) — English/Tamil ONNX model IDs, assets, labels, and integration reference - [README](../README.md) — build, run, configuration, and testing instructions - [ServiceCollectionExtensions.cs](../src/PiiRedaction.ConsoleApp/DependencyInjection/ServiceCollectionExtensions.cs) — DI registration and detector ordering - [PromptScenarioCatalog.cs](../tests/PiiRedaction.Core.Tests/TestSupport/PromptScenarioCatalog.cs) — focused golden pipeline scenarios including the canonical example diff --git a/docs/ner-models.md b/docs/ner-models.md new file mode 100644 index 0000000..d861d69 --- /dev/null +++ b/docs/ner-models.md @@ -0,0 +1,548 @@ +# NER Models for PII Redaction + +This document describes the **Named Entity Recognition (NER)** ONNX models at the core of the PII Redaction POC. Person-name detection is the only NER responsibility in this solution; structured identifiers (email, phone, PAN, domain IDs) are handled by regex and domain-rule detectors. + +For pipeline placement, trust boundaries, and routing diagrams, see [architecture.md](architecture.md). For the Tamil/Tanglish implementation plan and success metrics, see [tamil-tanglish-ner-plan.md](tamil-tanglish-ner-plan.md). + +--- + +## 1. Executive Summary + +The POC uses **dual-model ONNX NER routing** to redact **person-name PII** before prompts reach an LLM: + +| Script in prompt | Model invoked | Typical use case | +|------------------|---------------|------------------| +| Latin only (`LatinOnly`) | English (`dslim/bert-base-NER`) | English names, Indian names in Roman script, **Tanglish** | +| Tamil only (`TamilOnly`) | Tamil (`prachuryyaIITG/SampurNER_Tamil_IndicBERTv2`) | Tamil-script customer names | +| Mixed (`Mixed`) | **Both** models on the full text; spans merged | Code-mixed Indian CS prompts | +| No letters (`NoLetters`) | Neither | Digits-only or symbol-only text | + +`RoutingOnnxNerModelRunner` classifies script via `ScriptRouter`, delegates to `EnglishOnnxNerRunner` and/or `TamilOnnxNerRunner`, and merges overlapping PERSON spans (longer span wins). Only **PERSON** entities are emitted to the redaction pipeline; all other NER labels are discarded. + +--- + +## 2. English Model + +### Hugging Face model ID + +**[`dslim/bert-base-NER`](https://huggingface.co/dslim/bert-base-NER)** + +### Architecture + +| Property | Value | +|----------|-------| +| Base | BERT-base (uncased), ~110M parameters | +| Task | Token classification (NER) | +| Tokenizer | **WordPiece** via `vocab.txt` (`BertWordPieceEncoder`) | +| Runtime | ONNX via Microsoft.ML.OnnxRuntime | +| Export | Hugging Face Optimum (`ORTModelForTokenClassification`) or pre-exported ONNX from HF | + +### Labels (BIO) + +The English model uses standard CoNLL-style BIO tags. The POC maps only **person** labels to `PiiEntityType.Person`: + +| Label | Mapped to PERSON | +|-------|------------------| +| `O` | No | +| `B-PER`, `I-PER` | Yes | +| `B-PERSON`, `I-PERSON` | Yes | +| `B-ORG`, `I-ORG`, `B-LOC`, `I-LOC`, `B-MISC`, `I-MISC` | No | + +Full label list is written to `ner-labels.txt` at download time from the model `config.json` `id2label` map (typically 9 labels for this model). + +Label matching is implemented in `NerLabelConfig.English`: + +```19:22:src/PiiRedaction.Infrastructure/Onnx/NerLabelConfig.cs + private static bool IsEnglishPersonLabel(string label) => + label is "B-PER" or "I-PER" or "B-PERSON" or "I-PERSON" + || (label.EndsWith("-PER", StringComparison.Ordinal) && + (label.StartsWith("B-", StringComparison.Ordinal) || label.StartsWith("I-", StringComparison.Ordinal))); +``` + +### Asset paths + +| File | Primary path (`appsettings.json`) | Legacy fallback | +|------|----------------------------------|-----------------| +| ONNX model | `models/en/ner-model.onnx` | `models/ner-model.onnx` (`OnnxModelPath`) | +| Vocabulary | `models/en/vocab.txt` | `models/vocab.txt` | +| Labels | `models/en/ner-labels.txt` | `models/ner-labels.txt` | + +`EnglishOnnxNerRunner` resolves the model path with primary + legacy fallback: + +```15:22:src/PiiRedaction.Infrastructure/Onnx/EnglishOnnxNerRunner.cs + var modelPath = OnnxAssetPathResolver.ResolveModelPath( + options.Value.EnglishOnnxModelPath, + options.Value.OnnxModelPath); + + var modelDirectory = Path.GetDirectoryName(modelPath) ?? Environment.CurrentDirectory; + var labels = OnnxAssetPathResolver.LoadLabels(modelDirectory); + var encoder = new BertWordPieceEncoder(modelDirectory, logger); + _runner = new OnnxTokenClassifierRunner(modelPath, encoder, NerLabelConfig.English, labels, logger); +``` + +> **Note:** `scripts/download-ner-model.ps1` writes assets to `models/` (repository root). For the configured primary path, copy or move them into `models/en/`, or rely on the `OnnxModelPath` fallback. + +### Download script + +```powershell +.\scripts\download-ner-model.ps1 +``` + +Or with Python directly: + +```bash +python scripts/download-ner-model.py +``` + +**Behavior:** + +1. If Python + Optimum are available → exports `dslim/bert-base-NER` to ONNX under `models/`. +2. Otherwise → downloads pre-exported ONNX from `https://huggingface.co/dslim/bert-base-NER/resolve/main/onnx/` (`model.onnx`, `vocab.txt`, `config.json` → `ner-labels.txt`). + +--- + +## 3. Tamil Model + +### Hugging Face model ID + +**[`prachuryyaIITG/SampurNER_Tamil_IndicBERTv2`](https://huggingface.co/prachuryyaIITG/SampurNER_Tamil_IndicBERTv2)** + +(SampurNER Tamil IndicBERTv2 — fine-grained NER for Tamil script.) + +### Why SampurNER IndicBERTv2 vs MuRIL + +| Criterion | SampurNER Tamil IndicBERTv2 | MuRIL (fallback candidate) | +|-----------|----------------------------|----------------------------| +| Tamil NER training | Fine-grained SampurNER dataset (Tamil-specific labels) | General multilingual; NER requires separate fine-tune | +| Model size | ~0.3B parameters (IndicBERTv2, ~278M base) | ~0.6B parameters | +| POC fit | Lighter memory footprint; ONNX export path validated in this repo | Reserved for Phase 5 if Tamil recall is insufficient | +| Indian financial context | Trained on Indian-language NER corpus; person subtypes map cleanly to PERSON | Heavier; eval-driven swap only | + +See [tamil-tanglish-ner-plan.md](tamil-tanglish-ner-plan.md) §3 for the original selection rationale. + +### Architecture + +| Property | Value | +|----------|-------| +| Base | IndicBERTv2 (AI4Bharat), ~0.3B parameters | +| Task | Fine-grained token classification | +| Tokenizer | **WordPiece** when `vocab.txt` is present (this repo's export path); SentencePiece fallback if `sentencepiece.bpe.model` / `spiece.model` exists | +| Runtime | Same shared `OnnxTokenClassifierRunner` as English | + +### Tokenizer: WordPiece, not SentencePiece (in practice) + +The Hugging Face repo for this model does **not** ship a SentencePiece model file. The download scripts extract **WordPiece** assets from `tokenizer.json` → `vocab.txt`. `TokenClassifierEncoderFactory` prefers `vocab.txt`: + +```10:19:src/PiiRedaction.Infrastructure/Onnx/TokenClassifierEncoderFactory.cs + public static ITokenClassifierEncoder Create(string modelDirectory, ILogger logger) + { + var vocabPath = OnnxAssetPathResolver.ResolveAssetPath(Path.Combine(modelDirectory, "vocab.txt")); + if (File.Exists(vocabPath)) + { + logger.LogInformation( + "Using WordPiece tokenizer (vocab.txt) from {ModelDirectory}.", + modelDirectory); + return new BertWordPieceEncoder(modelDirectory, logger); + } +``` + +The PowerShell Tamil download script emits an explicit warning when WordPiece assets are saved instead of SentencePiece. + +### Labels (fine-grained person tags) + +SampurNER uses fine-grained BIO tags (e.g. `B-person-politician`, `I-person-artist`, `B-location`, `O`). The POC treats **any label containing `person`** (case-insensitive) as a person span: + +```24:25:src/PiiRedaction.Infrastructure/Onnx/NerLabelConfig.cs + private static bool IsTamilPersonLabel(string label) => + label.Contains("person", StringComparison.OrdinalIgnoreCase); +``` + +Unit tests lock this behavior: + +```20:27:tests/PiiRedaction.Infrastructure.Tests/Onnx/NerLabelConfigTests.cs + [TestCase("B-person-politician", true)] + [TestCase("I-person-artist", true)] + [TestCase("B-location", false)] + [TestCase("O", false)] + public void Tamil_IsPersonLabel_MatchesFineGrainedTags(string label, bool expected) + { + NerLabelConfig.Tamil.IsPersonLabel(label).Should().Be(expected); + } +``` + +### Asset paths + +| File | Path | +|------|------| +| ONNX model | `models/ta/model.onnx` | +| Tokenizer | `models/ta/vocab.txt` (WordPiece, preferred) **or** `models/ta/sentencepiece.bpe.model` | +| Labels | `models/ta/ner-labels.txt` | +| Optional | `models/ta/tokenizer.json` (intermediate export artifact) | + +### Download script + +```powershell +.\scripts\download-tamil-ner-model.ps1 +``` + +Or with Python directly: + +```bash +python scripts/download-tamil-ner-model.py +``` + +**Behavior:** + +1. Python + Optimum → full export to `models/ta/` including ONNX, labels, and tokenizer assets. +2. PowerShell fallback → downloads `onnx/model.onnx` from Hugging Face when published; otherwise requires Python export (pre-exported ONNX may return 404). + +`TamilOnnxNerRunner` wiring: + +```15:19:src/PiiRedaction.Infrastructure/Onnx/TamilOnnxNerRunner.cs + var modelPath = OnnxAssetPathResolver.ResolveModelPath(options.Value.TamilOnnxModelPath); + var modelDirectory = Path.GetDirectoryName(modelPath) ?? Environment.CurrentDirectory; + var labels = OnnxAssetPathResolver.LoadLabels(modelDirectory); + var encoder = TokenClassifierEncoderFactory.Create(modelDirectory, logger); + _runner = new OnnxTokenClassifierRunner(modelPath, encoder, NerLabelConfig.Tamil, labels, logger); +``` + +--- + +## 4. Why These Models + +Evidence-based rationale for this Indian financial POC: + +| Requirement | Decision | +|-------------|----------| +| **English + Indian Latin names** | `dslim/bert-base-NER` is industry-standard, pre-integrated, and handles many Indian names in Roman script (e.g. `Ravi Kumar`, `Anita Sharma`) | +| **Tamil script names** | English BERT is out-of-vocabulary for Tamil letters (U+0B80–U+0BFF); a Tamil-trained NER model is required | +| **Tanglish (Roman-script Tamil)** | Routed to the **English** model only (`ScriptComposition.LatinOnly`); no Tamil ONNX on Latin-only text | +| **Code-mixed prompts** | Both models run on the full prompt; `MergePersonSpans` deduplicates overlaps | +| **Deployable size** | English ~431 MB ONNX (FP32); Tamil IndicBERTv2 ~0.3B params — smaller than MuRIL ~0.6B | +| **ONNX export path** | Both models export via Hugging Face Optimum; English has pre-exported ONNX on HF; Tamil may require local Python export | +| **PERSON-only scope** | POC redacts person names via NER; org/location/misc labels are intentionally ignored to limit false positives | + +--- + +## 5. How They Integrate + +### Configuration (`appsettings.json`) + +```1:8:src/PiiRedaction.ConsoleApp/appsettings.json +{ + "PiiRedaction": { + "OnnxModelPath": "models/ner-model.onnx", + "EnglishOnnxModelPath": "models/en/ner-model.onnx", + "TamilOnnxModelPath": "models/ta/model.onnx", + "EnableTamilNer": true + } +} +``` + +Options type: + +```3:14:src/PiiRedaction.Core/Configuration/PiiRedactionOptions.cs +public sealed class PiiRedactionOptions +{ + public const string SectionName = "PiiRedaction"; + + public string OnnxModelPath { get; set; } = "models/ner-model.onnx"; + + public string EnglishOnnxModelPath { get; set; } = "models/en/ner-model.onnx"; + + public string TamilOnnxModelPath { get; set; } = "models/ta/model.onnx"; + + public bool EnableTamilNer { get; set; } = true; +} +``` + +Set `EnableTamilNer` to `false` for English-only routing. + +### Dependency injection + +```34:36:src/PiiRedaction.ConsoleApp/DependencyInjection/ServiceCollectionExtensions.cs + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); +``` + +`OnnxNerPiiDetector` consumes `IOnnxNerModelRunner` (the router) and returns `[]` when no model is available — **fail-open** for person detection. + +### Script routing (`ScriptRouter`) + +```11:41:src/PiiRedaction.Core/Detection/ScriptRouter.cs + public ScriptComposition GetComposition(string text) + { + ArgumentNullException.ThrowIfNull(text); + + var hasLatin = false; + var hasTamil = false; + + foreach (var character in text) + { + if (IsTamilLetter(character)) + { + hasTamil = true; + } + else if (char.IsAsciiLetter(character)) + { + hasLatin = true; + } + + if (hasLatin && hasTamil) + { + return ScriptComposition.Mixed; + } + } + // ... + return hasTamil ? ScriptComposition.TamilOnly : ScriptComposition.LatinOnly; + } +``` + +### Routing runner (`RoutingOnnxNerModelRunner`) + +```39:79:src/PiiRedaction.Infrastructure/Onnx/RoutingOnnxNerModelRunner.cs + public IReadOnlyList PredictEntities(string text) + { + var composition = _scriptRouter.GetComposition(text); + var entities = new List(); + + switch (composition) + { + case ScriptComposition.LatinOnly: + if (_englishRunner.IsModelAvailable) + { + entities.AddRange(_englishRunner.PredictEntities(text)); + } + break; + case ScriptComposition.TamilOnly: + if (_enableTamilNer && _tamilRunner.IsModelAvailable) + { + entities.AddRange(_tamilRunner.PredictEntities(text)); + } + break; + case ScriptComposition.Mixed: + if (_englishRunner.IsModelAvailable) + { + entities.AddRange(_englishRunner.PredictEntities(text)); + } + if (_enableTamilNer && _tamilRunner.IsModelAvailable) + { + entities.AddRange(_tamilRunner.PredictEntities(text)); + } + break; + case ScriptComposition.NoLetters: + break; + } + + return MergePersonSpans(entities); + } +``` + +### Shared inference (`OnnxTokenClassifierRunner`) + +Both runners share: + +- **Encode** → `input_ids`, `attention_mask`, optional `token_type_ids` +- **Argmax** over per-token logits +- **BIO decode** → `PiiEntityType.Person` with `PiiDetectionSource.Ner` +- **Max sequence length: 128 tokens** + +```13:13:src/PiiRedaction.Infrastructure/Onnx/OnnxTokenClassifierRunner.cs + private const int MaxSequenceLength = 128; +``` + +### End-to-end flow + +``` +Prompt → CompositePiiDetector → OnnxNerPiiDetector + → RoutingOnnxNerModelRunner → ScriptRouter + → EnglishOnnxNerRunner / TamilOnnxNerRunner + → OnnxTokenClassifierRunner → PERSON entities + → PlaceholderPiiRedactor → +``` + +--- + +## 6. Evidence from Codebase + +### Real-model tests (`Category=RealModel`) + +English direct inference — `RealNerModelRunnerTests`: + +```15:47:tests/PiiRedaction.Infrastructure.Tests/Onnx/RealNerModelRunnerTests.cs +[Category("RealModel")] +public sealed class RealNerModelRunnerTests : RealNerModelFixture +{ + [TestCase("Customer Ravi Kumar called about billing.", "Ravi", "Ravi Kumar")] + [TestCase("Mr. John Smith called about a duplicate debit.", "John", "John Smith")] + public void PredictEntities_DetectsPersonWithCorrectSpan(...) +``` + +English pipeline — `RealNerPipelineTests` (`Category=RealModel`): canonical `FullFinancialWithCustomer`, multi-person, clean-ticket negative. + +Fixture skips when model missing: + +```5:6:tests/TestSupport.Shared/RealNerModelPaths.cs + public const string ModelMissingMessage = + "ONNX model not found. Run scripts/download-ner-model.ps1 from the repository root."; +``` + +Resolves `models/en/ner-model.onnx` then `models/ner-model.onnx`. + +### Tamil tests (`Category=TamilNer`) + +Direct Tamil runner — `RealTamilNerModelRunnerTests`: + +```9:69:tests/PiiRedaction.Infrastructure.Tests/Onnx/RealTamilNerModelRunnerTests.cs +[Category("TamilNer")] +public sealed class RealTamilNerModelRunnerTests : RealTamilNerModelFixture +{ + [TestCase("வாடிக்கையாளர் ராஜேஷ் குமார் அழைத்தார்.", "ராஜேஷ்", "ராஜேஷ் குமார்")] + public void PredictEntities_TamilScript_DetectsPersonEntity(...) + // ... + [TestCase("Customer Senthil phone 9876543210", "Senthil")] + public void PredictEntities_TanglishLatinScript_DoesNotInvokeTamilRunner(...) +``` + +Routed pipeline — `RealTamilPipelineTests` covers Tamil-only, Tanglish (English path), mixed, full financial, and clean Tamil negative. + +### Console samples (`SamplePromptCatalog`) + +Tamil/Tanglish/mixed samples (indices 10–14): + +| Sample | Category | Input excerpt | +|--------|----------|---------------| +| `TamilCustomerNameOnly` | NER (Tamil) | `வாடிக்கையாளர் ராஜேஷ் குமார் …` | +| `TamilWithPhonePan` | NER (Tamil) + Regex | Tamil person + phone + PAN | +| `TanglishCustomer` | NER (English/Tanglish) | `Customer Senthil phone 9876543210 …` | +| `MixedTamilEnglish` | NER (Mixed) | `வாடிக்கையாளர் Ravi Kumar phone …` | +| `TamilFullFinancial` | NER (Tamil) + Regex + Domain | Tamil canonical demo | + +Run all samples (including Tamil) with no flags: + +```bash +dotnet run --project src/PiiRedaction.ConsoleApp +``` + +Or a single Tamil sample: + +```bash +dotnet run --project src/PiiRedaction.ConsoleApp -- --name TamilCustomerNameOnly +``` + +### Expected console output (English canonical) + +When models are loaded, person names appear as `[PERSON]` with source `Ner`: + +``` +Detected PII: + [PERSON ] Ravi Kumar (Ner) + [EMAIL ] ravi.kumar@gmail.com (Regex) + ... + +Sanitized Prompt: +Customer with email ... +``` + +--- + +## 7. Model Assets Table + +All model binaries are **gitignored**; only `.gitkeep` placeholders are committed. + +| Directory | File | Approx. size | Gitignored | Purpose | +|-----------|------|--------------|------------|---------| +| `models/` or `models/en/` | `ner-model.onnx` | ~431 MB | Yes | English BERT NER (FP32 ONNX from HF) | +| `models/` or `models/en/` | `vocab.txt` | ~213 KB | Yes | WordPiece vocabulary | +| `models/` or `models/en/` | `ner-labels.txt` | < 1 KB | Yes | BIO label index (one per line) | +| `models/ta/` | `model.onnx` | ~1.2 GB (FP32 export, varies) | Yes | Tamil IndicBERT NER | +| `models/ta/` | `vocab.txt` | varies | Yes | WordPiece vocab (preferred tokenizer) | +| `models/ta/` | `tokenizer.json` | varies | Yes | HF tokenizer export (optional) | +| `models/ta/` | `sentencepiece.bpe.model` | varies | Yes | SentencePiece (if present instead of vocab) | +| `models/ta/` | `ner-labels.txt` | few KB | Yes | Fine-grained SampurNER labels | + +`.gitignore` entries: + +``` +models/*.onnx +models/vocab.txt +models/ner-labels.txt +models/*.json +models/en/* +models/ta/* +``` + +English size reference: [docs/git-xenovex-setup.md](git-xenovex-setup.md) notes ~431 MB for the English ONNX file. + +--- + +## 8. Limitations + +| Limitation | Detail | +|------------|--------| +| **Tanglish on English model only** | Roman-script Tanglish (`Customer Senthil`) is classified `LatinOnly` and handled by English BERT. Recall is best-effort and inconsistent for non-standard spellings. Tamil ONNX is **not** invoked on Latin-only text. | +| **Fail-open if model missing** | `OnnxNerPiiDetector` and routing runners return `[]` when models are unavailable. Person names are **not** redacted; regex/domain layers still run. No regex fallback for names. | +| **128 token limit** | `OnnxTokenClassifierRunner` truncates encoding at 128 tokens. Very long prompts may miss person names beyond the window. | +| **PERSON-only NER scope** | Organization, location, and misc NER labels are ignored. Only person spans become ``. | +| **Mixed-script merge** | When both models run, overlapping spans are deduped by length; shorter overlapping spans are dropped. | +| **Tamil ONNX availability** | Pre-exported Tamil ONNX may not exist on Hugging Face; local Python export is often required. | +| **No fail-closed mode** | Missing NER does not block sanitization or LLM calls (optional Phase 5 enhancement). | + +--- + +## 9. How to Reproduce + +### Download models + +From the repository root: + +```powershell +.\scripts\download-ner-model.ps1 +.\scripts\download-tamil-ner-model.ps1 +``` + +If Tamil PowerShell download fails with a 404 on `onnx/model.onnx`, install Python 3.12+ and re-run: + +```powershell +winget install Python.Python.3.12 --accept-package-agreements --accept-source-agreements +.\scripts\download-tamil-ner-model.ps1 -Python "$env:LOCALAPPDATA\Programs\Python\Python312\python.exe" +``` + +Optional: copy English assets from `models/` to `models/en/` to match `EnglishOnnxModelPath`. + +### Verify with tests + +```bash +dotnet build +dotnet test +dotnet test --filter "Category=RealModel" +dotnet test --filter "Category=TamilNer" +dotnet test --logger "console;verbosity=detailed" +``` + +Tests skip gracefully when the corresponding ONNX files are absent. + +### Verify with console + +```bash +dotnet run --project src/PiiRedaction.ConsoleApp -- --name CustomerNameOnly +dotnet run --project src/PiiRedaction.ConsoleApp -- --name TamilCustomerNameOnly +dotnet run --project src/PiiRedaction.ConsoleApp -- --name TanglishCustomer +``` + +### Disable Tamil routing (English-only) + +Set in `appsettings.json`: + +```json +"EnableTamilNer": false +``` + +--- + +## Related Documentation + +- [README.md](../README.md) — build, run, and test overview +- [architecture.md](architecture.md) — dual-model routing diagrams and trust boundary +- [tamil-tanglish-ner-plan.md](tamil-tanglish-ner-plan.md) — implementation phases and Tanglish expectations diff --git a/docs/tamil-tanglish-ner-plan.md b/docs/tamil-tanglish-ner-plan.md new file mode 100644 index 0000000..088e5b6 --- /dev/null +++ b/docs/tamil-tanglish-ner-plan.md @@ -0,0 +1,307 @@ +# Tamil / Tanglish NER — Implementation Plan + +**Goal:** Raise language coverage from ~15% to production-viable for Tamil script and Tanglish (Roman-script Tamil-English) customer prompts, without changing the secure LLM boundary pattern. + +**Status:** Phase 1–3 implemented +**Approach:** Dual-model ONNX NER routing (English + Tamil) + lightweight text normalization + optional Tanglish heuristics +**Estimated effort:** 4–6 engineering days across 4 phases + +--- + +## 1. Current State vs Gap + +| Capability | Today | Tamil script | Tanglish (Latin) | +|------------|-------|--------------|------------------| +| Phone, PAN, Aadhaar, email, domain IDs | Regex + domain rules | Works (ASCII digits) | Works | +| Person names | `dslim/bert-base-NER` (English BERT) | **Fails** — out of vocabulary | **Partial** — inconsistent | +| Script / language routing | None | N/A | N/A | +| Tamil numerals (௦–௯) | Not normalized | **May miss** phone/Aadhaar | N/A | +| Label-aware cues (`பெயர்`, `peru`, `enga peru`) | None | **Misses** contextual names | **Misses** | + +**Root cause:** Person detection is a single English-only ONNX model behind `IOnnxNerModelRunner` → `OnnxNerPiiDetector`. Regex/domain layers are already language-agnostic. + +--- + +## 2. Target Architecture + +No change to the trust boundary: `PromptSanitizer` → `CompositePiiDetector` → `PlaceholderPiiRedactor` → sanitized text only to LLM. + +Only the **NER adapter** expands: + +```mermaid +flowchart TD + text["Prompt text"] + router["ScriptRouter (Core)"] + routing["RoutingOnnxNerModelRunner"] + en["EnglishOnnxRunner\nBERT WordPiece"] + ta["TamilOnnxRunner\nIndicBERT SentencePiece"] + nerDet["OnnxNerPiiDetector"] + composite["CompositePiiDetector"] + + text --> composite + text --> nerDet + nerDet --> routing + routing --> router + router -->|"LatinOnly / Mixed"| en + router -->|"TamilOnly / Mixed"| ta + en --> routing + ta --> routing +``` + +### Routing rules + +| `ScriptComposition` | Models invoked | Tanglish note | +|---------------------|----------------|---------------| +| `LatinOnly` | English NER only | Tanglish names in Roman script | +| `TamilOnly` | Tamil NER only | Tamil script names | +| `Mixed` | **Both**, merge person spans | Common in Indian CS prompts | +| `NoLetters` | Neither (or English fallback off) | Digits-only prompts | + +**Merge inside `RoutingOnnxNerModelRunner`:** dedupe overlapping person spans (prefer longer span; tie-break Tamil vs English by start index order). + +--- + +## 3. Model Selection + +| Role | Model | Rationale | +|------|-------|-----------| +| English / Tanglish (Latin) | **Keep** `dslim/bert-base-NER` | Already integrated; works for many Indian names in Latin script | +| Tamil script | **`prachuryyaIITG/SampurNER_Tamil_IndicBERTv2`** | Tamil NER; lighter than MuRIL (~0.6B) | +| Fallback (optional Phase 5) | MuRIL Tamil NER | Only if IndicBERT recall is insufficient on eval set | + +### Asset layout + +``` +models/ + en/ + ner-model.onnx # or model.onnx (BERT export) + vocab.txt + ner-labels.txt + ta/ + model.onnx + sentencepiece.bpe.model # or tokenizer.json from HF export + ner-labels.txt + ner-model.onnx # legacy path — keep for backward compatibility +``` + +### Label mapping (Tamil fine-grained NER) + +SampurNER uses fine-grained tags (e.g. `B-person-politician`, `I-person-artist`). Map **any label containing `person`** (case-insensitive) → `PiiEntityType.Person`. + +English labels remain: `B-PER`, `I-PER`, `B-PERSON`, `I-PERSON`. + +--- + +## 4. Implementation Phases + +### Phase 1 — Generic ONNX token classifier (1–2 days) + +**Objective:** Refactor `OnnxNerModelRunner` so BERT and SentencePiece are pluggable. + +| Action | Location | +|--------|----------| +| Add `ITokenClassifierEncoder` + `EncodedSequence` | `Infrastructure/Onnx/` | +| `BertWordPieceEncoder` — extract from current runner | Infrastructure | +| `SentencePieceEncoder` — IndicBERT tokenizer | Infrastructure | +| `OnnxTokenClassifierRunner` — shared inference + BIO decode | Infrastructure | +| `NerLabelConfig` — English vs Tamil person label predicates | Infrastructure | +| `OnnxAssetPathResolver` — resolve model dir from repo root | Infrastructure | +| Thin wrappers: `EnglishOnnxNerRunner`, `TamilOnnxNerRunner` | Infrastructure | + +**Backward compat:** If `models/en/` missing, fall back to `OnnxModelPath` (`models/ner-model.onnx`). + +**No behavior change** until Phase 3 wiring — existing tests must pass. + +--- + +### Phase 2 — Tamil model download + config (0.5–1 day) + +| Action | Details | +|--------|---------| +| `scripts/download-tamil-ner-model.ps1` + `.py` | Mirror `download-ner-model.ps1`; export via `optimum-cli export onnx --task token-classification` | +| Optional: `scripts/download-all-ner-models.ps1` | Calls English + Tamil scripts | +| Extend `PiiRedactionOptions` | `EnglishOnnxModelPath`, `TamilOnnxModelPath`, `EnableTamilNer` (default `true`) | +| Update `appsettings.json` | New paths under `PiiRedaction` section | +| `.gitignore` | `models/ta/*`, `models/en/*` (same as today for onnx/vocab) | + +--- + +### Phase 3 — Script routing + DI (0.5–1 day) + +| Action | Location | +|--------|----------| +| `ScriptRouter` + `ScriptComposition` enum | `Core/Detection/` | +| `RoutingOnnxNerModelRunner` implements `IOnnxNerModelRunner` | Infrastructure | +| DI registration | `ServiceCollectionExtensions.cs` | + +```csharp +services.AddSingleton(); +services.AddSingleton(); +services.AddSingleton(); +``` + +`OnnxNerPiiDetector` and `CompositePiiDetector` **unchanged**. + +--- + +### Phase 4 — Tests, samples, docs (1 day) + +#### Unit tests + +| Test class | Coverage | +|------------|----------| +| `ScriptRouterTests` | Tamil-only, Latin-only, mixed, no-letters, boundary chars U+0B80/U+0BFF | +| `RoutingOnnxNerModelRunnerTests` | Fake EN/TA runners; mixed script merges both | +| `NerLabelConfigTests` | Tamil fine-grained person labels map correctly | + +#### Real-model tests (`Category=RealModel` or `Category=TamilNer`) + +| Scenario | Input example | Assert | +|----------|---------------|--------| +| Tamil name | `வாடிக்கையாளர் ராஜேஷ் தொலைபேசி 9876543210` | ``, phone redacted | +| Tanglish name | `Customer Senthil phone 9876543210` | person + phone (best-effort) | +| Mixed | `Rajesh மற்றும் Priya` | both persons redacted | +| Clean Tamil | `பணத்தை திரும்பப் பெறுவது எப்படி?` | no false positives | +| Canonical English | existing golden tests | no regression | + +Skip gracefully when `models/ta/model.onnx` missing (mirror `RealNerModelFixture`). + +#### Console samples + +Add to `SamplePromptCatalog.cs`: + +- `TamilCustomerName` — Tamil script person +- `TanglishCustomerName` — `enga peru Rajesh` / `Customer Senthil` +- `MixedTamilEnglish` — code-mixed prompt + +#### Docs + +- Update `README.md` ONNX setup (dual models) +- Update `docs/architecture.md` NER section +- Link this plan from README + +--- + +### Phase 5 — Optional enhancements (post-MVP) + +| Enhancement | Benefit | Effort | +|-------------|---------|--------| +| **Unicode digit normalization** pre-pass | Tamil numerals → ASCII for regex | 0.5 day | +| **Label-based regex** (`பெயர்`, `peru`, `peyar`, `enga peru`) | Tanglish recall without ML | 0.5 day | +| **Tamil name gazetteer** `IPiiDetector` | High precision for top names | 1 day | +| **Fail-closed policy** when NER unavailable | Compliance option | 0.5 day | +| MuRIL model swap | Higher Tamil recall | eval-driven | + +--- + +## 5. Tanglish — Realistic Expectations + +| Input type | Primary handler | Expected recall | +|------------|-----------------|-----------------| +| Tamil script names | Tamil ONNX NER | High (with eval tuning) | +| Standard Latin Indian names (`Ravi Kumar`) | English ONNX NER | High (already works) | +| Tanglish spellings (`Senthil`, `senthil`, `Centhil`) | English NER + optional gazetteer | Medium | +| Code-mixed (`Rajesh oda account ACC-123456`) | English NER + domain regex | Medium–high for IDs; name variable | + +**MVP target:** Tamil script person names reliably redacted; Tanglish improved but not 100% without Phase 5 heuristics. + +--- + +## 6. Success Metrics + +Before marking language gap closed, run an **eval set of 20–30 real prompts** (anonymized production samples): + +| Metric | MVP target | +|--------|------------| +| Tamil script person-name recall | ≥ 85% | +| Tanglish person-name recall | ≥ 70% (with English model + optional heuristics) | +| False positive rate (clean prompts) | ≤ 5% | +| Structured PII (phone/PAN/domain) in Tamil prompts | ≥ 95% (regex layer) | +| Regression on English canonical demo | 100% (existing golden tests) | + +--- + +## 7. Files to Create / Modify (checklist) + +### New files + +- `src/PiiRedaction.Core/Detection/ScriptRouter.cs` +- `src/PiiRedaction.Core/Detection/ScriptComposition.cs` +- `src/PiiRedaction.Infrastructure/Onnx/OnnxTokenClassifierRunner.cs` +- `src/PiiRedaction.Infrastructure/Onnx/BertWordPieceEncoder.cs` +- `src/PiiRedaction.Infrastructure/Onnx/SentencePieceEncoder.cs` +- `src/PiiRedaction.Infrastructure/Onnx/ITokenClassifierEncoder.cs` +- `src/PiiRedaction.Infrastructure/Onnx/NerLabelConfig.cs` +- `src/PiiRedaction.Infrastructure/Onnx/OnnxAssetPathResolver.cs` +- `src/PiiRedaction.Infrastructure/Onnx/EnglishOnnxNerRunner.cs` +- `src/PiiRedaction.Infrastructure/Onnx/TamilOnnxNerRunner.cs` +- `src/PiiRedaction.Infrastructure/Onnx/RoutingOnnxNerModelRunner.cs` +- `scripts/download-tamil-ner-model.ps1` / `.py` +- `tests/.../ScriptRouterTests.cs` +- `tests/.../RoutingOnnxNerModelRunnerTests.cs` +- `tests/.../RealTamilNerPipelineTests.cs` +- `tests/TestSupport.Shared/RealTamilModelFixture.cs` + +### Modified files + +- `PiiRedactionOptions.cs` — dual model paths +- `ServiceCollectionExtensions.cs` — routing DI +- `appsettings.json` — config +- `SamplePromptCatalog.cs` — Tamil/Tanglish demos +- `ProductionPipelineFactory.cs` — `CreateWithRoutingRealModel()` for tests +- `RealNerModelFixture.cs` / paths — support EN + TA +- `README.md`, `docs/architecture.md` +- `.gitignore` — `models/en/`, `models/ta/` + +### Unchanged (by design) + +- `PromptSanitizer`, `PlaceholderPiiRedactor`, `CompositePiiDetector` +- `RegexPiiDetector`, `DomainRulePiiDetector` +- `MockLlmPromptService` / LLM boundary + +--- + +## 8. Rollout & Risk + +| Risk | Mitigation | +|------|------------| +| Tamil model export fails on Windows | PowerShell fallback downloads pre-exported ONNX from Hugging Face | +| Larger memory (two models) | Lazy-load Tamil runner only when `EnableTamilNer` and Tamil script detected | +| Fine-grained label mismatch | Load labels from `ner-labels.txt`; unit test label config | +| Tanglish disappointment | Set stakeholder expectation in README; Phase 5 heuristics | +| CI without models | Fast tests use fakes; `Category=TamilNer` skips like `RealModel` | + +**Feature flag:** `EnableTamilNer=false` reverts to English-only behavior for gradual rollout. + +--- + +## 9. Command Reference (after implementation) + +```powershell +# Download both models +.\scripts\download-ner-model.ps1 +.\scripts\download-tamil-ner-model.ps1 + +# Run Tamil-focused console sample +dotnet run --project src/PiiRedaction.ConsoleApp -- --name TamilCustomerName + +# Tests +dotnet test +dotnet test --filter "Category=TamilNer" +dotnet test --filter "Category=RealModel" +``` + +--- + +## 10. Approval Checklist + +- [ ] Stakeholder sign-off on dual-model approach (vs single multilingual model) +- [ ] Tamil eval prompt set collected (20–30 samples) +- [ ] Xenovex CI policy: models downloaded in pipeline or tests skip +- [x] Phase 1–3 implementation PR +- [ ] Phase 4 eval metrics met +- [ ] Optional Phase 5 for Tanglish heuristics if recall < 70% + +--- + +**Next step:** Implement Phase 1 in a feature branch (`feature/tamil-tanglish-ner`), open PR to `main` on `xts.xenovex.com/Bilal-Nazer-Ali/llm-pii-poc`. diff --git a/models/en/.gitkeep b/models/en/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/models/ta/.gitkeep b/models/ta/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/scripts/download-tamil-ner-model.ps1 b/scripts/download-tamil-ner-model.ps1 new file mode 100644 index 0000000..c45e8c1 --- /dev/null +++ b/scripts/download-tamil-ner-model.ps1 @@ -0,0 +1,229 @@ +# Downloads prachuryyaIITG/SampurNER_Tamil_IndicBERTv2 ONNX assets to models/ta/ for the PII Redaction POC. +param( + [string]$Python = "python" +) + +$ErrorActionPreference = "Stop" +$repoRoot = Split-Path -Parent $PSScriptRoot +$modelsDir = Join-Path (Join-Path $repoRoot "models") "ta" +$scriptPath = Join-Path $PSScriptRoot "download-tamil-ner-model.py" +$modelId = "prachuryyaIITG/SampurNER_Tamil_IndicBERTv2" +$baseUrl = "https://huggingface.co/$modelId/resolve/main" + +function Ensure-ModelsDirectory { + New-Item -ItemType Directory -Force -Path $modelsDir | Out-Null +} + +function Download-HuggingFaceAsset { + param( + [string]$RelativePath, + [string]$Destination + ) + + $url = "$baseUrl/$RelativePath" + Write-Host "Downloading $url" + Invoke-WebRequest -Uri $url -OutFile $Destination -UseBasicParsing +} + +function Export-LabelsFromConfig { + param([string]$ConfigPath, [string]$LabelsPath) + + $config = Get-Content $ConfigPath -Raw | ConvertFrom-Json + $labelMap = @{} + foreach ($property in $config.id2label.PSObject.Properties) { + $labelMap[[int]$property.Name] = [string]$property.Value + } + + $labels = for ($index = 0; $index -lt $labelMap.Count; $index++) { + $labelMap[$index] + } + + $labels | Set-Content -Path $LabelsPath -Encoding utf8 +} + +function Resolve-PythonExecutable { + param([string]$Preferred = "python") + + if ($Preferred -ne "python") { + if ((Get-Command $Preferred -ErrorAction SilentlyContinue) -and + -not ((Get-Command $Preferred).Source -like "*WindowsApps*")) { + return $Preferred + } + } + + $candidates = @( + (Join-Path $env:LOCALAPPDATA "Programs\Python\Python313\python.exe"), + (Join-Path $env:LOCALAPPDATA "Programs\Python\Python312\python.exe"), + (Join-Path $env:LOCALAPPDATA "Programs\Python\Python311\python.exe"), + "C:\Program Files\Python312\python.exe", + "C:\Program Files\Python313\python.exe" + ) + + foreach ($candidate in $candidates) { + if (Test-Path $candidate) { + return $candidate + } + } + + $pythonCommand = Get-Command python -ErrorAction SilentlyContinue + if ($pythonCommand -and $pythonCommand.Source -notlike "*WindowsApps*") { + return $pythonCommand.Source + } + + return $null +} + +function Export-VocabFromTokenizerJson { + param([string]$TokenizerJsonPath, [string]$VocabPath) + + $tokenizer = Get-Content $TokenizerJsonPath -Raw | ConvertFrom-Json + $vocab = $tokenizer.model.vocab + if (-not $vocab) { + return $false + } + + $orderedTokens = $vocab.PSObject.Properties | + Sort-Object { [int]$_.Value } | + ForEach-Object { $_.Name } + + $orderedTokens | Set-Content -Path $VocabPath -Encoding utf8 + return $true +} + +function Copy-TokenizerAssets { + param([string]$SourceDir) + + foreach ($name in @("sentencepiece.bpe.model", "spiece.model", "tokenizer.model")) { + $source = Join-Path $SourceDir $name + if (Test-Path $source) { + $destination = Join-Path $modelsDir $name + Copy-Item $source $destination -Force + return @($destination) + } + } + + $tokenizerJsonSource = Join-Path $SourceDir "tokenizer.json" + if (Test-Path $tokenizerJsonSource) { + $tokenizerJsonDestination = Join-Path $modelsDir "tokenizer.json" + Copy-Item $tokenizerJsonSource $tokenizerJsonDestination -Force + $saved = @($tokenizerJsonDestination) + + $vocabPath = Join-Path $modelsDir "vocab.txt" + if (Export-VocabFromTokenizerJson -TokenizerJsonPath $tokenizerJsonDestination -VocabPath $vocabPath) { + $saved += $vocabPath + } + + Write-Warning ( + "No SentencePiece model on Hugging Face; saved WordPiece assets ($( + ($saved | ForEach-Object { Split-Path $_ -Leaf }) -join ', ' + )). TamilOnnxNerRunner uses vocab.txt (WordPiece) when present, otherwise SentencePiece model files." + ) + return $saved + } + + return @() +} + +function Download-WithPowerShell { + Ensure-ModelsDirectory + + $modelPath = Join-Path $modelsDir "model.onnx" + $configPath = Join-Path $modelsDir "config.json" + $labelsPath = Join-Path $modelsDir "ner-labels.txt" + $tempDir = Join-Path $modelsDir "_download_temp" + New-Item -ItemType Directory -Force -Path $tempDir | Out-Null + + try { + Download-HuggingFaceAsset -RelativePath "onnx/model.onnx" -Destination $modelPath + } + catch { + Write-Host "Pre-exported ONNX not found; downloading config and tokenizer for manual export..." + Download-HuggingFaceAsset -RelativePath "config.json" -Destination $configPath + Export-LabelsFromConfig -ConfigPath $configPath -LabelsPath $labelsPath + + $tokenizerJsonPath = Join-Path $modelsDir "tokenizer.json" + try { + Download-HuggingFaceAsset -RelativePath "tokenizer.json" -Destination $tokenizerJsonPath + $vocabPath = Join-Path $modelsDir "vocab.txt" + Export-VocabFromTokenizerJson -TokenizerJsonPath $tokenizerJsonPath -VocabPath $vocabPath | Out-Null + } + catch { + Write-Host "tokenizer.json not available from Hugging Face." + } + + foreach ($name in @("sentencepiece.bpe.model", "spiece.model", "tokenizer.model")) { + try { + Download-HuggingFaceAsset -RelativePath $name -Destination (Join-Path $modelsDir $name) + break + } + catch { + continue + } + } + + $pythonExe = Resolve-PythonExecutable -Preferred $Python + if ($pythonExe) { + throw ( + "Tamil ONNX model is not published on Hugging Face. Re-run with Python export:`n" + + " .\scripts\download-tamil-ner-model.ps1 -Python `"$pythonExe`"" + ) + } + + throw @" +Tamil ONNX model is not published on Hugging Face (onnx/model.onnx returns 404). +Install Python 3.12+ and re-run this script: + winget install Python.Python.3.12 --accept-package-agreements --accept-source-agreements + .\scripts\download-tamil-ner-model.ps1 -Python `"`$env:LOCALAPPDATA\Programs\Python\Python312\python.exe`" +Only config.json, ner-labels.txt, and tokenizer.json were saved under models/ta/. +"@ + } + + Download-HuggingFaceAsset -RelativePath "config.json" -Destination $configPath + Export-LabelsFromConfig -ConfigPath $configPath -LabelsPath $labelsPath + Remove-Item $configPath -Force + + foreach ($name in @("sentencepiece.bpe.model", "spiece.model")) { + try { + Download-HuggingFaceAsset -RelativePath $name -Destination (Join-Path $modelsDir $name) + break + } + catch { + continue + } + } + + Write-Host "" + Write-Host "Tamil NER model assets saved to $modelsDir" +} + +function Download-WithPython { + $pythonExe = Resolve-PythonExecutable -Preferred $Python + if (-not $pythonExe) { + return $false + } + + Write-Host "Using Python: $pythonExe" + Write-Host "Repository root: $repoRoot" + Write-Host "" + + Push-Location $repoRoot + try { + & $pythonExe $scriptPath + if ($LASTEXITCODE -ne 0) { + throw "Tamil model download script failed with exit code $LASTEXITCODE." + } + } + finally { + Pop-Location + } + + return $true +} + +Write-Host "Repository root: $repoRoot" + +if (-not (Download-WithPython)) { + Write-Host "Python export unavailable; downloading pre-exported ONNX assets from Hugging Face..." + Write-Host "" + Download-WithPowerShell +} diff --git a/scripts/download-tamil-ner-model.py b/scripts/download-tamil-ner-model.py new file mode 100644 index 0000000..f53c01d --- /dev/null +++ b/scripts/download-tamil-ner-model.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Download and export prachuryyaIITG/SampurNER_Tamil_IndicBERTv2 to ONNX for the PII Redaction POC.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +MODELS_DIR = REPO_ROOT / "models" / "ta" +MODEL_ID = "prachuryyaIITG/SampurNER_Tamil_IndicBERTv2" +REQUIRED_PACKAGES = ("transformers", "optimum[onnxruntime]", "onnx", "torch") +SENTENCEPIECE_CANDIDATES = ( + "sentencepiece.bpe.model", + "spiece.model", + "tokenizer.model", +) +TOKENIZER_JSON = "tokenizer.json" +VOCAB_TXT = "vocab.txt" + + +def ensure_dependencies() -> None: + try: + import optimum.onnxruntime # noqa: F401 + import transformers # noqa: F401 + except ImportError: + print("Installing Python dependencies (this may take a few minutes)...") + subprocess.check_call( + [sys.executable, "-m", "pip", "install", *REQUIRED_PACKAGES], + stdout=sys.stdout, + stderr=sys.stderr, + ) + + +def copy_sentencepiece_model(source_dir: Path, target_dir: Path) -> Path | None: + for name in SENTENCEPIECE_CANDIDATES: + candidate = source_dir / name + if candidate.exists(): + destination = target_dir / name + shutil.copy(candidate, destination) + return destination + + return None + + +def export_wordpiece_assets(source_dir: Path, target_dir: Path) -> list[Path]: + saved: list[Path] = [] + tokenizer_json = source_dir / TOKENIZER_JSON + if not tokenizer_json.exists(): + return saved + + destination = target_dir / TOKENIZER_JSON + shutil.copy(tokenizer_json, destination) + saved.append(destination) + + with tokenizer_json.open(encoding="utf-8") as tokenizer_file: + tokenizer_data = json.load(tokenizer_file) + + vocab = tokenizer_data.get("model", {}).get("vocab") + if not isinstance(vocab, dict): + return saved + + vocab_path = target_dir / VOCAB_TXT + ordered_tokens = [token for token, _ in sorted(vocab.items(), key=lambda item: item[1])] + vocab_path.write_text("\n".join(ordered_tokens), encoding="utf-8") + saved.append(vocab_path) + return saved + + +def copy_tokenizer_assets(source_dir: Path, target_dir: Path) -> list[Path]: + sentencepiece_path = copy_sentencepiece_model(source_dir, target_dir) + if sentencepiece_path is not None: + return [sentencepiece_path] + + wordpiece_assets = export_wordpiece_assets(source_dir, target_dir) + if wordpiece_assets: + print( + "WARNING: Hugging Face repo has no SentencePiece model; saved WordPiece " + f"assets ({', '.join(path.name for path in wordpiece_assets)}). " + "TamilOnnxNerRunner uses vocab.txt (WordPiece) when present, " + "otherwise SentencePiece model files." + ) + return wordpiece_assets + + raise FileNotFoundError( + f"No tokenizer assets found in {source_dir}. " + f"Expected one of {SENTENCEPIECE_CANDIDATES} or {TOKENIZER_JSON}." + ) + + +def export_model() -> None: + from optimum.onnxruntime import ORTModelForTokenClassification + from transformers import AutoTokenizer + + MODELS_DIR.mkdir(parents=True, exist_ok=True) + temp_dir = MODELS_DIR / "_export_temp" + if temp_dir.exists(): + shutil.rmtree(temp_dir) + temp_dir.mkdir() + + print(f"Exporting {MODEL_ID} to ONNX...") + model = ORTModelForTokenClassification.from_pretrained(MODEL_ID, export=True) + tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) + + model.save_pretrained(temp_dir) + tokenizer.save_pretrained(temp_dir) + + onnx_files = sorted(temp_dir.glob("*.onnx")) + if not onnx_files: + raise FileNotFoundError("Export completed but no .onnx file was produced.") + + target_onnx = MODELS_DIR / "model.onnx" + shutil.copy(onnx_files[0], target_onnx) + + config_path = temp_dir / "config.json" + with config_path.open(encoding="utf-8") as config_file: + config = json.load(config_file) + + id2label = config.get("id2label", {}) + labels = [id2label[str(index)] for index in range(len(id2label))] + labels_path = MODELS_DIR / "ner-labels.txt" + labels_path.write_text("\n".join(labels), encoding="utf-8") + + tokenizer_assets = copy_tokenizer_assets(temp_dir, MODELS_DIR) + shutil.rmtree(temp_dir) + + print() + print("Tamil NER model assets saved:") + print(f" {target_onnx}") + for asset in tokenizer_assets: + print(f" {asset}") + print(f" {labels_path}") + print() + print("Run from repository root:") + print(" dotnet test --filter Category=TamilNer") + + +def main() -> int: + ensure_dependencies() + export_model() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tamil-ner-diagnostic/Program.cs b/scripts/tamil-ner-diagnostic/Program.cs new file mode 100644 index 0000000..66a3a5d --- /dev/null +++ b/scripts/tamil-ner-diagnostic/Program.cs @@ -0,0 +1,27 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.ML.Tokenizers; +using PiiRedaction.Core.Configuration; +using PiiRedaction.Infrastructure.Onnx; + +var modelDir = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "models", "ta")); +if (!Directory.Exists(modelDir)) +{ + modelDir = Path.GetFullPath("models/ta"); +} + +var vocabPath = Path.Combine(modelDir, "vocab.txt"); +var bertOptions = new BertOptions { LowerCaseBeforeTokenization = false, ApplyBasicTokenization = false }; +var tokenizer = BertTokenizer.Create(vocabPath, bertOptions); +var text = "வாடிக்கையாளர் ராஜேஷ் குமார் அழைத்தார்."; +var tokens = tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true); +Console.WriteLine($"count={tokens.Count}"); +foreach (var t in tokens) Console.WriteLine($"{t.Id}\t{t.Value}"); + +var runnerOptions = Options.Create(new PiiRedactionOptions { TamilOnnxModelPath = Path.Combine(modelDir, "model.onnx") }); +using var runner = new TamilOnnxNerRunner(runnerOptions, NullLogger.Instance); +Console.WriteLine($"Available: {runner.IsModelAvailable}"); +foreach (var e in runner.PredictEntities(text)) +{ + Console.WriteLine($"Entity: '{e.Value}' [{e.StartIndex},{e.Length}]"); +} diff --git a/scripts/tamil-ner-diagnostic/tamil-ner-diagnostic.csproj b/scripts/tamil-ner-diagnostic/tamil-ner-diagnostic.csproj new file mode 100644 index 0000000..dd3fd3f --- /dev/null +++ b/scripts/tamil-ner-diagnostic/tamil-ner-diagnostic.csproj @@ -0,0 +1,14 @@ + + + Exe + net10.0 + enable + enable + + + + + + + + diff --git a/src/PiiRedaction.ConsoleApp/DependencyInjection/ServiceCollectionExtensions.cs b/src/PiiRedaction.ConsoleApp/DependencyInjection/ServiceCollectionExtensions.cs index b5229e4..e96b67f 100644 --- a/src/PiiRedaction.ConsoleApp/DependencyInjection/ServiceCollectionExtensions.cs +++ b/src/PiiRedaction.ConsoleApp/DependencyInjection/ServiceCollectionExtensions.cs @@ -31,8 +31,9 @@ public static class ServiceCollectionExtensions services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(provider => provider.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/PiiRedaction.ConsoleApp/Program.cs b/src/PiiRedaction.ConsoleApp/Program.cs index d45f5c3..b16deba 100644 --- a/src/PiiRedaction.ConsoleApp/Program.cs +++ b/src/PiiRedaction.ConsoleApp/Program.cs @@ -5,6 +5,10 @@ using PiiRedaction.ConsoleApp.Samples; using PiiRedaction.Core.Abstractions; using PiiRedaction.Core.Models; +System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance); +Console.InputEncoding = System.Text.Encoding.UTF8; +Console.OutputEncoding = System.Text.Encoding.UTF8; + var interactive = args.Contains("--interactive", StringComparer.OrdinalIgnoreCase); var listSamples = args.Contains("--list", StringComparer.OrdinalIgnoreCase); diff --git a/src/PiiRedaction.ConsoleApp/Samples/PromptDemoRunner.cs b/src/PiiRedaction.ConsoleApp/Samples/PromptDemoRunner.cs index 2d1c28e..3e5eac8 100644 --- a/src/PiiRedaction.ConsoleApp/Samples/PromptDemoRunner.cs +++ b/src/PiiRedaction.ConsoleApp/Samples/PromptDemoRunner.cs @@ -59,7 +59,7 @@ public sealed class PromptDemoRunner Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --sample 2"); Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --name MrTitlePerson"); Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --list"); - Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --interactive"); + Console.WriteLine(" dotnet run --project src/PiiRedaction.ConsoleApp -- --interactive # UTF-8 input recommended for non-ASCII text"); } private static void DisplayDetectedEntities(IReadOnlyList entities) diff --git a/src/PiiRedaction.ConsoleApp/Samples/SamplePromptCatalog.cs b/src/PiiRedaction.ConsoleApp/Samples/SamplePromptCatalog.cs index f9768dd..1f2032c 100644 --- a/src/PiiRedaction.ConsoleApp/Samples/SamplePromptCatalog.cs +++ b/src/PiiRedaction.ConsoleApp/Samples/SamplePromptCatalog.cs @@ -68,6 +68,36 @@ public static class SamplePromptCatalog "Loan number, customer ID, and account number together.", "Please verify LN-100200 for CustomerId CID-3000 on AccountNumber ACC-400500."), + new( + "TamilCustomerNameOnly", + "NER (Tamil)", + "Tamil script person name detected via Tamil ONNX NER.", + "வாடிக்கையாளர் ராஜேஷ் குமார் சேமிப்பு கணக்கில் அங்கீகரிக்கப்படாத பரிவர்த்தனைகளைப் புகாரளித்தார்."), + + new( + "TamilWithPhonePan", + "NER (Tamil) + Regex", + "Tamil script person plus phone and PAN (regex).", + "வாடிக்கையாளர் ராஜேஷ் குமார் தொலைபேசி 9876543210 PAN ABCDE1234F."), + + new( + "TanglishCustomer", + "NER (English/Tanglish)", + "Latin-script Tanglish person name via English ONNX NER.", + "Customer Senthil phone 9876543210 reported a failed UPI transfer."), + + new( + "MixedTamilEnglish", + "NER (Mixed)", + "Code-mixed Tamil and English — both script routers may contribute person spans.", + "வாடிக்கையாளர் Ravi Kumar phone 9876543210 disputed the charge."), + + new( + "TamilFullFinancial", + "NER (Tamil) + Regex + Domain", + "Tamil person with email, phone, loan number, and PAN (canonical demo in Tamil).", + "வாடிக்கையாளர் ராஜேஷ் குமார் மின்னஞ்சல் ravi.kumar@gmail.com தொலைபேசி 9876543210 LoanNumber LN-456789 PAN ABCDE1234F. இந்த வாடிக்கையாளர் புகாரை சுருக்கமாக கூறுங்கள்."), + new( "NoPiiCleanTicket", "Negative", diff --git a/src/PiiRedaction.ConsoleApp/appsettings.json b/src/PiiRedaction.ConsoleApp/appsettings.json index 4d12bf6..cb79f50 100644 --- a/src/PiiRedaction.ConsoleApp/appsettings.json +++ b/src/PiiRedaction.ConsoleApp/appsettings.json @@ -1,5 +1,8 @@ { "PiiRedaction": { - "OnnxModelPath": "models/ner-model.onnx" + "OnnxModelPath": "models/ner-model.onnx", + "EnglishOnnxModelPath": "models/en/ner-model.onnx", + "TamilOnnxModelPath": "models/ta/model.onnx", + "EnableTamilNer": true } } diff --git a/src/PiiRedaction.Core/Configuration/PiiRedactionOptions.cs b/src/PiiRedaction.Core/Configuration/PiiRedactionOptions.cs index b216a8b..ac93f46 100644 --- a/src/PiiRedaction.Core/Configuration/PiiRedactionOptions.cs +++ b/src/PiiRedaction.Core/Configuration/PiiRedactionOptions.cs @@ -5,4 +5,10 @@ public sealed class PiiRedactionOptions public const string SectionName = "PiiRedaction"; public string OnnxModelPath { get; set; } = "models/ner-model.onnx"; + + public string EnglishOnnxModelPath { get; set; } = "models/en/ner-model.onnx"; + + public string TamilOnnxModelPath { get; set; } = "models/ta/model.onnx"; + + public bool EnableTamilNer { get; set; } = true; } diff --git a/src/PiiRedaction.Core/Detection/ScriptComposition.cs b/src/PiiRedaction.Core/Detection/ScriptComposition.cs new file mode 100644 index 0000000..ecaf823 --- /dev/null +++ b/src/PiiRedaction.Core/Detection/ScriptComposition.cs @@ -0,0 +1,9 @@ +namespace PiiRedaction.Core.Detection; + +public enum ScriptComposition +{ + LatinOnly, + TamilOnly, + Mixed, + NoLetters +} diff --git a/src/PiiRedaction.Core/Detection/ScriptRouter.cs b/src/PiiRedaction.Core/Detection/ScriptRouter.cs new file mode 100644 index 0000000..50ac958 --- /dev/null +++ b/src/PiiRedaction.Core/Detection/ScriptRouter.cs @@ -0,0 +1,45 @@ +namespace PiiRedaction.Core.Detection; + +/// +/// Classifies prompt text by script composition to route NER inference. +/// +public sealed class ScriptRouter +{ + private const char TamilRangeStart = '\u0B80'; + private const char TamilRangeEnd = '\u0BFF'; + + public ScriptComposition GetComposition(string text) + { + ArgumentNullException.ThrowIfNull(text); + + var hasLatin = false; + var hasTamil = false; + + foreach (var character in text) + { + if (IsTamilLetter(character)) + { + hasTamil = true; + } + else if (char.IsAsciiLetter(character)) + { + hasLatin = true; + } + + if (hasLatin && hasTamil) + { + return ScriptComposition.Mixed; + } + } + + if (!hasLatin && !hasTamil) + { + return ScriptComposition.NoLetters; + } + + return hasTamil ? ScriptComposition.TamilOnly : ScriptComposition.LatinOnly; + } + + internal static bool IsTamilLetter(char character) => + character is >= TamilRangeStart and <= TamilRangeEnd; +} diff --git a/src/PiiRedaction.Infrastructure/Onnx/BertWordPieceEncoder.cs b/src/PiiRedaction.Infrastructure/Onnx/BertWordPieceEncoder.cs new file mode 100644 index 0000000..211e6fd --- /dev/null +++ b/src/PiiRedaction.Infrastructure/Onnx/BertWordPieceEncoder.cs @@ -0,0 +1,107 @@ +using Microsoft.Extensions.Logging; +using Microsoft.ML.Tokenizers; + +namespace PiiRedaction.Infrastructure.Onnx; + +public sealed class BertWordPieceEncoder : ITokenClassifierEncoder +{ + private readonly BertTokenizer? _tokenizer; + private readonly ILogger _logger; + + public BertWordPieceEncoder(string modelDirectory, ILogger logger) + { + _logger = logger; + _tokenizer = TryLoadTokenizer(modelDirectory); + } + + public bool IsAvailable => _tokenizer is not null; + + public EncodedSequence? Encode(string text, int maxSequenceLength) + { + if (_tokenizer is null) + { + return null; + } + + var encodedTokens = _tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true); + var wordTokens = encodedTokens.Take(Math.Max(0, maxSequenceLength - 2)).ToList(); + if (wordTokens.Count == 0) + { + return null; + } + + var sequenceLength = wordTokens.Count + 2; + var inputIds = new long[sequenceLength]; + var attentionMask = new long[sequenceLength]; + var tokenTypeIds = new long[sequenceLength]; + var offsets = new (int Start, int End)[sequenceLength]; + var tokenIds = new int[sequenceLength]; + + inputIds[0] = _tokenizer.ClassificationTokenId; + attentionMask[0] = 1; + tokenIds[0] = _tokenizer.ClassificationTokenId; + offsets[0] = (0, 0); + + for (var i = 0; i < wordTokens.Count; i++) + { + var token = wordTokens[i]; + var index = i + 1; + inputIds[index] = token.Id; + attentionMask[index] = 1; + tokenIds[index] = token.Id; + offsets[index] = ToCharOffsets(token.Offset, text.Length); + } + + inputIds[sequenceLength - 1] = _tokenizer.SeparatorTokenId; + attentionMask[sequenceLength - 1] = 1; + tokenIds[sequenceLength - 1] = _tokenizer.SeparatorTokenId; + offsets[sequenceLength - 1] = (0, 0); + + return new EncodedSequence(inputIds, attentionMask, tokenTypeIds, offsets, tokenIds, sequenceLength); + } + + public bool IsSpecialToken(int tokenId) => + _tokenizer is not null && + (tokenId == _tokenizer.ClassificationTokenId || + tokenId == _tokenizer.SeparatorTokenId || + tokenId == _tokenizer.PaddingTokenId); + + private BertTokenizer? TryLoadTokenizer(string modelDirectory) + { + var vocabPath = OnnxAssetPathResolver.ResolveAssetPath(Path.Combine(modelDirectory, "vocab.txt")); + if (!File.Exists(vocabPath)) + { + _logger.LogWarning("Tokenizer vocabulary not found at {VocabPath}.", vocabPath); + return null; + } + + try + { + return BertTokenizer.Create(vocabPath, CreateBertOptions(modelDirectory)); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load BERT tokenizer from {VocabPath}.", vocabPath); + return null; + } + } + + private static BertOptions CreateBertOptions(string modelDirectory) + { + var tokenizerJsonPath = OnnxAssetPathResolver.ResolveAssetPath( + Path.Combine(modelDirectory, "tokenizer.json")); + var whitespaceOnlyPretokenization = File.Exists(tokenizerJsonPath); + + return new BertOptions + { + LowerCaseBeforeTokenization = false, + ApplyBasicTokenization = !whitespaceOnlyPretokenization + }; + } + + private static (int Start, int End) ToCharOffsets(Range offset, int textLength) + { + var (start, length) = offset.GetOffsetAndLength(textLength); + return (start, start + length); + } +} diff --git a/src/PiiRedaction.Infrastructure/Onnx/EnglishOnnxNerRunner.cs b/src/PiiRedaction.Infrastructure/Onnx/EnglishOnnxNerRunner.cs new file mode 100644 index 0000000..d70d2d9 --- /dev/null +++ b/src/PiiRedaction.Infrastructure/Onnx/EnglishOnnxNerRunner.cs @@ -0,0 +1,35 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using PiiRedaction.Core.Configuration; +using PiiRedaction.Core.Detection; +using PiiRedaction.Core.Models; + +namespace PiiRedaction.Infrastructure.Onnx; + +public class EnglishOnnxNerRunner : IOnnxNerModelRunner, IDisposable +{ + private readonly OnnxTokenClassifierRunner _runner; + + protected EnglishOnnxNerRunner(IOptions options, ILogger logger) + { + var modelPath = OnnxAssetPathResolver.ResolveModelPath( + options.Value.EnglishOnnxModelPath, + options.Value.OnnxModelPath); + + var modelDirectory = Path.GetDirectoryName(modelPath) ?? Environment.CurrentDirectory; + var labels = OnnxAssetPathResolver.LoadLabels(modelDirectory); + var encoder = new BertWordPieceEncoder(modelDirectory, logger); + _runner = new OnnxTokenClassifierRunner(modelPath, encoder, NerLabelConfig.English, labels, logger); + } + + public EnglishOnnxNerRunner(IOptions options, ILogger logger) + : this(options, (ILogger)logger) + { + } + + public bool IsModelAvailable => _runner.IsAvailable; + + public IReadOnlyList PredictEntities(string text) => _runner.PredictEntities(text); + + public void Dispose() => _runner.Dispose(); +} diff --git a/src/PiiRedaction.Infrastructure/Onnx/ITokenClassifierEncoder.cs b/src/PiiRedaction.Infrastructure/Onnx/ITokenClassifierEncoder.cs new file mode 100644 index 0000000..d14111d --- /dev/null +++ b/src/PiiRedaction.Infrastructure/Onnx/ITokenClassifierEncoder.cs @@ -0,0 +1,18 @@ +namespace PiiRedaction.Infrastructure.Onnx; + +public sealed record EncodedSequence( + long[] InputIds, + long[] AttentionMask, + long[] TokenTypeIds, + (int Start, int End)[] Offsets, + int[] TokenIds, + int SequenceLength); + +public interface ITokenClassifierEncoder +{ + bool IsAvailable { get; } + + EncodedSequence? Encode(string text, int maxSequenceLength); + + bool IsSpecialToken(int tokenId); +} diff --git a/src/PiiRedaction.Infrastructure/Onnx/NerLabelConfig.cs b/src/PiiRedaction.Infrastructure/Onnx/NerLabelConfig.cs new file mode 100644 index 0000000..415e7e3 --- /dev/null +++ b/src/PiiRedaction.Infrastructure/Onnx/NerLabelConfig.cs @@ -0,0 +1,26 @@ +namespace PiiRedaction.Infrastructure.Onnx; + +public sealed class NerLabelConfig +{ + private readonly Func _isPersonLabel; + + private NerLabelConfig(Func isPersonLabel) => _isPersonLabel = isPersonLabel; + + public static NerLabelConfig English { get; } = new(IsEnglishPersonLabel); + + public static NerLabelConfig Tamil { get; } = new(IsTamilPersonLabel); + + public bool IsPersonLabel(string label) => _isPersonLabel(label); + + public bool IsBeginLabel(string label) => label.StartsWith("B-", StringComparison.Ordinal); + + public bool IsInsideLabel(string label) => label.StartsWith("I-", StringComparison.Ordinal); + + private static bool IsEnglishPersonLabel(string label) => + label is "B-PER" or "I-PER" or "B-PERSON" or "I-PERSON" + || (label.EndsWith("-PER", StringComparison.Ordinal) && + (label.StartsWith("B-", StringComparison.Ordinal) || label.StartsWith("I-", StringComparison.Ordinal))); + + private static bool IsTamilPersonLabel(string label) => + label.Contains("person", StringComparison.OrdinalIgnoreCase); +} diff --git a/src/PiiRedaction.Infrastructure/Onnx/OnnxAssetPathResolver.cs b/src/PiiRedaction.Infrastructure/Onnx/OnnxAssetPathResolver.cs new file mode 100644 index 0000000..ff96b7a --- /dev/null +++ b/src/PiiRedaction.Infrastructure/Onnx/OnnxAssetPathResolver.cs @@ -0,0 +1,53 @@ +namespace PiiRedaction.Infrastructure.Onnx; + +public static class OnnxAssetPathResolver +{ + public static string ResolveAssetPath(string configuredPath) + { + if (Path.IsPathRooted(configuredPath) && File.Exists(configuredPath)) + { + return configuredPath; + } + + var directory = new DirectoryInfo(Environment.CurrentDirectory); + while (directory is not null) + { + var candidate = Path.GetFullPath(Path.Combine(directory.FullName, configuredPath)); + if (File.Exists(candidate)) + { + return candidate; + } + + directory = directory.Parent; + } + + return Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, configuredPath)); + } + + public static string ResolveModelPath(string primaryPath, params string[] fallbackPaths) + { + foreach (var candidatePath in new[] { primaryPath }.Concat(fallbackPaths)) + { + var resolved = ResolveAssetPath(candidatePath); + if (File.Exists(resolved)) + { + return resolved; + } + } + + return ResolveAssetPath(primaryPath); + } + + public static string[] LoadLabels(string modelDirectory) + { + var labelsPath = ResolveAssetPath(Path.Combine(modelDirectory, "ner-labels.txt")); + if (!File.Exists(labelsPath)) + { + return []; + } + + return File.ReadAllLines(labelsPath) + .Where(line => !string.IsNullOrWhiteSpace(line)) + .ToArray(); + } +} diff --git a/src/PiiRedaction.Infrastructure/Onnx/OnnxNerModelRunner.cs b/src/PiiRedaction.Infrastructure/Onnx/OnnxNerModelRunner.cs index 2b2646b..26722af 100644 --- a/src/PiiRedaction.Infrastructure/Onnx/OnnxNerModelRunner.cs +++ b/src/PiiRedaction.Infrastructure/Onnx/OnnxNerModelRunner.cs @@ -1,325 +1,16 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Microsoft.ML.OnnxRuntime; -using Microsoft.ML.OnnxRuntime.Tensors; -using Microsoft.ML.Tokenizers; using PiiRedaction.Core.Configuration; -using PiiRedaction.Core.Detection; -using PiiRedaction.Core.Models; namespace PiiRedaction.Infrastructure.Onnx; /// -/// Wraps ONNX Runtime inference for NER models. -/// Tokenization and tensor preparation are isolated here so detectors remain model-agnostic. +/// Backward-compatible alias for . /// -public sealed class OnnxNerModelRunner : IOnnxNerModelRunner, IDisposable +public sealed class OnnxNerModelRunner : EnglishOnnxNerRunner { - private const int MaxSequenceLength = 128; - - private readonly ILogger _logger; - private readonly string _modelPath; - private readonly BertTokenizer? _tokenizer; - private readonly string[] _labels; - private InferenceSession? _session; - public OnnxNerModelRunner(IOptions options, ILogger logger) + : base(options, logger) { - _logger = logger; - _modelPath = ResolveAssetPath(options.Value.OnnxModelPath); - var modelDirectory = Path.GetDirectoryName(_modelPath) ?? Environment.CurrentDirectory; - _labels = LoadLabels(modelDirectory); - _tokenizer = TryLoadTokenizer(modelDirectory); - _session = TryCreateSession(); } - - public bool IsModelAvailable => _session is not null && _tokenizer is not null && _labels.Length > 0; - - public IReadOnlyList PredictEntities(string text) - { - if (!IsModelAvailable || _session is null || _tokenizer is null) - { - return []; - } - - var encodedTokens = _tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true); - var wordTokens = encodedTokens.Take(Math.Max(0, MaxSequenceLength - 2)).ToList(); - if (wordTokens.Count == 0) - { - return []; - } - - var sequenceLength = wordTokens.Count + 2; - var inputIds = new long[sequenceLength]; - var attentionMask = new long[sequenceLength]; - var tokenTypeIds = new long[sequenceLength]; - var offsets = new (int Start, int End)[sequenceLength]; - var tokenIds = new int[sequenceLength]; - - inputIds[0] = _tokenizer.ClassificationTokenId; - attentionMask[0] = 1; - tokenIds[0] = _tokenizer.ClassificationTokenId; - offsets[0] = (0, 0); - - for (var i = 0; i < wordTokens.Count; i++) - { - var token = wordTokens[i]; - var index = i + 1; - inputIds[index] = token.Id; - attentionMask[index] = 1; - tokenIds[index] = token.Id; - offsets[index] = ToCharOffsets(token.Offset, text.Length); - } - - inputIds[sequenceLength - 1] = _tokenizer.SeparatorTokenId; - attentionMask[sequenceLength - 1] = 1; - tokenIds[sequenceLength - 1] = _tokenizer.SeparatorTokenId; - offsets[sequenceLength - 1] = (0, 0); - - var predictedLabelIds = RunInference(inputIds, attentionMask, tokenTypeIds, sequenceLength); - return DecodePersonEntities(text, predictedLabelIds, offsets, tokenIds, sequenceLength); - } - - private int[] RunInference(long[] inputIds, long[] attentionMask, long[] tokenTypeIds, int sequenceLength) - { - var inputIdsTensor = CreateTensor(inputIds, sequenceLength); - var attentionMaskTensor = CreateTensor(attentionMask, sequenceLength); - var inputs = new List - { - NamedOnnxValue.CreateFromTensor(_session!.InputMetadata.Keys.First(key => key.Contains("input_ids", StringComparison.OrdinalIgnoreCase)), inputIdsTensor), - NamedOnnxValue.CreateFromTensor(_session.InputMetadata.Keys.First(key => key.Contains("attention_mask", StringComparison.OrdinalIgnoreCase)), attentionMaskTensor) - }; - - var tokenTypeInputName = _session.InputMetadata.Keys.FirstOrDefault(key => key.Contains("token_type", StringComparison.OrdinalIgnoreCase)); - if (tokenTypeInputName is not null) - { - inputs.Add(NamedOnnxValue.CreateFromTensor(tokenTypeInputName, CreateTensor(tokenTypeIds, sequenceLength))); - } - - using var results = _session.Run(inputs); - var outputName = _session.OutputMetadata.Keys.FirstOrDefault(key => - key.Contains("logits", StringComparison.OrdinalIgnoreCase)) - ?? results.First().Name; - var logits = results.First(result => result.Name == outputName).AsTensor(); - var numLabels = _labels.Length; - var predictions = new int[sequenceLength]; - - for (var tokenIndex = 0; tokenIndex < sequenceLength; tokenIndex++) - { - var bestLabel = 0; - var bestScore = float.MinValue; - - for (var labelIndex = 0; labelIndex < numLabels; labelIndex++) - { - var score = logits[0, tokenIndex, labelIndex]; - if (score > bestScore) - { - bestScore = score; - bestLabel = labelIndex; - } - } - - predictions[tokenIndex] = bestLabel; - } - - return predictions; - } - - private static DenseTensor CreateTensor(long[] values, int sequenceLength) - { - var tensor = new DenseTensor([1, sequenceLength]); - for (var i = 0; i < sequenceLength; i++) - { - tensor[0, i] = values[i]; - } - - return tensor; - } - - private IReadOnlyList DecodePersonEntities( - string text, - int[] predictedLabelIds, - (int Start, int End)[] offsets, - int[] tokenIds, - int sequenceLength) - { - var entities = new List(); - int? entityStart = null; - int? entityEnd = null; - - void FlushEntity() - { - if (!entityStart.HasValue || !entityEnd.HasValue || entityEnd.Value <= entityStart.Value) - { - entityStart = null; - entityEnd = null; - return; - } - - var value = text[entityStart.Value..entityEnd.Value]; - if (!string.IsNullOrWhiteSpace(value)) - { - entities.Add(new PiiEntity( - PiiEntityType.Person, - value, - entityStart.Value, - entityEnd.Value - entityStart.Value, - PiiDetectionSource.Ner)); - } - - entityStart = null; - entityEnd = null; - } - - for (var i = 0; i < sequenceLength; i++) - { - if (IsSpecialToken(tokenIds[i])) - { - FlushEntity(); - continue; - } - - var label = _labels[predictedLabelIds[i]]; - var (start, end) = offsets[i]; - var hasOffset = end > start; - - if (!IsPersonLabel(label)) - { - FlushEntity(); - continue; - } - - if (label.StartsWith("B-", StringComparison.Ordinal)) - { - FlushEntity(); - if (hasOffset) - { - entityStart = start; - entityEnd = end; - } - } - else if (label.StartsWith("I-", StringComparison.Ordinal)) - { - if (!entityStart.HasValue && hasOffset) - { - entityStart = start; - entityEnd = end; - } - else if (hasOffset) - { - entityEnd = Math.Max(entityEnd ?? end, end); - } - } - } - - FlushEntity(); - return entities; - } - - private bool IsSpecialToken(int tokenId) => - tokenId == _tokenizer!.ClassificationTokenId || - tokenId == _tokenizer.SeparatorTokenId || - tokenId == _tokenizer.PaddingTokenId; - - private static (int Start, int End) ToCharOffsets(Range offset, int textLength) - { - var (start, length) = offset.GetOffsetAndLength(textLength); - return (start, start + length); - } - - private static bool IsPersonLabel(string label) => - label is "B-PER" or "I-PER" or "B-PERSON" or "I-PERSON" - || (label.EndsWith("-PER", StringComparison.Ordinal) && - (label.StartsWith("B-", StringComparison.Ordinal) || label.StartsWith("I-", StringComparison.Ordinal))); - - private InferenceSession? TryCreateSession() - { - if (!File.Exists(_modelPath)) - { - _logger.LogWarning( - "ONNX NER model not found at {ModelPath}. Person-name detection will return no results.", - _modelPath); - return null; - } - - if (_tokenizer is null || _labels.Length == 0) - { - _logger.LogWarning( - "Tokenizer vocabulary or label map missing for ONNX NER model at {ModelPath}.", - _modelPath); - return null; - } - - try - { - var session = new InferenceSession(_modelPath); - _logger.LogInformation("ONNX NER model loaded from {ModelPath}.", _modelPath); - return session; - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to load ONNX NER model from {ModelPath}.", _modelPath); - return null; - } - } - - private BertTokenizer? TryLoadTokenizer(string modelDirectory) - { - var vocabPath = ResolveAssetPath(Path.Combine(modelDirectory, "vocab.txt")); - if (!File.Exists(vocabPath)) - { - _logger.LogWarning("Tokenizer vocabulary not found at {VocabPath}.", vocabPath); - return null; - } - - try - { - return BertTokenizer.Create(vocabPath, new BertOptions - { - LowerCaseBeforeTokenization = false - }); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to load BERT tokenizer from {VocabPath}.", vocabPath); - return null; - } - } - - private static string[] LoadLabels(string modelDirectory) - { - var labelsPath = ResolveAssetPath(Path.Combine(modelDirectory, "ner-labels.txt")); - if (!File.Exists(labelsPath)) - { - return []; - } - - return File.ReadAllLines(labelsPath) - .Where(line => !string.IsNullOrWhiteSpace(line)) - .ToArray(); - } - - internal static string ResolveAssetPath(string configuredPath) - { - if (Path.IsPathRooted(configuredPath) && File.Exists(configuredPath)) - { - return configuredPath; - } - - var directory = new DirectoryInfo(Environment.CurrentDirectory); - while (directory is not null) - { - var candidate = Path.GetFullPath(Path.Combine(directory.FullName, configuredPath)); - if (File.Exists(candidate)) - { - return candidate; - } - - directory = directory.Parent; - } - - return Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, configuredPath)); - } - - public void Dispose() => _session?.Dispose(); } diff --git a/src/PiiRedaction.Infrastructure/Onnx/OnnxTokenClassifierRunner.cs b/src/PiiRedaction.Infrastructure/Onnx/OnnxTokenClassifierRunner.cs new file mode 100644 index 0000000..0601ce1 --- /dev/null +++ b/src/PiiRedaction.Infrastructure/Onnx/OnnxTokenClassifierRunner.cs @@ -0,0 +1,229 @@ +using Microsoft.Extensions.Logging; +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.OnnxRuntime.Tensors; +using PiiRedaction.Core.Models; + +namespace PiiRedaction.Infrastructure.Onnx; + +/// +/// Shared ONNX token-classification inference and BIO decoding for NER models. +/// +public sealed class OnnxTokenClassifierRunner : IDisposable +{ + private const int MaxSequenceLength = 128; + + private readonly ITokenClassifierEncoder _encoder; + private readonly NerLabelConfig _labelConfig; + private readonly string[] _labels; + private readonly ILogger _logger; + private readonly string _modelPath; + private InferenceSession? _session; + + public OnnxTokenClassifierRunner( + string modelPath, + ITokenClassifierEncoder encoder, + NerLabelConfig labelConfig, + string[] labels, + ILogger logger) + { + _modelPath = modelPath; + _encoder = encoder; + _labelConfig = labelConfig; + _labels = labels; + _logger = logger; + _session = TryCreateSession(); + } + + public bool IsAvailable => _session is not null && _encoder.IsAvailable && _labels.Length > 0; + + public IReadOnlyList PredictEntities(string text) + { + if (!IsAvailable || _session is null) + { + return []; + } + + var encoded = _encoder.Encode(text, MaxSequenceLength); + if (encoded is null) + { + return []; + } + + var predictedLabelIds = RunInference(encoded); + return DecodePersonEntities(text, predictedLabelIds, encoded); + } + + private int[] RunInference(EncodedSequence encoded) + { + var inputIdsTensor = CreateTensor(encoded.InputIds, encoded.SequenceLength); + var attentionMaskTensor = CreateTensor(encoded.AttentionMask, encoded.SequenceLength); + var inputs = new List + { + NamedOnnxValue.CreateFromTensor( + _session!.InputMetadata.Keys.First(key => key.Contains("input_ids", StringComparison.OrdinalIgnoreCase)), + inputIdsTensor), + NamedOnnxValue.CreateFromTensor( + _session.InputMetadata.Keys.First(key => key.Contains("attention_mask", StringComparison.OrdinalIgnoreCase)), + attentionMaskTensor) + }; + + var tokenTypeInputName = _session.InputMetadata.Keys.FirstOrDefault(key => + key.Contains("token_type", StringComparison.OrdinalIgnoreCase)); + if (tokenTypeInputName is not null) + { + inputs.Add(NamedOnnxValue.CreateFromTensor( + tokenTypeInputName, + CreateTensor(encoded.TokenTypeIds, encoded.SequenceLength))); + } + + using var results = _session.Run(inputs); + var outputName = _session.OutputMetadata.Keys.FirstOrDefault(key => + key.Contains("logits", StringComparison.OrdinalIgnoreCase)) + ?? results.First().Name; + var logits = results.First(result => result.Name == outputName).AsTensor(); + var numLabels = _labels.Length; + var predictions = new int[encoded.SequenceLength]; + + for (var tokenIndex = 0; tokenIndex < encoded.SequenceLength; tokenIndex++) + { + var bestLabel = 0; + var bestScore = float.MinValue; + + for (var labelIndex = 0; labelIndex < numLabels; labelIndex++) + { + var score = logits[0, tokenIndex, labelIndex]; + if (score > bestScore) + { + bestScore = score; + bestLabel = labelIndex; + } + } + + predictions[tokenIndex] = bestLabel; + } + + return predictions; + } + + private static DenseTensor CreateTensor(long[] values, int sequenceLength) + { + var tensor = new DenseTensor([1, sequenceLength]); + for (var i = 0; i < sequenceLength; i++) + { + tensor[0, i] = values[i]; + } + + return tensor; + } + + private IReadOnlyList DecodePersonEntities( + string text, + int[] predictedLabelIds, + EncodedSequence encoded) + { + var entities = new List(); + int? entityStart = null; + int? entityEnd = null; + + void FlushEntity() + { + if (!entityStart.HasValue || !entityEnd.HasValue || entityEnd.Value <= entityStart.Value) + { + entityStart = null; + entityEnd = null; + return; + } + + var value = text[entityStart.Value..entityEnd.Value]; + if (!string.IsNullOrWhiteSpace(value)) + { + entities.Add(new PiiEntity( + PiiEntityType.Person, + value, + entityStart.Value, + entityEnd.Value - entityStart.Value, + PiiDetectionSource.Ner)); + } + + entityStart = null; + entityEnd = null; + } + + for (var i = 0; i < encoded.SequenceLength; i++) + { + if (_encoder.IsSpecialToken(encoded.TokenIds[i])) + { + FlushEntity(); + continue; + } + + var label = _labels[predictedLabelIds[i]]; + var (start, end) = encoded.Offsets[i]; + var hasOffset = end > start; + + if (!_labelConfig.IsPersonLabel(label)) + { + FlushEntity(); + continue; + } + + if (_labelConfig.IsBeginLabel(label)) + { + FlushEntity(); + if (hasOffset) + { + entityStart = start; + entityEnd = end; + } + } + else if (_labelConfig.IsInsideLabel(label)) + { + if (!entityStart.HasValue && hasOffset) + { + entityStart = start; + entityEnd = end; + } + else if (hasOffset) + { + entityEnd = Math.Max(entityEnd ?? end, end); + } + } + } + + FlushEntity(); + return entities; + } + + private InferenceSession? TryCreateSession() + { + if (!File.Exists(_modelPath)) + { + _logger.LogWarning( + "ONNX NER model not found at {ModelPath}. Person-name detection will return no results.", + _modelPath); + return null; + } + + if (!_encoder.IsAvailable || _labels.Length == 0) + { + _logger.LogWarning( + "Tokenizer or label map missing for ONNX NER model at {ModelPath}.", + _modelPath); + return null; + } + + try + { + var session = new InferenceSession(_modelPath); + _logger.LogInformation("ONNX NER model loaded from {ModelPath}.", _modelPath); + return session; + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load ONNX NER model from {ModelPath}.", _modelPath); + return null; + } + } + + public void Dispose() => _session?.Dispose(); +} diff --git a/src/PiiRedaction.Infrastructure/Onnx/RoutingOnnxNerModelRunner.cs b/src/PiiRedaction.Infrastructure/Onnx/RoutingOnnxNerModelRunner.cs new file mode 100644 index 0000000..db8196d --- /dev/null +++ b/src/PiiRedaction.Infrastructure/Onnx/RoutingOnnxNerModelRunner.cs @@ -0,0 +1,105 @@ +using Microsoft.Extensions.Options; +using PiiRedaction.Core.Configuration; +using PiiRedaction.Core.Detection; +using PiiRedaction.Core.Models; + +namespace PiiRedaction.Infrastructure.Onnx; + +/// +/// Routes NER inference to English and/or Tamil ONNX models based on script composition. +/// +public sealed class RoutingOnnxNerModelRunner : IOnnxNerModelRunner +{ + private readonly ScriptRouter _scriptRouter = new(); + private readonly IOnnxNerModelRunner _englishRunner; + private readonly IOnnxNerModelRunner _tamilRunner; + private readonly bool _enableTamilNer; + + public RoutingOnnxNerModelRunner( + EnglishOnnxNerRunner englishRunner, + TamilOnnxNerRunner tamilRunner, + IOptions options) + : this(englishRunner, tamilRunner, options.Value.EnableTamilNer) + { + } + + internal RoutingOnnxNerModelRunner( + IOnnxNerModelRunner englishRunner, + IOnnxNerModelRunner tamilRunner, + bool enableTamilNer) + { + _englishRunner = englishRunner; + _tamilRunner = tamilRunner; + _enableTamilNer = enableTamilNer; + } + + public bool IsModelAvailable => + _englishRunner.IsModelAvailable || (_enableTamilNer && _tamilRunner.IsModelAvailable); + + public IReadOnlyList PredictEntities(string text) + { + var composition = _scriptRouter.GetComposition(text); + var entities = new List(); + + switch (composition) + { + case ScriptComposition.LatinOnly: + if (_englishRunner.IsModelAvailable) + { + entities.AddRange(_englishRunner.PredictEntities(text)); + } + + break; + + case ScriptComposition.TamilOnly: + if (_enableTamilNer && _tamilRunner.IsModelAvailable) + { + entities.AddRange(_tamilRunner.PredictEntities(text)); + } + + break; + + case ScriptComposition.Mixed: + if (_englishRunner.IsModelAvailable) + { + entities.AddRange(_englishRunner.PredictEntities(text)); + } + + if (_enableTamilNer && _tamilRunner.IsModelAvailable) + { + entities.AddRange(_tamilRunner.PredictEntities(text)); + } + + break; + + case ScriptComposition.NoLetters: + break; + } + + return MergePersonSpans(entities); + } + + internal static IReadOnlyList MergePersonSpans(IReadOnlyList entities) + { + if (entities.Count <= 1) + { + return entities; + } + + var accepted = new List(); + foreach (var candidate in entities.OrderByDescending(entity => entity.Length).ThenBy(entity => entity.StartIndex)) + { + if (accepted.Any(existing => Overlaps(existing, candidate))) + { + continue; + } + + accepted.Add(candidate); + } + + return accepted.OrderBy(entity => entity.StartIndex).ToList(); + } + + private static bool Overlaps(PiiEntity left, PiiEntity right) => + left.StartIndex < right.EndIndex && right.StartIndex < left.EndIndex; +} diff --git a/src/PiiRedaction.Infrastructure/Onnx/SentencePieceEncoder.cs b/src/PiiRedaction.Infrastructure/Onnx/SentencePieceEncoder.cs new file mode 100644 index 0000000..06d3c95 --- /dev/null +++ b/src/PiiRedaction.Infrastructure/Onnx/SentencePieceEncoder.cs @@ -0,0 +1,102 @@ +using Microsoft.Extensions.Logging; +using Microsoft.ML.Tokenizers; + +namespace PiiRedaction.Infrastructure.Onnx; + +public sealed class SentencePieceEncoder : ITokenClassifierEncoder +{ + private readonly SentencePieceTokenizer? _tokenizer; + private readonly ILogger _logger; + + public SentencePieceEncoder(string modelDirectory, ILogger logger) + { + _logger = logger; + _tokenizer = TryLoadTokenizer(modelDirectory); + } + + public bool IsAvailable => _tokenizer is not null; + + public EncodedSequence? Encode(string text, int maxSequenceLength) + { + if (_tokenizer is null) + { + return null; + } + + var encodedTokens = _tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true); + var wordTokens = encodedTokens.Take(Math.Max(0, maxSequenceLength - 2)).ToList(); + if (wordTokens.Count == 0) + { + return null; + } + + var sequenceLength = wordTokens.Count + 2; + var inputIds = new long[sequenceLength]; + var attentionMask = new long[sequenceLength]; + var tokenTypeIds = new long[sequenceLength]; + var offsets = new (int Start, int End)[sequenceLength]; + var tokenIds = new int[sequenceLength]; + + inputIds[0] = _tokenizer.BeginningOfSentenceId; + attentionMask[0] = 1; + tokenIds[0] = _tokenizer.BeginningOfSentenceId; + offsets[0] = (0, 0); + + for (var i = 0; i < wordTokens.Count; i++) + { + var token = wordTokens[i]; + var index = i + 1; + inputIds[index] = token.Id; + attentionMask[index] = 1; + tokenIds[index] = token.Id; + offsets[index] = ToCharOffsets(token.Offset, text.Length); + } + + inputIds[sequenceLength - 1] = _tokenizer.EndOfSentenceId; + attentionMask[sequenceLength - 1] = 1; + tokenIds[sequenceLength - 1] = _tokenizer.EndOfSentenceId; + offsets[sequenceLength - 1] = (0, 0); + + return new EncodedSequence(inputIds, attentionMask, tokenTypeIds, offsets, tokenIds, sequenceLength); + } + + public bool IsSpecialToken(int tokenId) => + _tokenizer is not null && + (tokenId == _tokenizer.BeginningOfSentenceId || + tokenId == _tokenizer.EndOfSentenceId || + tokenId == _tokenizer.UnknownId); + + private SentencePieceTokenizer? TryLoadTokenizer(string modelDirectory) + { + foreach (var fileName in new[] { "sentencepiece.bpe.model", "spiece.model", "tokenizer.model" }) + { + var modelPath = OnnxAssetPathResolver.ResolveAssetPath(Path.Combine(modelDirectory, fileName)); + if (!File.Exists(modelPath)) + { + continue; + } + + try + { + using var stream = File.OpenRead(modelPath); + return SentencePieceTokenizer.Create(stream, addBeginningOfSentence: false, addEndOfSentence: false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load SentencePiece tokenizer from {ModelPath}.", modelPath); + return null; + } + } + + _logger.LogWarning( + "SentencePiece model not found in {ModelDirectory}. Expected sentencepiece.bpe.model or spiece.model.", + modelDirectory); + return null; + } + + private static (int Start, int End) ToCharOffsets(Range offset, int textLength) + { + var (start, length) = offset.GetOffsetAndLength(textLength); + return (start, start + length); + } +} diff --git a/src/PiiRedaction.Infrastructure/Onnx/TamilOnnxNerRunner.cs b/src/PiiRedaction.Infrastructure/Onnx/TamilOnnxNerRunner.cs new file mode 100644 index 0000000..54c4150 --- /dev/null +++ b/src/PiiRedaction.Infrastructure/Onnx/TamilOnnxNerRunner.cs @@ -0,0 +1,27 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using PiiRedaction.Core.Configuration; +using PiiRedaction.Core.Detection; +using PiiRedaction.Core.Models; + +namespace PiiRedaction.Infrastructure.Onnx; + +public sealed class TamilOnnxNerRunner : IOnnxNerModelRunner, IDisposable +{ + private readonly OnnxTokenClassifierRunner _runner; + + public TamilOnnxNerRunner(IOptions options, ILogger logger) + { + var modelPath = OnnxAssetPathResolver.ResolveModelPath(options.Value.TamilOnnxModelPath); + var modelDirectory = Path.GetDirectoryName(modelPath) ?? Environment.CurrentDirectory; + var labels = OnnxAssetPathResolver.LoadLabels(modelDirectory); + var encoder = TokenClassifierEncoderFactory.Create(modelDirectory, logger); + _runner = new OnnxTokenClassifierRunner(modelPath, encoder, NerLabelConfig.Tamil, labels, logger); + } + + public bool IsModelAvailable => _runner.IsAvailable; + + public IReadOnlyList PredictEntities(string text) => _runner.PredictEntities(text); + + public void Dispose() => _runner.Dispose(); +} diff --git a/src/PiiRedaction.Infrastructure/Onnx/TokenClassifierEncoderFactory.cs b/src/PiiRedaction.Infrastructure/Onnx/TokenClassifierEncoderFactory.cs new file mode 100644 index 0000000..3072fc5 --- /dev/null +++ b/src/PiiRedaction.Infrastructure/Onnx/TokenClassifierEncoderFactory.cs @@ -0,0 +1,40 @@ +using Microsoft.Extensions.Logging; + +namespace PiiRedaction.Infrastructure.Onnx; + +public static class TokenClassifierEncoderFactory +{ + private static readonly string[] SentencePieceFileNames = + ["sentencepiece.bpe.model", "spiece.model", "tokenizer.model"]; + + public static ITokenClassifierEncoder Create(string modelDirectory, ILogger logger) + { + var vocabPath = OnnxAssetPathResolver.ResolveAssetPath(Path.Combine(modelDirectory, "vocab.txt")); + if (File.Exists(vocabPath)) + { + logger.LogInformation( + "Using WordPiece tokenizer (vocab.txt) from {ModelDirectory}.", + modelDirectory); + return new BertWordPieceEncoder(modelDirectory, logger); + } + + foreach (var fileName in SentencePieceFileNames) + { + var sentencePiecePath = OnnxAssetPathResolver.ResolveAssetPath( + Path.Combine(modelDirectory, fileName)); + if (File.Exists(sentencePiecePath)) + { + logger.LogInformation( + "Using SentencePiece tokenizer ({FileName}) from {ModelDirectory}.", + fileName, + modelDirectory); + return new SentencePieceEncoder(modelDirectory, logger); + } + } + + logger.LogWarning( + "No tokenizer assets found in {ModelDirectory}. Expected vocab.txt or a SentencePiece model file.", + modelDirectory); + return new BertWordPieceEncoder(modelDirectory, logger); + } +} diff --git a/src/PiiRedaction.Infrastructure/PiiRedaction.Infrastructure.csproj b/src/PiiRedaction.Infrastructure/PiiRedaction.Infrastructure.csproj index 8b1696c..ad4a12f 100644 --- a/src/PiiRedaction.Infrastructure/PiiRedaction.Infrastructure.csproj +++ b/src/PiiRedaction.Infrastructure/PiiRedaction.Infrastructure.csproj @@ -19,4 +19,8 @@ enable + + + + diff --git a/src/PiiRedaction.TestHarness.Wpf/App.xaml b/src/PiiRedaction.TestHarness.Wpf/App.xaml new file mode 100644 index 0000000..77fe965 --- /dev/null +++ b/src/PiiRedaction.TestHarness.Wpf/App.xaml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + diff --git a/src/PiiRedaction.TestHarness.Wpf/App.xaml.cs b/src/PiiRedaction.TestHarness.Wpf/App.xaml.cs new file mode 100644 index 0000000..bdb531d --- /dev/null +++ b/src/PiiRedaction.TestHarness.Wpf/App.xaml.cs @@ -0,0 +1,57 @@ +using System.IO; +using System.Windows; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using PiiRedaction.TestHarness.Wpf.DependencyInjection; +using PiiRedaction.TestHarness.Wpf.Services; +using PiiRedaction.TestHarness.Wpf.ViewModels; + +namespace PiiRedaction.TestHarness.Wpf; + +public partial class App : Application +{ + private IHost? _host; + + protected override async void OnStartup(StartupEventArgs e) + { + base.OnStartup(e); + + _host = Host.CreateDefaultBuilder() + .ConfigureAppConfiguration((_, configuration) => + { + configuration.SetBasePath(AppContext.BaseDirectory); + configuration.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true); + configuration.AddEnvironmentVariables(); + }) + .ConfigureServices((context, services) => + { + services.AddPiiRedactionServices(context.Configuration); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + }) + .Build(); + + Directory.SetCurrentDirectory(AppContext.BaseDirectory); + + await _host.StartAsync().ConfigureAwait(true); + + var mainWindow = _host.Services.GetRequiredService(); + mainWindow.Show(); + } + + protected override async void OnExit(ExitEventArgs e) + { + if (_host is not null) + { + await _host.StopAsync().ConfigureAwait(true); + _host.Dispose(); + } + + base.OnExit(e); + } +} diff --git a/src/PiiRedaction.TestHarness.Wpf/Converters/ValueConverters.cs b/src/PiiRedaction.TestHarness.Wpf/Converters/ValueConverters.cs new file mode 100644 index 0000000..cf7d4af --- /dev/null +++ b/src/PiiRedaction.TestHarness.Wpf/Converters/ValueConverters.cs @@ -0,0 +1,71 @@ +using System.Globalization; +using System.Windows; +using System.Windows.Data; +using System.Windows.Media; +using PiiRedaction.Core.Detection; + +namespace PiiRedaction.TestHarness.Wpf.Converters; + +public sealed class ScriptCompositionToBrushConverter : IValueConverter +{ + public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture) + { + if (value is not ScriptComposition composition) + { + return Brushes.Gray; + } + + return composition switch + { + ScriptComposition.LatinOnly => new SolidColorBrush(Color.FromRgb(37, 99, 235)), + ScriptComposition.TamilOnly => new SolidColorBrush(Color.FromRgb(124, 58, 237)), + ScriptComposition.Mixed => new SolidColorBrush(Color.FromRgb(217, 119, 6)), + ScriptComposition.NoLetters => new SolidColorBrush(Color.FromRgb(107, 114, 128)), + _ => Brushes.Gray + }; + } + + public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) => + throw new NotSupportedException(); +} + +public sealed class BoolToVisibilityConverter : IValueConverter +{ + public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture) => + value is true ? Visibility.Visible : Visibility.Collapsed; + + public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) => + throw new NotSupportedException(); +} + +public sealed class StringNotEmptyToVisibilityConverter : IValueConverter +{ + public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture) => + value is string text && !string.IsNullOrWhiteSpace(text) + ? Visibility.Visible + : Visibility.Collapsed; + + public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) => + throw new NotSupportedException(); +} + +public sealed class PassFailBrushConverter : IValueConverter +{ + public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture) + { + if (value is true) + { + return new SolidColorBrush(Color.FromRgb(22, 163, 74)); + } + + if (value is false) + { + return new SolidColorBrush(Color.FromRgb(220, 38, 38)); + } + + return Brushes.Gray; + } + + public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) => + throw new NotSupportedException(); +} diff --git a/src/PiiRedaction.TestHarness.Wpf/DependencyInjection/ServiceCollectionExtensions.cs b/src/PiiRedaction.TestHarness.Wpf/DependencyInjection/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..24fd93d --- /dev/null +++ b/src/PiiRedaction.TestHarness.Wpf/DependencyInjection/ServiceCollectionExtensions.cs @@ -0,0 +1,42 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.AI; +using PiiRedaction.Core.Abstractions; +using PiiRedaction.Core.Configuration; +using PiiRedaction.Core.Detection; +using PiiRedaction.Core.Redaction; +using PiiRedaction.Core.Sanitization; +using PiiRedaction.Infrastructure.Llm; +using PiiRedaction.Infrastructure.Onnx; + +namespace PiiRedaction.TestHarness.Wpf.DependencyInjection; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddPiiRedactionServices(this IServiceCollection services, IConfiguration configuration) + { + services.Configure(configuration.GetSection(PiiRedactionOptions.SectionName)); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + services.AddSingleton(provider => new CompositePiiDetector( + [ + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService() + ])); + + services.AddSingleton(); + services.AddSingleton(); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/PiiRedaction.TestHarness.Wpf/MainWindow.xaml b/src/PiiRedaction.TestHarness.Wpf/MainWindow.xaml new file mode 100644 index 0000000..66c80e8 --- /dev/null +++ b/src/PiiRedaction.TestHarness.Wpf/MainWindow.xaml @@ -0,0 +1,311 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +