Add Tamil NER routing and WPF test harness for POC validation.

Introduce dual-script ONNX NER routing (English/Tamil/mixed), Tamil console samples and integration tests, model download scripts, and a resizable WPF MVVM harness with click-to-load prompts, batch validation, and runtime-adjustable detection panels.

scripts/download-tamil-ner-model.ps1

@@ -0,0 +1,229 @@
+# Downloads prachuryyaIITG/SampurNER_Tamil_IndicBERTv2 ONNX assets to models/ta/ for the PII Redaction POC.
+param(
+    [string]$Python = "python"
+)
+
+$ErrorActionPreference = "Stop"
+$repoRoot = Split-Path -Parent $PSScriptRoot
+$modelsDir = Join-Path (Join-Path $repoRoot "models") "ta"
+$scriptPath = Join-Path $PSScriptRoot "download-tamil-ner-model.py"
+$modelId = "prachuryyaIITG/SampurNER_Tamil_IndicBERTv2"
+$baseUrl = "https://huggingface.co/$modelId/resolve/main"
+
+function Ensure-ModelsDirectory {
+    New-Item -ItemType Directory -Force -Path $modelsDir | Out-Null
+}
+
+function Download-HuggingFaceAsset {
+    param(
+        [string]$RelativePath,
+        [string]$Destination
+    )
+
+    $url = "$baseUrl/$RelativePath"
+    Write-Host "Downloading $url"
+    Invoke-WebRequest -Uri $url -OutFile $Destination -UseBasicParsing
+}
+
+function Export-LabelsFromConfig {
+    param([string]$ConfigPath, [string]$LabelsPath)
+
+    $config = Get-Content $ConfigPath -Raw | ConvertFrom-Json
+    $labelMap = @{}
+    foreach ($property in $config.id2label.PSObject.Properties) {
+        $labelMap[[int]$property.Name] = [string]$property.Value
+    }
+
+    $labels = for ($index = 0; $index -lt $labelMap.Count; $index++) {
+        $labelMap[$index]
+    }
+
+    $labels | Set-Content -Path $LabelsPath -Encoding utf8
+}
+
+function Resolve-PythonExecutable {
+    param([string]$Preferred = "python")
+
+    if ($Preferred -ne "python") {
+        if ((Get-Command $Preferred -ErrorAction SilentlyContinue) -and
+            -not ((Get-Command $Preferred).Source -like "*WindowsApps*")) {
+            return $Preferred
+        }
+    }
+
+    $candidates = @(
+        (Join-Path $env:LOCALAPPDATA "Programs\Python\Python313\python.exe"),
+        (Join-Path $env:LOCALAPPDATA "Programs\Python\Python312\python.exe"),
+        (Join-Path $env:LOCALAPPDATA "Programs\Python\Python311\python.exe"),
+        "C:\Program Files\Python312\python.exe",
+        "C:\Program Files\Python313\python.exe"
+    )
+
+    foreach ($candidate in $candidates) {
+        if (Test-Path $candidate) {
+            return $candidate
+        }
+    }
+
+    $pythonCommand = Get-Command python -ErrorAction SilentlyContinue
+    if ($pythonCommand -and $pythonCommand.Source -notlike "*WindowsApps*") {
+        return $pythonCommand.Source
+    }
+
+    return $null
+}
+
+function Export-VocabFromTokenizerJson {
+    param([string]$TokenizerJsonPath, [string]$VocabPath)
+
+    $tokenizer = Get-Content $TokenizerJsonPath -Raw | ConvertFrom-Json
+    $vocab = $tokenizer.model.vocab
+    if (-not $vocab) {
+        return $false
+    }
+
+    $orderedTokens = $vocab.PSObject.Properties |
+        Sort-Object { [int]$_.Value } |
+        ForEach-Object { $_.Name }
+
+    $orderedTokens | Set-Content -Path $VocabPath -Encoding utf8
+    return $true
+}
+
+function Copy-TokenizerAssets {
+    param([string]$SourceDir)
+
+    foreach ($name in @("sentencepiece.bpe.model", "spiece.model", "tokenizer.model")) {
+        $source = Join-Path $SourceDir $name
+        if (Test-Path $source) {
+            $destination = Join-Path $modelsDir $name
+            Copy-Item $source $destination -Force
+            return @($destination)
+        }
+    }
+
+    $tokenizerJsonSource = Join-Path $SourceDir "tokenizer.json"
+    if (Test-Path $tokenizerJsonSource) {
+        $tokenizerJsonDestination = Join-Path $modelsDir "tokenizer.json"
+        Copy-Item $tokenizerJsonSource $tokenizerJsonDestination -Force
+        $saved = @($tokenizerJsonDestination)
+
+        $vocabPath = Join-Path $modelsDir "vocab.txt"
+        if (Export-VocabFromTokenizerJson -TokenizerJsonPath $tokenizerJsonDestination -VocabPath $vocabPath) {
+            $saved += $vocabPath
+        }
+
+        Write-Warning (
+            "No SentencePiece model on Hugging Face; saved WordPiece assets ($(
+                ($saved | ForEach-Object { Split-Path $_ -Leaf }) -join ', '
+            )). TamilOnnxNerRunner uses vocab.txt (WordPiece) when present, otherwise SentencePiece model files."
+        )
+        return $saved
+    }
+
+    return @()
+}
+
+function Download-WithPowerShell {
+    Ensure-ModelsDirectory
+
+    $modelPath = Join-Path $modelsDir "model.onnx"
+    $configPath = Join-Path $modelsDir "config.json"
+    $labelsPath = Join-Path $modelsDir "ner-labels.txt"
+    $tempDir = Join-Path $modelsDir "_download_temp"
+    New-Item -ItemType Directory -Force -Path $tempDir | Out-Null
+
+    try {
+        Download-HuggingFaceAsset -RelativePath "onnx/model.onnx" -Destination $modelPath
+    }
+    catch {
+        Write-Host "Pre-exported ONNX not found; downloading config and tokenizer for manual export..."
+        Download-HuggingFaceAsset -RelativePath "config.json" -Destination $configPath
+        Export-LabelsFromConfig -ConfigPath $configPath -LabelsPath $labelsPath
+
+        $tokenizerJsonPath = Join-Path $modelsDir "tokenizer.json"
+        try {
+            Download-HuggingFaceAsset -RelativePath "tokenizer.json" -Destination $tokenizerJsonPath
+            $vocabPath = Join-Path $modelsDir "vocab.txt"
+            Export-VocabFromTokenizerJson -TokenizerJsonPath $tokenizerJsonPath -VocabPath $vocabPath | Out-Null
+        }
+        catch {
+            Write-Host "tokenizer.json not available from Hugging Face."
+        }
+
+        foreach ($name in @("sentencepiece.bpe.model", "spiece.model", "tokenizer.model")) {
+            try {
+                Download-HuggingFaceAsset -RelativePath $name -Destination (Join-Path $modelsDir $name)
+                break
+            }
+            catch {
+                continue
+            }
+        }
+
+        $pythonExe = Resolve-PythonExecutable -Preferred $Python
+        if ($pythonExe) {
+            throw (
+                "Tamil ONNX model is not published on Hugging Face. Re-run with Python export:`n" +
+                "  .\scripts\download-tamil-ner-model.ps1 -Python `"$pythonExe`""
+            )
+        }
+
+        throw @"
+Tamil ONNX model is not published on Hugging Face (onnx/model.onnx returns 404).
+Install Python 3.12+ and re-run this script:
+  winget install Python.Python.3.12 --accept-package-agreements --accept-source-agreements
+  .\scripts\download-tamil-ner-model.ps1 -Python `"`$env:LOCALAPPDATA\Programs\Python\Python312\python.exe`"
+Only config.json, ner-labels.txt, and tokenizer.json were saved under models/ta/.
+"@
+    }
+
+    Download-HuggingFaceAsset -RelativePath "config.json" -Destination $configPath
+    Export-LabelsFromConfig -ConfigPath $configPath -LabelsPath $labelsPath
+    Remove-Item $configPath -Force
+
+    foreach ($name in @("sentencepiece.bpe.model", "spiece.model")) {
+        try {
+            Download-HuggingFaceAsset -RelativePath $name -Destination (Join-Path $modelsDir $name)
+            break
+        }
+        catch {
+            continue
+        }
+    }
+
+    Write-Host ""
+    Write-Host "Tamil NER model assets saved to $modelsDir"
+}
+
+function Download-WithPython {
+    $pythonExe = Resolve-PythonExecutable -Preferred $Python
+    if (-not $pythonExe) {
+        return $false
+    }
+
+    Write-Host "Using Python: $pythonExe"
+    Write-Host "Repository root: $repoRoot"
+    Write-Host ""
+
+    Push-Location $repoRoot
+    try {
+        & $pythonExe $scriptPath
+        if ($LASTEXITCODE -ne 0) {
+            throw "Tamil model download script failed with exit code $LASTEXITCODE."
+        }
+    }
+    finally {
+        Pop-Location
+    }
+
+    return $true
+}
+
+Write-Host "Repository root: $repoRoot"
+
+if (-not (Download-WithPython)) {
+    Write-Host "Python export unavailable; downloading pre-exported ONNX assets from Hugging Face..."
+    Write-Host ""
+    Download-WithPowerShell
+}

scripts/download-tamil-ner-model.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""Download and export prachuryyaIITG/SampurNER_Tamil_IndicBERTv2 to ONNX for the PII Redaction POC."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+MODELS_DIR = REPO_ROOT / "models" / "ta"
+MODEL_ID = "prachuryyaIITG/SampurNER_Tamil_IndicBERTv2"
+REQUIRED_PACKAGES = ("transformers", "optimum[onnxruntime]", "onnx", "torch")
+SENTENCEPIECE_CANDIDATES = (
+    "sentencepiece.bpe.model",
+    "spiece.model",
+    "tokenizer.model",
+)
+TOKENIZER_JSON = "tokenizer.json"
+VOCAB_TXT = "vocab.txt"
+
+
+def ensure_dependencies() -> None:
+    try:
+        import optimum.onnxruntime  # noqa: F401
+        import transformers  # noqa: F401
+    except ImportError:
+        print("Installing Python dependencies (this may take a few minutes)...")
+        subprocess.check_call(
+            [sys.executable, "-m", "pip", "install", *REQUIRED_PACKAGES],
+            stdout=sys.stdout,
+            stderr=sys.stderr,
+        )
+
+
+def copy_sentencepiece_model(source_dir: Path, target_dir: Path) -> Path | None:
+    for name in SENTENCEPIECE_CANDIDATES:
+        candidate = source_dir / name
+        if candidate.exists():
+            destination = target_dir / name
+            shutil.copy(candidate, destination)
+            return destination
+
+    return None
+
+
+def export_wordpiece_assets(source_dir: Path, target_dir: Path) -> list[Path]:
+    saved: list[Path] = []
+    tokenizer_json = source_dir / TOKENIZER_JSON
+    if not tokenizer_json.exists():
+        return saved
+
+    destination = target_dir / TOKENIZER_JSON
+    shutil.copy(tokenizer_json, destination)
+    saved.append(destination)
+
+    with tokenizer_json.open(encoding="utf-8") as tokenizer_file:
+        tokenizer_data = json.load(tokenizer_file)
+
+    vocab = tokenizer_data.get("model", {}).get("vocab")
+    if not isinstance(vocab, dict):
+        return saved
+
+    vocab_path = target_dir / VOCAB_TXT
+    ordered_tokens = [token for token, _ in sorted(vocab.items(), key=lambda item: item[1])]
+    vocab_path.write_text("\n".join(ordered_tokens), encoding="utf-8")
+    saved.append(vocab_path)
+    return saved
+
+
+def copy_tokenizer_assets(source_dir: Path, target_dir: Path) -> list[Path]:
+    sentencepiece_path = copy_sentencepiece_model(source_dir, target_dir)
+    if sentencepiece_path is not None:
+        return [sentencepiece_path]
+
+    wordpiece_assets = export_wordpiece_assets(source_dir, target_dir)
+    if wordpiece_assets:
+        print(
+            "WARNING: Hugging Face repo has no SentencePiece model; saved WordPiece "
+            f"assets ({', '.join(path.name for path in wordpiece_assets)}). "
+            "TamilOnnxNerRunner uses vocab.txt (WordPiece) when present, "
+            "otherwise SentencePiece model files."
+        )
+        return wordpiece_assets
+
+    raise FileNotFoundError(
+        f"No tokenizer assets found in {source_dir}. "
+        f"Expected one of {SENTENCEPIECE_CANDIDATES} or {TOKENIZER_JSON}."
+    )
+
+
+def export_model() -> None:
+    from optimum.onnxruntime import ORTModelForTokenClassification
+    from transformers import AutoTokenizer
+
+    MODELS_DIR.mkdir(parents=True, exist_ok=True)
+    temp_dir = MODELS_DIR / "_export_temp"
+    if temp_dir.exists():
+        shutil.rmtree(temp_dir)
+    temp_dir.mkdir()
+
+    print(f"Exporting {MODEL_ID} to ONNX...")
+    model = ORTModelForTokenClassification.from_pretrained(MODEL_ID, export=True)
+    tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+
+    model.save_pretrained(temp_dir)
+    tokenizer.save_pretrained(temp_dir)
+
+    onnx_files = sorted(temp_dir.glob("*.onnx"))
+    if not onnx_files:
+        raise FileNotFoundError("Export completed but no .onnx file was produced.")
+
+    target_onnx = MODELS_DIR / "model.onnx"
+    shutil.copy(onnx_files[0], target_onnx)
+
+    config_path = temp_dir / "config.json"
+    with config_path.open(encoding="utf-8") as config_file:
+        config = json.load(config_file)
+
+    id2label = config.get("id2label", {})
+    labels = [id2label[str(index)] for index in range(len(id2label))]
+    labels_path = MODELS_DIR / "ner-labels.txt"
+    labels_path.write_text("\n".join(labels), encoding="utf-8")
+
+    tokenizer_assets = copy_tokenizer_assets(temp_dir, MODELS_DIR)
+    shutil.rmtree(temp_dir)
+
+    print()
+    print("Tamil NER model assets saved:")
+    print(f"  {target_onnx}")
+    for asset in tokenizer_assets:
+        print(f"  {asset}")
+    print(f"  {labels_path}")
+    print()
+    print("Run from repository root:")
+    print("  dotnet test --filter Category=TamilNer")
+
+
+def main() -> int:
+    ensure_dependencies()
+    export_model()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/tamil-ner-diagnostic/Program.cs

@@ -0,0 +1,27 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Microsoft.ML.Tokenizers;
+using PiiRedaction.Core.Configuration;
+using PiiRedaction.Infrastructure.Onnx;
+
+var modelDir = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "models", "ta"));
+if (!Directory.Exists(modelDir))
+{
+    modelDir = Path.GetFullPath("models/ta");
+}
+
+var vocabPath = Path.Combine(modelDir, "vocab.txt");
+var bertOptions = new BertOptions { LowerCaseBeforeTokenization = false, ApplyBasicTokenization = false };
+var tokenizer = BertTokenizer.Create(vocabPath, bertOptions);
+var text = "வாடிக்கையாளர் ராஜேஷ் குமார் அழைத்தார்.";
+var tokens = tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true);
+Console.WriteLine($"count={tokens.Count}");
+foreach (var t in tokens) Console.WriteLine($"{t.Id}\t{t.Value}");
+
+var runnerOptions = Options.Create(new PiiRedactionOptions { TamilOnnxModelPath = Path.Combine(modelDir, "model.onnx") });
+using var runner = new TamilOnnxNerRunner(runnerOptions, NullLogger<TamilOnnxNerRunner>.Instance);
+Console.WriteLine($"Available: {runner.IsModelAvailable}");
+foreach (var e in runner.PredictEntities(text))
+{
+    Console.WriteLine($"Entity: '{e.Value}' [{e.StartIndex},{e.Length}]");
+}

scripts/tamil-ner-diagnostic/tamil-ner-diagnostic.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\PiiRedaction.Infrastructure\PiiRedaction.Infrastructure.csproj" />
+  </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.ML.Tokenizers" Version="2.0.0" />
+  </ItemGroup>
+</Project>