Add PII redaction POC for secure LLM prompting.

Implements detect-redact-sanitize pipeline with regex, domain rules, and ONNX NER before the LLM boundary, plus NUnit tests and Xenovex push documentation. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-07 13:05:07 +05:30
commit dfc81dea28
60 changed files with 3283 additions and 0 deletions
--- a/scripts/download-ner-model.py
+++ b/scripts/download-ner-model.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""Download and export dslim/bert-base-NER to ONNX for the PII Redaction POC."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+MODELS_DIR = REPO_ROOT / "models"
+MODEL_ID = "dslim/bert-base-NER"
+REQUIRED_PACKAGES = ("transformers", "optimum[onnxruntime]", "onnx", "torch")
+
+
+def ensure_dependencies() -> None:
+    try:
+        import optimum.onnxruntime  # noqa: F401
+        import transformers  # noqa: F401
+    except ImportError:
+        print("Installing Python dependencies (this may take a few minutes)...")
+        subprocess.check_call(
+            [sys.executable, "-m", "pip", "install", *REQUIRED_PACKAGES],
+            stdout=sys.stdout,
+            stderr=sys.stderr,
+        )
+
+
+def export_model() -> None:
+    from optimum.onnxruntime import ORTModelForTokenClassification
+    from transformers import AutoTokenizer
+
+    MODELS_DIR.mkdir(parents=True, exist_ok=True)
+    temp_dir = MODELS_DIR / "_export_temp"
+    if temp_dir.exists():
+        shutil.rmtree(temp_dir)
+    temp_dir.mkdir()
+
+    print(f"Exporting {MODEL_ID} to ONNX...")
+    model = ORTModelForTokenClassification.from_pretrained(MODEL_ID, export=True)
+    tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+
+    model.save_pretrained(temp_dir)
+    tokenizer.save_pretrained(temp_dir)
+
+    onnx_files = sorted(temp_dir.glob("*.onnx"))
+    if not onnx_files:
+        raise FileNotFoundError("Export completed but no .onnx file was produced.")
+
+    target_onnx = MODELS_DIR / "ner-model.onnx"
+    shutil.copy(onnx_files[0], target_onnx)
+    shutil.copy(temp_dir / "vocab.txt", MODELS_DIR / "vocab.txt")
+
+    config_path = temp_dir / "config.json"
+    with config_path.open(encoding="utf-8") as config_file:
+        config = json.load(config_file)
+
+    id2label = config.get("id2label", {})
+    labels = [id2label[str(index)] for index in range(len(id2label))]
+    (MODELS_DIR / "ner-labels.txt").write_text("\n".join(labels), encoding="utf-8")
+
+    shutil.rmtree(temp_dir)
+
+    print()
+    print("NER model assets saved:")
+    print(f"  {target_onnx}")
+    print(f"  {MODELS_DIR / 'vocab.txt'}")
+    print(f"  {MODELS_DIR / 'ner-labels.txt'}")
+    print()
+    print("Run from repository root:")
+    print("  dotnet run --project src/PiiRedaction.ConsoleApp -- --name CustomerNameOnly")
+
+
+def main() -> int:
+    ensure_dependencies()
+    export_model()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())