dfc81dea28
Implements detect-redact-sanitize pipeline with regex, domain rules, and ONNX NER before the LLM boundary, plus NUnit tests and Xenovex push documentation. Co-authored-by: Cursor <cursoragent@cursor.com>
84 lines
2.5 KiB
Python
84 lines
2.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Download and export dslim/bert-base-NER to ONNX for the PII Redaction POC."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
MODELS_DIR = REPO_ROOT / "models"
|
|
MODEL_ID = "dslim/bert-base-NER"
|
|
REQUIRED_PACKAGES = ("transformers", "optimum[onnxruntime]", "onnx", "torch")
|
|
|
|
|
|
def ensure_dependencies() -> None:
|
|
try:
|
|
import optimum.onnxruntime # noqa: F401
|
|
import transformers # noqa: F401
|
|
except ImportError:
|
|
print("Installing Python dependencies (this may take a few minutes)...")
|
|
subprocess.check_call(
|
|
[sys.executable, "-m", "pip", "install", *REQUIRED_PACKAGES],
|
|
stdout=sys.stdout,
|
|
stderr=sys.stderr,
|
|
)
|
|
|
|
|
|
def export_model() -> None:
|
|
from optimum.onnxruntime import ORTModelForTokenClassification
|
|
from transformers import AutoTokenizer
|
|
|
|
MODELS_DIR.mkdir(parents=True, exist_ok=True)
|
|
temp_dir = MODELS_DIR / "_export_temp"
|
|
if temp_dir.exists():
|
|
shutil.rmtree(temp_dir)
|
|
temp_dir.mkdir()
|
|
|
|
print(f"Exporting {MODEL_ID} to ONNX...")
|
|
model = ORTModelForTokenClassification.from_pretrained(MODEL_ID, export=True)
|
|
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
|
|
|
|
model.save_pretrained(temp_dir)
|
|
tokenizer.save_pretrained(temp_dir)
|
|
|
|
onnx_files = sorted(temp_dir.glob("*.onnx"))
|
|
if not onnx_files:
|
|
raise FileNotFoundError("Export completed but no .onnx file was produced.")
|
|
|
|
target_onnx = MODELS_DIR / "ner-model.onnx"
|
|
shutil.copy(onnx_files[0], target_onnx)
|
|
shutil.copy(temp_dir / "vocab.txt", MODELS_DIR / "vocab.txt")
|
|
|
|
config_path = temp_dir / "config.json"
|
|
with config_path.open(encoding="utf-8") as config_file:
|
|
config = json.load(config_file)
|
|
|
|
id2label = config.get("id2label", {})
|
|
labels = [id2label[str(index)] for index in range(len(id2label))]
|
|
(MODELS_DIR / "ner-labels.txt").write_text("\n".join(labels), encoding="utf-8")
|
|
|
|
shutil.rmtree(temp_dir)
|
|
|
|
print()
|
|
print("NER model assets saved:")
|
|
print(f" {target_onnx}")
|
|
print(f" {MODELS_DIR / 'vocab.txt'}")
|
|
print(f" {MODELS_DIR / 'ner-labels.txt'}")
|
|
print()
|
|
print("Run from repository root:")
|
|
print(" dotnet run --project src/PiiRedaction.ConsoleApp -- --name CustomerNameOnly")
|
|
|
|
|
|
def main() -> int:
|
|
ensure_dependencies()
|
|
export_model()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|