Add Tamil NER routing and WPF test harness for POC validation.
Introduce dual-script ONNX NER routing (English/Tamil/mixed), Tamil console samples and integration tests, model download scripts, and a resizable WPF MVVM harness with click-to-load prompts, batch validation, and runtime-adjustable detection panels.
This commit is contained in:
Bilal Nazer Ali
229
scripts/download-tamil-ner-model.ps1
Normal file
229
scripts/download-tamil-ner-model.ps1
Normal file
@@ -0,0 +1,229 @@
|
||||
# Downloads prachuryyaIITG/SampurNER_Tamil_IndicBERTv2 ONNX assets to models/ta/ for the PII Redaction POC.
|
||||
param(
|
||||
[string]$Python = "python"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$repoRoot = Split-Path -Parent $PSScriptRoot
|
||||
$modelsDir = Join-Path (Join-Path $repoRoot "models") "ta"
|
||||
$scriptPath = Join-Path $PSScriptRoot "download-tamil-ner-model.py"
|
||||
$modelId = "prachuryyaIITG/SampurNER_Tamil_IndicBERTv2"
|
||||
$baseUrl = "https://huggingface.co/$modelId/resolve/main"
|
||||
|
||||
function Ensure-ModelsDirectory {
|
||||
New-Item -ItemType Directory -Force -Path $modelsDir | Out-Null
|
||||
}
|
||||
|
||||
function Download-HuggingFaceAsset {
|
||||
param(
|
||||
[string]$RelativePath,
|
||||
[string]$Destination
|
||||
)
|
||||
|
||||
$url = "$baseUrl/$RelativePath"
|
||||
Write-Host "Downloading $url"
|
||||
Invoke-WebRequest -Uri $url -OutFile $Destination -UseBasicParsing
|
||||
}
|
||||
|
||||
function Export-LabelsFromConfig {
|
||||
param([string]$ConfigPath, [string]$LabelsPath)
|
||||
|
||||
$config = Get-Content $ConfigPath -Raw | ConvertFrom-Json
|
||||
$labelMap = @{}
|
||||
foreach ($property in $config.id2label.PSObject.Properties) {
|
||||
$labelMap[[int]$property.Name] = [string]$property.Value
|
||||
}
|
||||
|
||||
$labels = for ($index = 0; $index -lt $labelMap.Count; $index++) {
|
||||
$labelMap[$index]
|
||||
}
|
||||
|
||||
$labels | Set-Content -Path $LabelsPath -Encoding utf8
|
||||
}
|
||||
|
||||
function Resolve-PythonExecutable {
|
||||
param([string]$Preferred = "python")
|
||||
|
||||
if ($Preferred -ne "python") {
|
||||
if ((Get-Command $Preferred -ErrorAction SilentlyContinue) -and
|
||||
-not ((Get-Command $Preferred).Source -like "*WindowsApps*")) {
|
||||
return $Preferred
|
||||
}
|
||||
}
|
||||
|
||||
$candidates = @(
|
||||
(Join-Path $env:LOCALAPPDATA "Programs\Python\Python313\python.exe"),
|
||||
(Join-Path $env:LOCALAPPDATA "Programs\Python\Python312\python.exe"),
|
||||
(Join-Path $env:LOCALAPPDATA "Programs\Python\Python311\python.exe"),
|
||||
"C:\Program Files\Python312\python.exe",
|
||||
"C:\Program Files\Python313\python.exe"
|
||||
)
|
||||
|
||||
foreach ($candidate in $candidates) {
|
||||
if (Test-Path $candidate) {
|
||||
return $candidate
|
||||
}
|
||||
}
|
||||
|
||||
$pythonCommand = Get-Command python -ErrorAction SilentlyContinue
|
||||
if ($pythonCommand -and $pythonCommand.Source -notlike "*WindowsApps*") {
|
||||
return $pythonCommand.Source
|
||||
}
|
||||
|
||||
return $null
|
||||
}
|
||||
|
||||
function Export-VocabFromTokenizerJson {
|
||||
param([string]$TokenizerJsonPath, [string]$VocabPath)
|
||||
|
||||
$tokenizer = Get-Content $TokenizerJsonPath -Raw | ConvertFrom-Json
|
||||
$vocab = $tokenizer.model.vocab
|
||||
if (-not $vocab) {
|
||||
return $false
|
||||
}
|
||||
|
||||
$orderedTokens = $vocab.PSObject.Properties |
|
||||
Sort-Object { [int]$_.Value } |
|
||||
ForEach-Object { $_.Name }
|
||||
|
||||
$orderedTokens | Set-Content -Path $VocabPath -Encoding utf8
|
||||
return $true
|
||||
}
|
||||
|
||||
function Copy-TokenizerAssets {
|
||||
param([string]$SourceDir)
|
||||
|
||||
foreach ($name in @("sentencepiece.bpe.model", "spiece.model", "tokenizer.model")) {
|
||||
$source = Join-Path $SourceDir $name
|
||||
if (Test-Path $source) {
|
||||
$destination = Join-Path $modelsDir $name
|
||||
Copy-Item $source $destination -Force
|
||||
return @($destination)
|
||||
}
|
||||
}
|
||||
|
||||
$tokenizerJsonSource = Join-Path $SourceDir "tokenizer.json"
|
||||
if (Test-Path $tokenizerJsonSource) {
|
||||
$tokenizerJsonDestination = Join-Path $modelsDir "tokenizer.json"
|
||||
Copy-Item $tokenizerJsonSource $tokenizerJsonDestination -Force
|
||||
$saved = @($tokenizerJsonDestination)
|
||||
|
||||
$vocabPath = Join-Path $modelsDir "vocab.txt"
|
||||
if (Export-VocabFromTokenizerJson -TokenizerJsonPath $tokenizerJsonDestination -VocabPath $vocabPath) {
|
||||
$saved += $vocabPath
|
||||
}
|
||||
|
||||
Write-Warning (
|
||||
"No SentencePiece model on Hugging Face; saved WordPiece assets ($(
|
||||
($saved | ForEach-Object { Split-Path $_ -Leaf }) -join ', '
|
||||
)). TamilOnnxNerRunner uses vocab.txt (WordPiece) when present, otherwise SentencePiece model files."
|
||||
)
|
||||
return $saved
|
||||
}
|
||||
|
||||
return @()
|
||||
}
|
||||
|
||||
function Download-WithPowerShell {
|
||||
Ensure-ModelsDirectory
|
||||
|
||||
$modelPath = Join-Path $modelsDir "model.onnx"
|
||||
$configPath = Join-Path $modelsDir "config.json"
|
||||
$labelsPath = Join-Path $modelsDir "ner-labels.txt"
|
||||
$tempDir = Join-Path $modelsDir "_download_temp"
|
||||
New-Item -ItemType Directory -Force -Path $tempDir | Out-Null
|
||||
|
||||
try {
|
||||
Download-HuggingFaceAsset -RelativePath "onnx/model.onnx" -Destination $modelPath
|
||||
}
|
||||
catch {
|
||||
Write-Host "Pre-exported ONNX not found; downloading config and tokenizer for manual export..."
|
||||
Download-HuggingFaceAsset -RelativePath "config.json" -Destination $configPath
|
||||
Export-LabelsFromConfig -ConfigPath $configPath -LabelsPath $labelsPath
|
||||
|
||||
$tokenizerJsonPath = Join-Path $modelsDir "tokenizer.json"
|
||||
try {
|
||||
Download-HuggingFaceAsset -RelativePath "tokenizer.json" -Destination $tokenizerJsonPath
|
||||
$vocabPath = Join-Path $modelsDir "vocab.txt"
|
||||
Export-VocabFromTokenizerJson -TokenizerJsonPath $tokenizerJsonPath -VocabPath $vocabPath | Out-Null
|
||||
}
|
||||
catch {
|
||||
Write-Host "tokenizer.json not available from Hugging Face."
|
||||
}
|
||||
|
||||
foreach ($name in @("sentencepiece.bpe.model", "spiece.model", "tokenizer.model")) {
|
||||
try {
|
||||
Download-HuggingFaceAsset -RelativePath $name -Destination (Join-Path $modelsDir $name)
|
||||
break
|
||||
}
|
||||
catch {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
$pythonExe = Resolve-PythonExecutable -Preferred $Python
|
||||
if ($pythonExe) {
|
||||
throw (
|
||||
"Tamil ONNX model is not published on Hugging Face. Re-run with Python export:`n" +
|
||||
" .\scripts\download-tamil-ner-model.ps1 -Python `"$pythonExe`""
|
||||
)
|
||||
}
|
||||
|
||||
throw @"
|
||||
Tamil ONNX model is not published on Hugging Face (onnx/model.onnx returns 404).
|
||||
Install Python 3.12+ and re-run this script:
|
||||
winget install Python.Python.3.12 --accept-package-agreements --accept-source-agreements
|
||||
.\scripts\download-tamil-ner-model.ps1 -Python `"`$env:LOCALAPPDATA\Programs\Python\Python312\python.exe`"
|
||||
Only config.json, ner-labels.txt, and tokenizer.json were saved under models/ta/.
|
||||
"@
|
||||
}
|
||||
|
||||
Download-HuggingFaceAsset -RelativePath "config.json" -Destination $configPath
|
||||
Export-LabelsFromConfig -ConfigPath $configPath -LabelsPath $labelsPath
|
||||
Remove-Item $configPath -Force
|
||||
|
||||
foreach ($name in @("sentencepiece.bpe.model", "spiece.model")) {
|
||||
try {
|
||||
Download-HuggingFaceAsset -RelativePath $name -Destination (Join-Path $modelsDir $name)
|
||||
break
|
||||
}
|
||||
catch {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Tamil NER model assets saved to $modelsDir"
|
||||
}
|
||||
|
||||
function Download-WithPython {
|
||||
$pythonExe = Resolve-PythonExecutable -Preferred $Python
|
||||
if (-not $pythonExe) {
|
||||
return $false
|
||||
}
|
||||
|
||||
Write-Host "Using Python: $pythonExe"
|
||||
Write-Host "Repository root: $repoRoot"
|
||||
Write-Host ""
|
||||
|
||||
Push-Location $repoRoot
|
||||
try {
|
||||
& $pythonExe $scriptPath
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Tamil model download script failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
}
|
||||
finally {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
return $true
|
||||
}
|
||||
|
||||
Write-Host "Repository root: $repoRoot"
|
||||
|
||||
if (-not (Download-WithPython)) {
|
||||
Write-Host "Python export unavailable; downloading pre-exported ONNX assets from Hugging Face..."
|
||||
Write-Host ""
|
||||
Download-WithPowerShell
|
||||
}
|
||||
Reference in New Issue
Block a user