Add Tamil NER routing and WPF test harness for POC validation.

Introduce dual-script ONNX NER routing (English/Tamil/mixed), Tamil console samples and integration tests, model download scripts, and a resizable WPF MVVM harness with click-to-load prompts, batch validation, and runtime-adjustable detection panels.
This commit is contained in:
Bilal Nazer Ali
2026-07-07 17:12:38 +05:30
parent a707c6c9cf
commit cf8f5a7232
71 changed files with 4494 additions and 356 deletions

View File

@@ -0,0 +1,27 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Microsoft.ML.Tokenizers;
using PiiRedaction.Core.Configuration;
using PiiRedaction.Infrastructure.Onnx;
var modelDir = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "models", "ta"));
if (!Directory.Exists(modelDir))
{
modelDir = Path.GetFullPath("models/ta");
}
var vocabPath = Path.Combine(modelDir, "vocab.txt");
var bertOptions = new BertOptions { LowerCaseBeforeTokenization = false, ApplyBasicTokenization = false };
var tokenizer = BertTokenizer.Create(vocabPath, bertOptions);
var text = "வாடிக்கையாளர் ராஜேஷ் குமார் அழைத்தார்.";
var tokens = tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true);
Console.WriteLine($"count={tokens.Count}");
foreach (var t in tokens) Console.WriteLine($"{t.Id}\t{t.Value}");
var runnerOptions = Options.Create(new PiiRedactionOptions { TamilOnnxModelPath = Path.Combine(modelDir, "model.onnx") });
using var runner = new TamilOnnxNerRunner(runnerOptions, NullLogger<TamilOnnxNerRunner>.Instance);
Console.WriteLine($"Available: {runner.IsModelAvailable}");
foreach (var e in runner.PredictEntities(text))
{
Console.WriteLine($"Entity: '{e.Value}' [{e.StartIndex},{e.Length}]");
}