Add Tamil NER routing and WPF test harness for POC validation.
Introduce dual-script ONNX NER routing (English/Tamil/mixed), Tamil console samples and integration tests, model download scripts, and a resizable WPF MVVM harness with click-to-load prompts, batch validation, and runtime-adjustable detection panels.
This commit is contained in:
27
scripts/tamil-ner-diagnostic/Program.cs
Normal file
27
scripts/tamil-ner-diagnostic/Program.cs
Normal file
@@ -0,0 +1,27 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Microsoft.ML.Tokenizers;
|
||||
using PiiRedaction.Core.Configuration;
|
||||
using PiiRedaction.Infrastructure.Onnx;
|
||||
|
||||
var modelDir = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "models", "ta"));
|
||||
if (!Directory.Exists(modelDir))
|
||||
{
|
||||
modelDir = Path.GetFullPath("models/ta");
|
||||
}
|
||||
|
||||
var vocabPath = Path.Combine(modelDir, "vocab.txt");
|
||||
var bertOptions = new BertOptions { LowerCaseBeforeTokenization = false, ApplyBasicTokenization = false };
|
||||
var tokenizer = BertTokenizer.Create(vocabPath, bertOptions);
|
||||
var text = "வாடிக்கையாளர் ராஜேஷ் குமார் அழைத்தார்.";
|
||||
var tokens = tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true);
|
||||
Console.WriteLine($"count={tokens.Count}");
|
||||
foreach (var t in tokens) Console.WriteLine($"{t.Id}\t{t.Value}");
|
||||
|
||||
var runnerOptions = Options.Create(new PiiRedactionOptions { TamilOnnxModelPath = Path.Combine(modelDir, "model.onnx") });
|
||||
using var runner = new TamilOnnxNerRunner(runnerOptions, NullLogger<TamilOnnxNerRunner>.Instance);
|
||||
Console.WriteLine($"Available: {runner.IsModelAvailable}");
|
||||
foreach (var e in runner.PredictEntities(text))
|
||||
{
|
||||
Console.WriteLine($"Entity: '{e.Value}' [{e.StartIndex},{e.Length}]");
|
||||
}
|
||||
Reference in New Issue
Block a user