llm-pii-poc/scripts/tamil-ner-diagnostic/Program.cs

using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Microsoft.ML.Tokenizers;
using PiiRedaction.Core.Configuration;
using PiiRedaction.Infrastructure.Onnx;

var modelDir = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "models", "ta"));
if (!Directory.Exists(modelDir))
{
    modelDir = Path.GetFullPath("models/ta");
}

var vocabPath = Path.Combine(modelDir, "vocab.txt");
var bertOptions = new BertOptions { LowerCaseBeforeTokenization = false, ApplyBasicTokenization = false };
var tokenizer = BertTokenizer.Create(vocabPath, bertOptions);
var text = "வாடிக்கையாளர் ராஜேஷ் குமார் அழைத்தார்.";
var tokens = tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true);
Console.WriteLine($"count={tokens.Count}");
foreach (var t in tokens) Console.WriteLine($"{t.Id}\t{t.Value}");

var runnerOptions = Options.Create(new PiiRedactionOptions { TamilOnnxModelPath = Path.Combine(modelDir, "model.onnx") });
using var runner = new TamilOnnxNerRunner(runnerOptions, NullLogger<TamilOnnxNerRunner>.Instance);
Console.WriteLine($"Available: {runner.IsModelAvailable}");
foreach (var e in runner.PredictEntities(text).Entities)
{
    Console.WriteLine($"Entity: '{e.Value}' [{e.StartIndex},{e.Length}]");
}