28 lines
1.4 KiB
C#
28 lines
1.4 KiB
C#
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
using Microsoft.ML.Tokenizers;
|
|
using PiiRedaction.Core.Configuration;
|
|
using PiiRedaction.Infrastructure.Onnx;
|
|
|
|
var modelDir = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "models", "ta"));
|
|
if (!Directory.Exists(modelDir))
|
|
{
|
|
modelDir = Path.GetFullPath("models/ta");
|
|
}
|
|
|
|
var vocabPath = Path.Combine(modelDir, "vocab.txt");
|
|
var bertOptions = new BertOptions { LowerCaseBeforeTokenization = false, ApplyBasicTokenization = false };
|
|
var tokenizer = BertTokenizer.Create(vocabPath, bertOptions);
|
|
var text = "வாடிக்கையாளர் ராஜேஷ் குமார் அழைத்தார்.";
|
|
var tokens = tokenizer.EncodeToTokens(text, out _, considerPreTokenization: true, considerNormalization: true);
|
|
Console.WriteLine($"count={tokens.Count}");
|
|
foreach (var t in tokens) Console.WriteLine($"{t.Id}\t{t.Value}");
|
|
|
|
var runnerOptions = Options.Create(new PiiRedactionOptions { TamilOnnxModelPath = Path.Combine(modelDir, "model.onnx") });
|
|
using var runner = new TamilOnnxNerRunner(runnerOptions, NullLogger<TamilOnnxNerRunner>.Instance);
|
|
Console.WriteLine($"Available: {runner.IsModelAvailable}");
|
|
foreach (var e in runner.PredictEntities(text).Entities)
|
|
{
|
|
Console.WriteLine($"Entity: '{e.Value}' [{e.StartIndex},{e.Length}]");
|
|
}
|