llm-pii-poc/tests/PiiRedaction.Infrastructure.Tests/Onnx/TokenClassifierEncoderFactoryTests.cs

using FluentAssertions;
using Microsoft.Extensions.Logging.Abstractions;
using PiiRedaction.Infrastructure.Onnx;

namespace PiiRedaction.Infrastructure.Tests.Onnx;

[TestFixture]
public sealed class TokenClassifierEncoderFactoryTests
{
    [Test]
    public void Create_PrefersWordPieceWhenVocabExists()
    {
        var modelDirectory = ResolveTamilModelDirectory();
        if (!File.Exists(Path.Combine(modelDirectory, "vocab.txt")))
        {
            Assert.Ignore("Tamil vocab.txt not found.");
        }

        var encoder = TokenClassifierEncoderFactory.Create(
            modelDirectory,
            NullLogger.Instance);

        encoder.Should().BeOfType<BertWordPieceEncoder>();
        encoder.IsAvailable.Should().BeTrue();
    }

    private static string ResolveTamilModelDirectory()
    {
        var directory = new DirectoryInfo(AppContext.BaseDirectory);
        while (directory is not null)
        {
            var candidate = Path.Combine(directory.FullName, "models", "ta");
            if (Directory.Exists(candidate))
            {
                return candidate;
            }

            directory = directory.Parent;
        }

        return Path.Combine(Environment.CurrentDirectory, "models", "ta");
    }
}