import pytest
from src.core import SimpleTokenizer, JiebaTokenizer

@pytest.fixture
def stop_words():
    return {"the", "and", "a"}

def test_simple_tokenizer(stop_words):
    tokenizer = SimpleTokenizer()
    text = "the quick brown fox and a dog"
    assert tokenizer.tokenize(text, stop_words) == ["quick", "brown", "fox", "dog"]

def test_jieba_tokenizer(stop_words):
    tokenizer = JiebaTokenizer()
    text = "我爱北京天安门"
    assert tokenizer.tokenize(text, set()) == ["我", "爱", "北京", "天安门"]