| 123456789101112 |
- import os
- import tiktoken
- from core.model_runtime.model_providers.__base.tokenizers.gpt2_tokenzier import GPT2Tokenizer
- def test_tiktoken():
- os.environ["TIKTOKEN_CACHE_DIR"] = "/tmp/.tiktoken_cache"
- GPT2Tokenizer.get_num_tokens("Hello, world!")
- assert tiktoken.registry.ENCODING_CONSTRUCTORS is not None
|
