Merge pull request #175 from langgenius/fix/tiktoken

refactor: update tiktoken encoding retrieval to handle full encodings

docker/local.dockerfile

@@ -46,7 +46,7 @@ RUN mv /usr/lib/python3.12/EXTERNALLY-MANAGED /usr/lib/python3.12/EXTERNALLY-MAN
     && python3 -m pip install uv \
     && uv pip install --system dify_plugin \
     && python3 -c "from uv._find_uv import find_uv_bin;print(find_uv_bin());" \
-    && python3 -c "import tiktoken; tiktoken.get_encoding('gpt2').special_tokens_set; tiktoken.get_encoding('cl100k_base').special_tokens_set"
+    && python3 -c "import tiktoken; encodings = ['o200k_base', 'cl100k_base', 'p50k_base', 'r50k_base', 'p50k_edit', 'gpt2']; [tiktoken.get_encoding(encoding).special_tokens_set for encoding in encodings]"
 
 ENV PLATFORM=$PLATFORM
 ENV GIN_MODE=release