Merge pull request #127 from langgenius/enhance/tiktoken

enhance: preload tiktoken

docker/local.dockerfile

@@ -38,14 +38,15 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl pyt
     && rm -rf /var/lib/apt/lists/* \
     && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1;
 
-# Install uv
-RUN mv /usr/lib/python3.12/EXTERNALLY-MANAGED /usr/lib/python3.12/EXTERNALLY-MANAGED.bk && python3 -m pip install uv
-
-# Install dify_plugin to speedup the environment setup
-RUN uv pip install --system dify_plugin
-
-# Test uv
-RUN python3 -c "from uv._find_uv import find_uv_bin;print(find_uv_bin())"
+# preload tiktoken
+ENV TIKTOKEN_CACHE_DIR=/app/.tiktoken
+
+# Install dify_plugin to speedup the environment setup, test uv and preload tiktoken
+RUN mv /usr/lib/python3.12/EXTERNALLY-MANAGED /usr/lib/python3.12/EXTERNALLY-MANAGED.bk \
+    && python3 -m pip install uv \
+    && uv pip install --system dify_plugin \
+    && python3 -c "from uv._find_uv import find_uv_bin;print(find_uv_bin());" \
+    && python3 -c "import tiktoken; tiktoken.get_encoding('gpt2').special_tokens_set; tiktoken.get_encoding('cl100k_base').special_tokens_set"
 
 ENV PLATFORM=$PLATFORM
 ENV GIN_MODE=release