ソースを参照

enhance: preload tiktoken

Yeuoly 4 ヶ月 前
コミット
6584080106
共有1 個のファイルを変更した6 個の追加7 個の削除を含む
  1. 6 7
      docker/local.dockerfile

+ 6 - 7
docker/local.dockerfile

@@ -38,14 +38,13 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl pyt
     && rm -rf /var/lib/apt/lists/* \
     && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1;
 
-# Install uv
-RUN mv /usr/lib/python3.12/EXTERNALLY-MANAGED /usr/lib/python3.12/EXTERNALLY-MANAGED.bk && python3 -m pip install uv
+# preload tiktoken
+ENV TIKTOKEN_CACHE_DIR=/app/.tiktoken
 
-# Install dify_plugin to speedup the environment setup
-RUN uv pip install --system dify_plugin
-
-# Test uv
-RUN python3 -c "from uv._find_uv import find_uv_bin;print(find_uv_bin())"
+# Install dify_plugin to speedup the environment setup and test uv
+RUN mv /usr/lib/python3.12/EXTERNALLY-MANAGED /usr/lib/python3.12/EXTERNALLY-MANAGED.bk && python3 -m pip install uv && \
+    python3 -c "from uv._find_uv import find_uv_bin;print(find_uv_bin());" \
+    && python3 -c "import tiktoken; tiktoken.get_encoding('gpt2').special_tokens_set; tiktoken.get_encoding('cl100k_base').special_tokens_set"
 
 ENV PLATFORM=$PLATFORM
 ENV GIN_MODE=release