[ci] introduce uv to accelerate pip install

.dev_scripts/ci_container_test.sh

@@ -1,6 +1,6 @@
 if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then
-    # pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
-    pip install -r requirements/tests.txt -i https://mirrors.aliyun.com/pypi/simple/
+    pip install uv
+    uv pip install -r requirements/tests.txt
     git config --global --add safe.directory /ms-swift
     git config --global user.email tmp
     git config --global user.name tmp.com
@@ -20,17 +20,20 @@ if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then
         fi
     fi
 
-    pip install -r requirements/framework.txt -U -i https://mirrors.aliyun.com/pypi/simple/
-    pip install decord einops -U -i https://mirrors.aliyun.com/pypi/simple/
+    uv pip install -r requirements/framework.txt -U
+    uv pip install decord einops -U
     pip uninstall autoawq -y
-    pip install optimum
-    pip install diffusers
-    pip install "transformers<5.0" "peft<0.19"
+    uv pip install optimum
+    uv pip install diffusers
+    uv pip install "transformers<5.0" "peft<0.19"
     # pip install autoawq -U --no-deps
 
     # test with install
-    pip install .
-    pip install auto_gptq bitsandbytes deepspeed -U -i https://mirrors.aliyun.com/pypi/simple/
+    uv pip install .
+    uv pip install auto_gptq bitsandbytes deepspeed -U
+    if [ -f requirements/npu.txt ]; then
+        uv pip install -r requirements/npu.txt
+    fi
 else
     echo "Running case in release image, run case directly!"
 fi

.dev_scripts/dockerci.sh

@@ -48,31 +48,41 @@ do
               -e TEST_UPLOAD_MS_TOKEN=$TEST_UPLOAD_MS_TOKEN \
               -e MODEL_TAG_URL=$MODEL_TAG_URL \
               -e MODELSCOPE_API_TOKEN=$MODELSCOPE_API_TOKEN \
-	            -e PR_CHANGED_FILES=$PR_CHANGED_FILES \
+              -e PR_CHANGED_FILES=$PR_CHANGED_FILES \
+              -e UV_INDEX_URL=${UV_INDEX_URL:-https://mirrors.aliyun.com/pypi/simple/} \
+              -e UV_EXTRA_INDEX_URL=$UV_EXTRA_INDEX_URL \
+              -e UV_INDEX_STRATEGY=$UV_INDEX_STRATEGY \
+              -e UV_NO_CACHE=$UV_NO_CACHE \
+              -e UV_SYSTEM_PYTHON=${UV_SYSTEM_PYTHON:-1} \
               --workdir=$CODE_DIR_IN_CONTAINER \
               ${IMAGE_NAME}:${IMAGE_VERSION} \
               $CI_COMMAND
-  else
-    docker run --rm --name $CONTAINER_NAME --shm-size=16gb \
-              --cpuset-cpus=${cpu_sets_arr[$idx]} \
-              --gpus='"'"device=$gpu"'"' \
-              -v $CODE_DIR:$CODE_DIR_IN_CONTAINER \
-              -v $MODELSCOPE_CACHE:$MODELSCOPE_CACHE_DIR_IN_CONTAINER \
-              -v $MODELSCOPE_HOME_CACHE/$idx:/root \
-              -v /home/admin/pre-commit:/home/admin/pre-commit \
-              -e CI_TEST=True \
-              -e TEST_LEVEL=$TEST_LEVEL \
-              -e MODELSCOPE_CACHE=$MODELSCOPE_CACHE_DIR_IN_CONTAINER \
-              -e MODELSCOPE_DOMAIN=$MODELSCOPE_DOMAIN \
-              -e HUB_DATASET_ENDPOINT=$HUB_DATASET_ENDPOINT \
+   else
+     docker run --rm --name $CONTAINER_NAME --shm-size=16gb \
+               --cpuset-cpus=${cpu_sets_arr[$idx]} \
+               --gpus='"'"device=$gpu"'"' \
+               -v $CODE_DIR:$CODE_DIR_IN_CONTAINER \
+               -v $MODELSCOPE_CACHE:$MODELSCOPE_CACHE_DIR_IN_CONTAINER \
+               -v $MODELSCOPE_HOME_CACHE/$idx:/root \
+               -v /home/admin/pre-commit:/home/admin/pre-commit \
+               -e CI_TEST=True \
+               -e TEST_LEVEL=$TEST_LEVEL \
+               -e MODELSCOPE_CACHE=$MODELSCOPE_CACHE_DIR_IN_CONTAINER \
+               -e MODELSCOPE_DOMAIN=$MODELSCOPE_DOMAIN \
+               -e HUB_DATASET_ENDPOINT=$HUB_DATASET_ENDPOINT \
               -e TEST_ACCESS_TOKEN_CITEST=$TEST_ACCESS_TOKEN_CITEST \
               -e TEST_ACCESS_TOKEN_SDKDEV=$TEST_ACCESS_TOKEN_SDKDEV \
               -e TEST_LEVEL=$TEST_LEVEL \
               -e MODELSCOPE_ENVIRONMENT='ci' \
               -e TEST_UPLOAD_MS_TOKEN=$TEST_UPLOAD_MS_TOKEN \
               -e MODEL_TAG_URL=$MODEL_TAG_URL \
               -e MODELSCOPE_API_TOKEN=$MODELSCOPE_API_TOKEN \
-	            -e PR_CHANGED_FILES=$PR_CHANGED_FILES \
+              -e PR_CHANGED_FILES=$PR_CHANGED_FILES \
+              -e UV_INDEX_URL=${UV_INDEX_URL:-https://mirrors.aliyun.com/pypi/simple/} \
+              -e UV_EXTRA_INDEX_URL=$UV_EXTRA_INDEX_URL \
+              -e UV_INDEX_STRATEGY=$UV_INDEX_STRATEGY \
+              -e UV_NO_CACHE=$UV_NO_CACHE \
+              -e UV_SYSTEM_PYTHON=${UV_SYSTEM_PYTHON:-1} \
               --workdir=$CODE_DIR_IN_CONTAINER \
               ${IMAGE_NAME}:${IMAGE_VERSION} \
               $CI_COMMAND

.github/workflows/citest.yaml

@@ -71,6 +71,11 @@ jobs:
         run: git lfs checkout
       - name: Run unittest
         shell: bash
+        env:
+          UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          UV_INDEX_STRATEGY: "unsafe-best-match"
+          UV_NO_CACHE: 1
+          UV_SYSTEM_PYTHON: 1
         run: |
           set -e
           source /mnt/modelscope/ci_env.sh

.github/workflows/citest_npu.yaml

@@ -43,15 +43,22 @@ jobs:
     timeout-minutes: 240
     container:
       image: 'ascendai/cann:8.3.rc2-910b-ubuntu22.04-py3.11'
+    env:
+      UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+      UV_EXTRA_INDEX_URL: "https://mirrors.huaweicloud.com/ascend/repos/pypi"
+      UV_INDEX_STRATEGY: "unsafe-best-match"
+      UV_NO_CACHE: 1
+      UV_SYSTEM_PYTHON: 1
     steps:
       - name: Config mirrors
         run: |
           sed -Ei 's@(ports|archive).ubuntu.com@cache-service.nginx-pypi-cache.svc.cluster.local:8081@g' /etc/apt/sources.list
           pip config set global.index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple
           pip config set global.trusted-host cache-service.nginx-pypi-cache.svc.cluster.local
+          pip install uv
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
             fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }}
       - name: Get changed files

.github/workflows/lint.yaml

@@ -10,13 +10,21 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python 3.10
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
+          cache: pip
+      - name: Cache pre-commit environments
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pre-commit
+          key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }}
+          restore-keys: pre-commit-${{ runner.os }}-
       - name: Install pre-commit hook
         run: |
-          pip install pre-commit
+          pip install uv
+          uv pip install --system pre-commit
       - name: Linting
         run: pre-commit run --all-files

.github/workflows/publish.yaml

@@ -20,10 +20,13 @@ jobs:
         with:
           python-version: '3.10'
       - name: Install wheel
-        run: pip install wheel packaging setuptools==69.5.1
+        run: |
+          pip install uv
+          uv pip install --system wheel packaging setuptools==69.5.1
       - name: Build ModelScope Swift
         run: python setup.py sdist bdist_wheel
       - name: Publish package to PyPI
         run: |
-          pip install twine
+          pip install uv
+          uv pip install --system twine
           twine upload dist/* --skip-existing -u __token__ -p ${{ secrets.PYPI_API_TOKEN }}

requirements/npu.txt

@@ -0,0 +1,5 @@
+torch==2.7.1
+torch-npu==2.7.1.post4
+torchvision==0.22.1
+torchaudio==2.7.1
+decorator