diff --git a/.dev_scripts/ci_container_test.sh b/.dev_scripts/ci_container_test.sh index 4323be1899..d2e270a498 100644 --- a/.dev_scripts/ci_container_test.sh +++ b/.dev_scripts/ci_container_test.sh @@ -1,6 +1,6 @@ if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then - # pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple - pip install -r requirements/tests.txt -i https://mirrors.aliyun.com/pypi/simple/ + pip install uv + uv pip install -r requirements/tests.txt git config --global --add safe.directory /ms-swift git config --global user.email tmp git config --global user.name tmp.com @@ -20,17 +20,20 @@ if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then fi fi - pip install -r requirements/framework.txt -U -i https://mirrors.aliyun.com/pypi/simple/ - pip install decord einops -U -i https://mirrors.aliyun.com/pypi/simple/ + uv pip install -r requirements/framework.txt -U + uv pip install decord einops -U pip uninstall autoawq -y - pip install optimum - pip install diffusers - pip install "transformers<5.0" "peft<0.19" + uv pip install optimum + uv pip install diffusers + uv pip install "transformers<5.0" "peft<0.19" # pip install autoawq -U --no-deps # test with install - pip install . - pip install auto_gptq bitsandbytes deepspeed -U -i https://mirrors.aliyun.com/pypi/simple/ + uv pip install . + uv pip install auto_gptq bitsandbytes deepspeed -U + if [ -f requirements/npu.txt ]; then + uv pip install -r requirements/npu.txt + fi else echo "Running case in release image, run case directly!" fi diff --git a/.dev_scripts/dockerci.sh b/.dev_scripts/dockerci.sh index f42e819fb4..89a185fd15 100644 --- a/.dev_scripts/dockerci.sh +++ b/.dev_scripts/dockerci.sh @@ -48,23 +48,28 @@ do -e TEST_UPLOAD_MS_TOKEN=$TEST_UPLOAD_MS_TOKEN \ -e MODEL_TAG_URL=$MODEL_TAG_URL \ -e MODELSCOPE_API_TOKEN=$MODELSCOPE_API_TOKEN \ - -e PR_CHANGED_FILES=$PR_CHANGED_FILES \ + -e PR_CHANGED_FILES=$PR_CHANGED_FILES \ + -e UV_INDEX_URL=${UV_INDEX_URL:-https://mirrors.aliyun.com/pypi/simple/} \ + -e UV_EXTRA_INDEX_URL=$UV_EXTRA_INDEX_URL \ + -e UV_INDEX_STRATEGY=$UV_INDEX_STRATEGY \ + -e UV_NO_CACHE=$UV_NO_CACHE \ + -e UV_SYSTEM_PYTHON=${UV_SYSTEM_PYTHON:-1} \ --workdir=$CODE_DIR_IN_CONTAINER \ ${IMAGE_NAME}:${IMAGE_VERSION} \ $CI_COMMAND - else - docker run --rm --name $CONTAINER_NAME --shm-size=16gb \ - --cpuset-cpus=${cpu_sets_arr[$idx]} \ - --gpus='"'"device=$gpu"'"' \ - -v $CODE_DIR:$CODE_DIR_IN_CONTAINER \ - -v $MODELSCOPE_CACHE:$MODELSCOPE_CACHE_DIR_IN_CONTAINER \ - -v $MODELSCOPE_HOME_CACHE/$idx:/root \ - -v /home/admin/pre-commit:/home/admin/pre-commit \ - -e CI_TEST=True \ - -e TEST_LEVEL=$TEST_LEVEL \ - -e MODELSCOPE_CACHE=$MODELSCOPE_CACHE_DIR_IN_CONTAINER \ - -e MODELSCOPE_DOMAIN=$MODELSCOPE_DOMAIN \ - -e HUB_DATASET_ENDPOINT=$HUB_DATASET_ENDPOINT \ + else + docker run --rm --name $CONTAINER_NAME --shm-size=16gb \ + --cpuset-cpus=${cpu_sets_arr[$idx]} \ + --gpus='"'"device=$gpu"'"' \ + -v $CODE_DIR:$CODE_DIR_IN_CONTAINER \ + -v $MODELSCOPE_CACHE:$MODELSCOPE_CACHE_DIR_IN_CONTAINER \ + -v $MODELSCOPE_HOME_CACHE/$idx:/root \ + -v /home/admin/pre-commit:/home/admin/pre-commit \ + -e CI_TEST=True \ + -e TEST_LEVEL=$TEST_LEVEL \ + -e MODELSCOPE_CACHE=$MODELSCOPE_CACHE_DIR_IN_CONTAINER \ + -e MODELSCOPE_DOMAIN=$MODELSCOPE_DOMAIN \ + -e HUB_DATASET_ENDPOINT=$HUB_DATASET_ENDPOINT \ -e TEST_ACCESS_TOKEN_CITEST=$TEST_ACCESS_TOKEN_CITEST \ -e TEST_ACCESS_TOKEN_SDKDEV=$TEST_ACCESS_TOKEN_SDKDEV \ -e TEST_LEVEL=$TEST_LEVEL \ @@ -72,7 +77,12 @@ do -e TEST_UPLOAD_MS_TOKEN=$TEST_UPLOAD_MS_TOKEN \ -e MODEL_TAG_URL=$MODEL_TAG_URL \ -e MODELSCOPE_API_TOKEN=$MODELSCOPE_API_TOKEN \ - -e PR_CHANGED_FILES=$PR_CHANGED_FILES \ + -e PR_CHANGED_FILES=$PR_CHANGED_FILES \ + -e UV_INDEX_URL=${UV_INDEX_URL:-https://mirrors.aliyun.com/pypi/simple/} \ + -e UV_EXTRA_INDEX_URL=$UV_EXTRA_INDEX_URL \ + -e UV_INDEX_STRATEGY=$UV_INDEX_STRATEGY \ + -e UV_NO_CACHE=$UV_NO_CACHE \ + -e UV_SYSTEM_PYTHON=${UV_SYSTEM_PYTHON:-1} \ --workdir=$CODE_DIR_IN_CONTAINER \ ${IMAGE_NAME}:${IMAGE_VERSION} \ $CI_COMMAND diff --git a/.github/workflows/citest.yaml b/.github/workflows/citest.yaml index 101e9f3b1a..6bea98ea39 100644 --- a/.github/workflows/citest.yaml +++ b/.github/workflows/citest.yaml @@ -71,6 +71,11 @@ jobs: run: git lfs checkout - name: Run unittest shell: bash + env: + UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" + UV_INDEX_STRATEGY: "unsafe-best-match" + UV_NO_CACHE: 1 + UV_SYSTEM_PYTHON: 1 run: | set -e source /mnt/modelscope/ci_env.sh diff --git a/.github/workflows/citest_npu.yaml b/.github/workflows/citest_npu.yaml index eb30bcc2c2..45afef747c 100644 --- a/.github/workflows/citest_npu.yaml +++ b/.github/workflows/citest_npu.yaml @@ -43,15 +43,22 @@ jobs: timeout-minutes: 240 container: image: 'ascendai/cann:8.3.rc2-910b-ubuntu22.04-py3.11' + env: + UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" + UV_EXTRA_INDEX_URL: "https://mirrors.huaweicloud.com/ascend/repos/pypi" + UV_INDEX_STRATEGY: "unsafe-best-match" + UV_NO_CACHE: 1 + UV_SYSTEM_PYTHON: 1 steps: - name: Config mirrors run: | sed -Ei 's@(ports|archive).ubuntu.com@cache-service.nginx-pypi-cache.svc.cluster.local:8081@g' /etc/apt/sources.list pip config set global.index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple pip config set global.trusted-host cache-service.nginx-pypi-cache.svc.cluster.local + pip install uv - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }} - name: Get changed files diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 6ff84517d8..52607b46b7 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -10,13 +10,21 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python 3.10 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: '3.10' + cache: pip + - name: Cache pre-commit environments + uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} + restore-keys: pre-commit-${{ runner.os }}- - name: Install pre-commit hook run: | - pip install pre-commit + pip install uv + uv pip install --system pre-commit - name: Linting run: pre-commit run --all-files diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 7a0c7471a7..4a3a0cc241 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -20,10 +20,13 @@ jobs: with: python-version: '3.10' - name: Install wheel - run: pip install wheel packaging setuptools==69.5.1 + run: | + pip install uv + uv pip install --system wheel packaging setuptools==69.5.1 - name: Build ModelScope Swift run: python setup.py sdist bdist_wheel - name: Publish package to PyPI run: | - pip install twine + pip install uv + uv pip install --system twine twine upload dist/* --skip-existing -u __token__ -p ${{ secrets.PYPI_API_TOKEN }} diff --git a/requirements/npu.txt b/requirements/npu.txt new file mode 100644 index 0000000000..bc07bf8853 --- /dev/null +++ b/requirements/npu.txt @@ -0,0 +1,5 @@ +torch==2.7.1 +torch-npu==2.7.1.post4 +torchvision==0.22.1 +torchaudio==2.7.1 +decorator