mirror of
https://github.com/modelscope/ms-swift.git
synced 2024-11-25 18:32:44 +08:00
Commit missing files
This commit is contained in:
yuze.zyz
parent
a61c11b704
commit
90d86c3b09
38
.dev_scripts/ci_container_test.sh
Normal file
38
.dev_scripts/ci_container_test.sh
Normal file
@ -0,0 +1,38 @@
|
||||
if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then
|
||||
# pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
pip install -r requirements/tests.txt -i https://mirrors.aliyun.com/pypi/simple/
|
||||
git config --global --add safe.directory /swift
|
||||
git config --global user.email tmp
|
||||
git config --global user.name tmp.com
|
||||
|
||||
# linter test
|
||||
# use internal project for pre-commit due to the network problem
|
||||
if [ `git remote -v | grep alibaba | wc -l` -gt 1 ]; then
|
||||
pre-commit run -c .pre-commit-config_local.yaml --all-files
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "linter test failed, please run 'pre-commit run --all-files' to check"
|
||||
echo "From the repository folder"
|
||||
echo "Run 'pip install -r requirements/tests.txt' install test dependencies."
|
||||
echo "Run 'pre-commit install' install pre-commit hooks."
|
||||
echo "Finally run linter with command: 'pre-commit run --all-files' to check."
|
||||
echo "Ensure there is no failure!!!!!!!!"
|
||||
exit -1
|
||||
fi
|
||||
fi
|
||||
|
||||
pip install -r requirements/framework.txt -i https://mirrors.aliyun.com/pypi/simple/
|
||||
|
||||
# test with install
|
||||
pip install .
|
||||
else
|
||||
echo "Running case in release image, run case directly!"
|
||||
fi
|
||||
# remove torch_extensions folder to avoid ci hang.
|
||||
rm -rf ~/.cache/torch_extensions
|
||||
if [ $# -eq 0 ]; then
|
||||
ci_command="python tests/run.py --subprocess"
|
||||
else
|
||||
ci_command="$@"
|
||||
fi
|
||||
echo "Running case with command: $ci_command"
|
||||
$ci_command
|
||||
82
.dev_scripts/dockerci.sh
Normal file
82
.dev_scripts/dockerci.sh
Normal file
@ -0,0 +1,82 @@
|
||||
#!/bin/bash
|
||||
MODELSCOPE_CACHE_DIR_IN_CONTAINER=/modelscope_cache
|
||||
CODE_DIR=$PWD
|
||||
CODE_DIR_IN_CONTAINER=/swift
|
||||
echo "$USER"
|
||||
gpus='0,1 2,3 4,5 6,7'
|
||||
cpu_sets='45-58 31-44 16-30 0-15'
|
||||
cpu_sets_arr=($cpu_sets)
|
||||
is_get_file_lock=false
|
||||
CI_COMMAND=${CI_COMMAND:-bash .dev_scripts/ci_container_test.sh python tests/run.py --parallel 2 --run_config tests/run_config.yaml}
|
||||
echo "ci command: $CI_COMMAND"
|
||||
idx=0
|
||||
for gpu in $gpus
|
||||
do
|
||||
exec {lock_fd}>"/tmp/gpu$gpu" || exit 1
|
||||
flock -n "$lock_fd" || { echo "WARN: gpu $gpu is in use!" >&2; idx=$((idx+1)); continue; }
|
||||
echo "get gpu lock $gpu"
|
||||
|
||||
CONTAINER_NAME="swift-ci-$idx"
|
||||
let is_get_file_lock=true
|
||||
|
||||
# pull image if there are update
|
||||
docker pull ${IMAGE_NAME}:${IMAGE_VERSION}
|
||||
if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then
|
||||
echo 'debugging'
|
||||
docker run --rm --name $CONTAINER_NAME --shm-size=16gb \
|
||||
--cpuset-cpus=${cpu_sets_arr[$idx]} \
|
||||
--gpus='"'"device=$gpu"'"' \
|
||||
-v $CODE_DIR:$CODE_DIR_IN_CONTAINER \
|
||||
-v $MODELSCOPE_CACHE:$MODELSCOPE_CACHE_DIR_IN_CONTAINER \
|
||||
-v $MODELSCOPE_HOME_CACHE/$idx:/root \
|
||||
-v /home/admin/pre-commit:/home/admin/pre-commit \
|
||||
-e CI_TEST=True \
|
||||
-e TEST_LEVEL=$TEST_LEVEL \
|
||||
-e MODELSCOPE_CACHE=$MODELSCOPE_CACHE_DIR_IN_CONTAINER \
|
||||
-e MODELSCOPE_DOMAIN=$MODELSCOPE_DOMAIN \
|
||||
-e MODELSCOPE_SDK_DEBUG=True \
|
||||
-e HUB_DATASET_ENDPOINT=$HUB_DATASET_ENDPOINT \
|
||||
-e TEST_ACCESS_TOKEN_CITEST=$TEST_ACCESS_TOKEN_CITEST \
|
||||
-e TEST_ACCESS_TOKEN_SDKDEV=$TEST_ACCESS_TOKEN_SDKDEV \
|
||||
-e TEST_LEVEL=$TEST_LEVEL \
|
||||
-e MODELSCOPE_ENVIRONMENT='ci' \
|
||||
-e TEST_UPLOAD_MS_TOKEN=$TEST_UPLOAD_MS_TOKEN \
|
||||
-e MODEL_TAG_URL=$MODEL_TAG_URL \
|
||||
-e MODELSCOPE_API_TOKEN=$MODELSCOPE_API_TOKEN \
|
||||
--workdir=$CODE_DIR_IN_CONTAINER \
|
||||
${IMAGE_NAME}:${IMAGE_VERSION} \
|
||||
$CI_COMMAND
|
||||
else
|
||||
docker run --rm --name $CONTAINER_NAME --shm-size=16gb \
|
||||
--cpuset-cpus=${cpu_sets_arr[$idx]} \
|
||||
--gpus='"'"device=$gpu"'"' \
|
||||
-v $CODE_DIR:$CODE_DIR_IN_CONTAINER \
|
||||
-v $MODELSCOPE_CACHE:$MODELSCOPE_CACHE_DIR_IN_CONTAINER \
|
||||
-v $MODELSCOPE_HOME_CACHE/$idx:/root \
|
||||
-v /home/admin/pre-commit:/home/admin/pre-commit \
|
||||
-e CI_TEST=True \
|
||||
-e TEST_LEVEL=$TEST_LEVEL \
|
||||
-e MODELSCOPE_CACHE=$MODELSCOPE_CACHE_DIR_IN_CONTAINER \
|
||||
-e MODELSCOPE_DOMAIN=$MODELSCOPE_DOMAIN \
|
||||
-e HUB_DATASET_ENDPOINT=$HUB_DATASET_ENDPOINT \
|
||||
-e TEST_ACCESS_TOKEN_CITEST=$TEST_ACCESS_TOKEN_CITEST \
|
||||
-e TEST_ACCESS_TOKEN_SDKDEV=$TEST_ACCESS_TOKEN_SDKDEV \
|
||||
-e TEST_LEVEL=$TEST_LEVEL \
|
||||
-e MODELSCOPE_ENVIRONMENT='ci' \
|
||||
-e TEST_UPLOAD_MS_TOKEN=$TEST_UPLOAD_MS_TOKEN \
|
||||
-e MODEL_TAG_URL=$MODEL_TAG_URL \
|
||||
-e MODELSCOPE_API_TOKEN=$MODELSCOPE_API_TOKEN \
|
||||
--workdir=$CODE_DIR_IN_CONTAINER \
|
||||
${IMAGE_NAME}:${IMAGE_VERSION} \
|
||||
$CI_COMMAND
|
||||
fi
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Running test case failed, please check the log!"
|
||||
exit -1
|
||||
fi
|
||||
break
|
||||
done
|
||||
if [ "$is_get_file_lock" = false ] ; then
|
||||
echo 'No free GPU!'
|
||||
exit 1
|
||||
fi
|
||||
133
.gitignore
vendored
Normal file
133
.gitignore
vendored
Normal file
@ -0,0 +1,133 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
test.py
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
/package
|
||||
/temp
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# celery beat schedule file
|
||||
celerybeat-schedule
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
|
||||
.vscode
|
||||
.idea
|
||||
|
||||
# custom
|
||||
*.pkl
|
||||
*.pkl.json
|
||||
*.log.json
|
||||
*.whl
|
||||
*.tar.gz
|
||||
*.swp
|
||||
*.log
|
||||
*.tar.gz
|
||||
source.sh
|
||||
tensorboard.sh
|
||||
.DS_Store
|
||||
replace.sh
|
||||
result.png
|
||||
result.jpg
|
||||
result.mp4
|
||||
|
||||
# Pytorch
|
||||
*.pth
|
||||
*.pt
|
||||
|
||||
# ast template
|
||||
ast_index_file.py
|
||||
51
.pre-commit-config.yaml
Normal file
51
.pre-commit-config.yaml
Normal file
@ -0,0 +1,51 @@
|
||||
repos:
|
||||
- repo: https://github.com/pycqa/flake8.git
|
||||
rev: 4.0.0
|
||||
hooks:
|
||||
- id: flake8
|
||||
exclude: |
|
||||
(?x)^(
|
||||
thirdparty/|
|
||||
examples/|
|
||||
tests/run.py
|
||||
)$
|
||||
- repo: https://github.com/PyCQA/isort.git
|
||||
rev: 4.3.21
|
||||
hooks:
|
||||
- id: isort
|
||||
exclude: |
|
||||
(?x)^(
|
||||
examples/|
|
||||
tests/run.py
|
||||
)$
|
||||
- repo: https://github.com/pre-commit/mirrors-yapf.git
|
||||
rev: v0.30.0
|
||||
hooks:
|
||||
- id: yapf
|
||||
exclude: |
|
||||
(?x)^(
|
||||
thirdparty/|
|
||||
examples/|
|
||||
tests/run.py
|
||||
)$
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks.git
|
||||
rev: v3.1.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
exclude: thirdparty/|tests/run.py
|
||||
- id: check-yaml
|
||||
exclude: thirdparty/|tests/run.py
|
||||
- id: end-of-file-fixer
|
||||
exclude: thirdparty/|tests/run.py
|
||||
- id: requirements-txt-fixer
|
||||
exclude: thirdparty/|tests/run.py
|
||||
- id: double-quote-string-fixer
|
||||
exclude: thirdparty/|tests/run.py
|
||||
- id: check-merge-conflict
|
||||
exclude: thirdparty/|tests/run.py
|
||||
- id: fix-encoding-pragma
|
||||
exclude: thirdparty/|tests/run.py
|
||||
args: ["--remove"]
|
||||
- id: mixed-line-ending
|
||||
exclude: thirdparty/|tests/run.py
|
||||
args: ["--fix=lf"]
|
||||
51
.pre-commit-config_local.yaml
Normal file
51
.pre-commit-config_local.yaml
Normal file
@ -0,0 +1,51 @@
|
||||
repos:
|
||||
- repo: /home/admin/pre-commit/flake8
|
||||
rev: 4.0.0
|
||||
hooks:
|
||||
- id: flake8
|
||||
exclude: |
|
||||
(?x)^(
|
||||
thirdparty/|
|
||||
examples/|
|
||||
tests/run.py
|
||||
)$
|
||||
- repo: /home/admin/pre-commit/isort
|
||||
rev: 4.3.21
|
||||
hooks:
|
||||
- id: isort
|
||||
exclude: |
|
||||
(?x)^(
|
||||
examples/|
|
||||
tests/run.py
|
||||
)$
|
||||
- repo: /home/admin/pre-commit/mirrors-yapf
|
||||
rev: v0.30.0
|
||||
hooks:
|
||||
- id: yapf
|
||||
exclude: |
|
||||
(?x)^(
|
||||
thirdparty/|
|
||||
examples/|
|
||||
tests/run.py
|
||||
)$
|
||||
- repo: /home/admin/pre-commit/pre-commit-hooks
|
||||
rev: v3.1.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
exclude: thirdparty/|tests/run.py
|
||||
- id: check-yaml
|
||||
exclude: thirdparty/|tests/run.py
|
||||
- id: end-of-file-fixer
|
||||
exclude: thirdparty/
|
||||
- id: requirements-txt-fixer
|
||||
exclude: thirdparty/|tests/run.py
|
||||
- id: double-quote-string-fixer
|
||||
exclude: thirdparty/|tests/run.py
|
||||
- id: check-merge-conflict
|
||||
exclude: thirdparty/|tests/run.py
|
||||
- id: fix-encoding-pragma
|
||||
exclude: thirdparty/|tests/run.py
|
||||
args: ["--remove"]
|
||||
- id: mixed-line-ending
|
||||
exclude: thirdparty/|tests/run.py
|
||||
args: ["--fix=lf"]
|
||||
Loading…
Reference in New Issue
Block a user