mirror of
https://github.com/csukuangfj/kaldifeat.git
synced 2025-08-09 18:12:17 +00:00
Publish to PyPI.
This commit is contained in:
Fangjun Kuang
parent
40a9906ad2
commit
59c26b0cca
85
.github/workflows/publish_to_pypi.yml
vendored
85
.github/workflows/publish_to_pypi.yml
vendored
@ -1,36 +1,91 @@
|
||||
name: Publish to PyPI
|
||||
|
||||
on:
|
||||
# push:
|
||||
# tags:
|
||||
# - '*'
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
branches:
|
||||
- master
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
pypi:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-18.04]
|
||||
cuda: ["10.1"]
|
||||
gcc: ["5"]
|
||||
torch: ["1.8.1"]
|
||||
python-version: [3.6, 3.7, 3.8]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.6
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install Python dependencies
|
||||
- name: Install CUDA Toolkit ${{ matrix.cuda }}
|
||||
shell: bash
|
||||
env:
|
||||
cuda: ${{ matrix.cuda }}
|
||||
run: |
|
||||
source ./scripts/github_actions/install_cuda.sh
|
||||
echo "CUDA_HOME=${CUDA_HOME}" >> $GITHUB_ENV
|
||||
echo "${CUDA_HOME}/bin" >> $GITHUB_PATH
|
||||
echo "LD_LIBRARY_PATH=${CUDA_HOME}/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
|
||||
|
||||
- name: Display NVCC version
|
||||
run: |
|
||||
which nvcc
|
||||
nvcc --version
|
||||
|
||||
- name: Install GCC ${{ matrix.gcc }}
|
||||
run: |
|
||||
sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }}
|
||||
echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV
|
||||
echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV
|
||||
echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV
|
||||
|
||||
- name: Install PyTorch ${{ matrix.torch }}
|
||||
env:
|
||||
cuda: ${{ matrix.cuda }}
|
||||
torch: ${{ matrix.torch }}
|
||||
shell: bash
|
||||
run: |
|
||||
python3 -m pip install --upgrade pip
|
||||
python3 -m pip install wheel twine setuptools
|
||||
python3 -m pip install wheel twine typing_extensions
|
||||
python3 -m pip install bs4 requests tqdm
|
||||
|
||||
- name: Build
|
||||
shell: bash
|
||||
run: |
|
||||
python3 setup.py sdist
|
||||
ls -l dist/*
|
||||
./scripts/github_actions/install_torch.sh
|
||||
python3 -c "import torch; print('torch version:', torch.__version__)"
|
||||
|
||||
- name: Publish wheels to PyPI
|
||||
- name: Download cudnn 8.0
|
||||
env:
|
||||
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
|
||||
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
|
||||
cuda: ${{ matrix.cuda }}
|
||||
run: |
|
||||
twine upload dist/kaldifeat-*.tar.gz
|
||||
./scripts/github_actions/install_cudnn.sh
|
||||
|
||||
- name: Build pip packages
|
||||
shell: bash
|
||||
env:
|
||||
KALDIFEAT_IS_FOR_PYPI: 1
|
||||
run: |
|
||||
tag=$(python3 -c "import sys; print(''.join(sys.version[:3].split('.')))")
|
||||
export KALDIFEAT_MAKE_ARGS="-j2"
|
||||
python3 setup.py bdist_wheel --python-tag=py${tag}
|
||||
ls -lh dist/
|
||||
|
||||
# - name: Publish wheels to PyPI
|
||||
# env:
|
||||
# TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
|
||||
# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
|
||||
# run: |
|
||||
# twine upload dist/kaldifeat-*.whl
|
||||
|
||||
@ -9,6 +9,12 @@ from pathlib import Path
|
||||
import setuptools
|
||||
from setuptools.command.build_ext import build_ext
|
||||
|
||||
|
||||
def is_for_pypi():
|
||||
ans = os.environ.get("KALDIFEAT_IS_FOR_PYPI", None)
|
||||
return ans is not None
|
||||
|
||||
|
||||
try:
|
||||
from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
|
||||
|
||||
@ -17,11 +23,12 @@ try:
|
||||
_bdist_wheel.finalize_options(self)
|
||||
# In this case, the generated wheel has a name in the form
|
||||
# k2-xxx-pyxx-none-any.whl
|
||||
# self.root_is_pure = True
|
||||
|
||||
# The generated wheel has a name ending with
|
||||
# -linux_x86_64.whl
|
||||
self.root_is_pure = False
|
||||
if is_for_pypi():
|
||||
self.root_is_pure = True
|
||||
else:
|
||||
# The generated wheel has a name ending with
|
||||
# -linux_x86_64.whl
|
||||
self.root_is_pure = False
|
||||
|
||||
|
||||
except ImportError:
|
||||
|
||||
59
scripts/github_actions/install_cuda.sh
Executable file
59
scripts/github_actions/install_cuda.sh
Executable file
@ -0,0 +1,59 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2020 Mobvoi Inc. (authors: Fangjun Kuang)
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
echo "cuda version: $cuda"
|
||||
|
||||
case "$cuda" in
|
||||
10.0)
|
||||
url=https://developer.nvidia.com/compute/cuda/10.0/Prod/local_installers/cuda_10.0.130_410.48_linux
|
||||
;;
|
||||
10.1)
|
||||
# WARNING: there are bugs in
|
||||
# https://developer.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.105_418.39_linux.run
|
||||
# with GCC 7. Please use the following version
|
||||
url=http://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_418.87.00_linux.run
|
||||
;;
|
||||
10.2)
|
||||
url=http://developer.download.nvidia.com/compute/cuda/10.2/Prod/local_installers/cuda_10.2.89_440.33.01_linux.run
|
||||
;;
|
||||
11.0)
|
||||
url=http://developer.download.nvidia.com/compute/cuda/11.0.2/local_installers/cuda_11.0.2_450.51.05_linux.run
|
||||
;;
|
||||
11.1)
|
||||
# url=https://developer.download.nvidia.com/compute/cuda/11.1.0/local_installers/cuda_11.1.0_455.23.05_linux.run
|
||||
url=https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda_11.1.1_455.32.00_linux.run
|
||||
;;
|
||||
*)
|
||||
echo "Unknown cuda version: $cuda"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
function retry() {
|
||||
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
|
||||
}
|
||||
|
||||
retry curl -LSs -O $url
|
||||
filename=$(basename $url)
|
||||
echo "filename: $filename"
|
||||
chmod +x ./$filename
|
||||
sudo ./$filename --toolkit --silent
|
||||
rm -fv ./$filename
|
||||
|
||||
export CUDA_HOME=/usr/local/cuda
|
||||
export PATH=$CUDA_HOME/bin:$PATH
|
||||
export LD_LIBRARY_PATH=$CUDA_HOME/lib:$LD_LIBRARY_PATH
|
||||
export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH
|
||||
58
scripts/github_actions/install_cudnn.sh
Executable file
58
scripts/github_actions/install_cudnn.sh
Executable file
@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2020 Mobvoi Inc. (authors: Fangjun Kuang)
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
case $cuda in
|
||||
10.0)
|
||||
filename=cudnn-10.0-linux-x64-v7.6.5.32.tgz
|
||||
url=http://www.mediafire.com/file/1037lb1vmj9qdtq/cudnn-10.0-linux-x64-v7.6.5.32.tgz/file
|
||||
;;
|
||||
10.1)
|
||||
filename=cudnn-10.1-linux-x64-v8.0.2.39.tgz
|
||||
url=http://www.mediafire.com/file/fnl2wg0h757qhd7/cudnn-10.1-linux-x64-v8.0.2.39.tgz/file
|
||||
;;
|
||||
10.2)
|
||||
filename=cudnn-10.2-linux-x64-v8.0.2.39.tgz
|
||||
url=http://www.mediafire.com/file/sc2nvbtyg0f7ien/cudnn-10.2-linux-x64-v8.0.2.39.tgz/file
|
||||
;;
|
||||
11.0)
|
||||
filename=cudnn-11.0-linux-x64-v8.0.5.39.tgz
|
||||
url=https://www.mediafire.com/file/abyhnls106ko9kp/cudnn-11.0-linux-x64-v8.0.5.39.tgz/file
|
||||
;;
|
||||
11.1)
|
||||
filename=cudnn-11.1-linux-x64-v8.0.5.39.tgz
|
||||
url=https://www.mediafire.com/file/qx55zd65773xonv/cudnn-11.1-linux-x64-v8.0.5.39.tgz/file
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported cuda version: $cuda"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
function retry() {
|
||||
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
|
||||
}
|
||||
|
||||
# It is forked from https://github.com/Juvenal-Yescas/mediafire-dl
|
||||
# https://github.com/Juvenal-Yescas/mediafire-dl/pull/2 changes the filename and breaks the CI.
|
||||
# We use a separate fork to keep the link fixed.
|
||||
retry wget https://raw.githubusercontent.com/csukuangfj/mediafire-dl/master/mediafire_dl.py
|
||||
|
||||
sed -i 's/quiet=False/quiet=True/' mediafire_dl.py
|
||||
retry python3 mediafire_dl.py "$url"
|
||||
sudo tar xf ./$filename -C /usr/local
|
||||
rm -v ./$filename
|
||||
|
||||
sudo sed -i '59i#define CUDNN_MAJOR 8' /usr/local/cuda/include/cudnn.h
|
||||
108
scripts/github_actions/install_torch.sh
Executable file
108
scripts/github_actions/install_torch.sh
Executable file
@ -0,0 +1,108 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2020 Mobvoi Inc. (authors: Fangjun Kuang)
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
case ${torch} in
|
||||
1.5.*)
|
||||
case ${cuda} in
|
||||
10.1)
|
||||
package="torch==${torch}+cu101"
|
||||
url=https://download.pytorch.org/whl/torch_stable.html
|
||||
;;
|
||||
10.2)
|
||||
package="torch==${torch}"
|
||||
# Leave url empty to use PyPI.
|
||||
# torch_stable provides cu92 but we want cu102
|
||||
url=
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
1.6.0)
|
||||
case ${cuda} in
|
||||
10.1)
|
||||
package="torch==1.6.0+cu101"
|
||||
url=https://download.pytorch.org/whl/torch_stable.html
|
||||
;;
|
||||
10.2)
|
||||
package="torch==1.6.0"
|
||||
# Leave it empty to use PyPI.
|
||||
# torch_stable provides cu92 but we want cu102
|
||||
url=
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
1.7.*)
|
||||
case ${cuda} in
|
||||
10.1)
|
||||
package="torch==${torch}+cu101"
|
||||
url=https://download.pytorch.org/whl/torch_stable.html
|
||||
;;
|
||||
10.2)
|
||||
package="torch==${torch}"
|
||||
# Leave it empty to use PyPI.
|
||||
# torch_stable provides cu92 but we want cu102
|
||||
url=
|
||||
;;
|
||||
11.0)
|
||||
package="torch==${torch}+cu110"
|
||||
url=https://download.pytorch.org/whl/torch_stable.html
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
1.8.*)
|
||||
case ${cuda} in
|
||||
10.1)
|
||||
package="torch==${torch}+cu101"
|
||||
url=https://download.pytorch.org/whl/torch_stable.html
|
||||
;;
|
||||
10.2)
|
||||
package="torch==${torch}"
|
||||
# Leave it empty to use PyPI.
|
||||
url=
|
||||
;;
|
||||
11.1)
|
||||
package="torch==${torch}+cu111"
|
||||
url=https://download.pytorch.org/whl/torch_stable.html
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
1.9.0)
|
||||
case ${cuda} in
|
||||
10.2)
|
||||
package="torch==${torch}"
|
||||
# Leave it empty to use PyPI.
|
||||
url=
|
||||
;;
|
||||
11.1)
|
||||
package="torch==${torch}+cu111"
|
||||
url=https://download.pytorch.org/whl/torch_stable.html
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported PyTorch version: ${torch}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
function retry() {
|
||||
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
|
||||
}
|
||||
|
||||
if [ x"${url}" == "x" ]; then
|
||||
retry python3 -m pip install -q $package
|
||||
else
|
||||
retry python3 -m pip install -q $package -f $url
|
||||
fi
|
||||
Loading…
x
Reference in New Issue
Block a user