From c69735e3b6d434913ba50a17d9b2cde944adfe2c Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Wed, 28 Jul 2021 17:52:51 +0200 Subject: [PATCH] Build swig and decoder on manylinux_2_24 --- .github/workflows/build-and-test.yml | 37 +++++++++++++++++----------- native_client/ctcdecode/Makefile | 4 +-- native_client/definitions.mk | 5 +--- native_client/python/Makefile | 2 +- 4 files changed, 27 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 68fc5b7f..ab3ab4ac 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -71,6 +71,10 @@ jobs: swig_Linux: name: "Lin|Build SWIG" runs-on: ubuntu-20.04 + container: + image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c + volumes: + - ${{ github.workspace }}:${{ github.workspace }} env: swig_hash: "90cdbee6a69d13b39d734083b9f91069533b0d7b" steps: @@ -84,10 +88,7 @@ jobs: id: swig-build-cache with: path: build-static/ - key: swig-2-${{ runner.os }}-${{ env.swig_hash }} - - run: | - sudo apt-get install -y --no-install-recommends autoconf automake bison build-essential - if: steps.swig-build-cache.outputs.cache-hit != 'true' + key: swig-4-${{ runner.os }}-${{ env.swig_hash }} - run: | curl -sSL https://ftp.pcre.org/pub/pcre/pcre-8.43.tar.gz > pcre-8.43.tar.gz if: steps.swig-build-cache.outputs.cache-hit != 'true' @@ -114,6 +115,10 @@ jobs: name: "Lin|Build CTC decoder Python package" needs: [ swig_Linux ] runs-on: ubuntu-20.04 + container: + image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c + volumes: + - ${{ github.workspace }}:${{ github.workspace }} strategy: matrix: python-version: [3.6, 3.7, 3.8, 3.9] @@ -121,12 +126,6 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - run: | - python --version - pip --version - uses: actions/download-artifact@v2 with: name: "swig_Linux" @@ -142,17 +141,27 @@ jobs: pyver: ${{ matrix.python-version }} - name: Make decoder package run: | + # Setup venv + /opt/_internal/cpython-${{ matrix.python-version }}*/bin/python -m venv /tmp/venv-${{ matrix.python-version }} + source /tmp/venv-${{ matrix.python-version }}/bin/activate + + # Check versions + python --version + pip --version + + # Build decoder package NUMPY_BUILD_VERSION="${{ steps.get_numpy.outputs.build_version }}" \ NUMPY_DEP_VERSION="${{ steps.get_numpy.outputs.dep_version }}" \ make -C native_client/ctcdecode/ \ NUM_PROCESSES=$(nproc) \ bindings + - name: Auditwheel repair + run: | + auditwheel repair native_client/ctcdecode/dist/*.whl - uses: actions/upload-artifact@v2 with: name: "coqui_stt_ctcdecoder-Linux-${{ matrix.python-version }}.whl" - path: ${{ github.workspace }}/native_client/ctcdecode/dist/*.whl - - run: | - make -C native_client/ctcdecode clean-keep-third-party + path: ${{ github.workspace }}/wheelhouse/*.whl train-test-model-Linux: name: "Lin|Train a test model" needs: [ "build-ctc-decoder-Linux" ] @@ -175,7 +184,7 @@ jobs: python --version pip --version - run: | - pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 + pip install --upgrade pip setuptools wheel - run: | pip install coqui_stt_ctcdecoder-*-cp36-cp36m-*_x86_64.whl DS_NODECODER=y pip install --upgrade . diff --git a/native_client/ctcdecode/Makefile b/native_client/ctcdecode/Makefile index 509573e6..a069e809 100644 --- a/native_client/ctcdecode/Makefile +++ b/native_client/ctcdecode/Makefile @@ -45,14 +45,14 @@ workspace_status.cc: # Enforce PATH here because swig calls from build_ext looses track of some # variables over several runs bindings: clean-keep-third-party workspace_status.cc $(DS_SWIG_DEP) - python -m pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0 + python -m pip install --quiet $(PYTHON_PACKAGES) wheel setuptools DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN_DIR):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) find temp_build -type f -name "*.o" -delete DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) rm -rf temp_build bindings-debug: clean-keep-third-party workspace_status.cc $(DS_SWIG_DEP) - python -m pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0 + python -m pip install --quiet $(PYTHON_PACKAGES) wheel setuptools DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN_DIR):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) $(GENERATE_DEBUG_SYMS) find temp_build -type f -name "*.o" -delete diff --git a/native_client/definitions.mk b/native_client/definitions.mk index 03adcdd9..e630001b 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -50,9 +50,6 @@ else SOX_LDFLAGS := `pkg-config --libs sox` endif # OS others PYTHON_PACKAGES := numpy${NUMPY_BUILD_VERSION} -ifeq ($(OS),Linux) -PYTHON_PLATFORM_NAME ?= --plat-name manylinux1_x86_64 -endif endif ifeq ($(findstring _NT,$(OS)),_NT) @@ -237,7 +234,7 @@ DS_SWIG_ENV := SWIG_LIB="$(SWIG_LIB)" PATH="$(DS_SWIG_BIN_PATH):${PATH}" $(DS_SWIG_BIN_PATH)/swig: mkdir -p $(SWIG_ROOT) - wget -O - "$(SWIG_DIST_URL)" | tar -C $(SWIG_ROOT) -zxf - + curl -sSL "$(SWIG_DIST_URL)" | tar -C $(SWIG_ROOT) -zxf - ln -s $(DS_SWIG_BIN) $(DS_SWIG_BIN_PATH)/$(SWIG_BIN) ds-swig: $(DS_SWIG_BIN_PATH)/swig diff --git a/native_client/python/Makefile b/native_client/python/Makefile index 2ead3b9e..fb313204 100644 --- a/native_client/python/Makefile +++ b/native_client/python/Makefile @@ -9,7 +9,7 @@ bindings-clean: # Enforce PATH here because swig calls from build_ext looses track of some # variables over several runs bindings-build: ds-swig - pip3 install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0 + pip3 install --quiet $(PYTHON_PACKAGES) wheel setuptools DISTUTILS_USE_SDK=1 PATH=$(TOOLCHAIN_DIR):$(DS_SWIG_BIN_PATH):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python3 ./setup.py build_ext $(PYTHON_PLATFORM_NAME) MANIFEST.in: bindings-build