From ff401732a394b273e0e7d87fa190b9aa02b4e83b Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Fri, 31 Jan 2020 23:43:19 +0100 Subject: [PATCH] Enforce CTC decoder version check Fix #2710 --- DeepSpeech.py | 1 + evaluate.py | 1 + native_client/bazel_workspace_status_cmd.sh | 5 ++++- native_client/ctcdecode/Makefile | 13 ++++++++--- native_client/ctcdecode/__init__.py | 3 ++- native_client/ctcdecode/setup.py | 3 ++- native_client/ctcdecode/swigwrapper.i | 4 ++++ native_client/gen_workspace_status.sh | 4 ++++ native_client/workspace_status.h | 1 + requirements.txt | 1 + util/helpers.py | 25 +++++++++++++++++++++ 11 files changed, 55 insertions(+), 6 deletions(-) diff --git a/DeepSpeech.py b/DeepSpeech.py index 941eeb81..67971b48 100755 --- a/DeepSpeech.py +++ b/DeepSpeech.py @@ -27,6 +27,7 @@ from util.config import Config, initialize_globals from util.feeding import create_dataset, samples_to_mfccs, audiofile_to_features from util.flags import create_flags, FLAGS from util.logging import log_info, log_error, log_debug, log_progress, create_progressbar +from util.helpers import check_ctcdecoder_version; check_ctcdecoder_version() # Graph Creation diff --git a/evaluate.py b/evaluate.py index 01385039..4b0602c1 100755 --- a/evaluate.py +++ b/evaluate.py @@ -22,6 +22,7 @@ from util.evaluate_tools import calculate_report from util.feeding import create_dataset from util.flags import create_flags, FLAGS from util.logging import log_error, log_progress, create_progressbar +from util.helpers import check_ctcdecoder_version; check_ctcdecoder_version() def sparse_tensor_value_to_texts(value, alphabet): diff --git a/native_client/bazel_workspace_status_cmd.sh b/native_client/bazel_workspace_status_cmd.sh index dfdb3fa4..7969b19a 100755 --- a/native_client/bazel_workspace_status_cmd.sh +++ b/native_client/bazel_workspace_status_cmd.sh @@ -18,9 +18,12 @@ set -ex tf_git_rev=$(git describe --long --tags) echo "STABLE_TF_GIT_VERSION ${tf_git_rev}" -pushd native_client +# use this trick to be able to use the script from anywhere +pushd $(dirname "$0") ds_git_rev=$(git describe --long --tags) echo "STABLE_DS_GIT_VERSION ${ds_git_rev}" +ds_version=$(cat ../VERSION) +echo "STABLE_DS_VERSION ${ds_version}" ds_graph_version=$(cat ../GRAPH_VERSION) echo "STABLE_DS_GRAPH_VERSION ${ds_graph_version}" popd diff --git a/native_client/ctcdecode/Makefile b/native_client/ctcdecode/Makefile index 23319e78..eff83f38 100644 --- a/native_client/ctcdecode/Makefile +++ b/native_client/ctcdecode/Makefile @@ -1,4 +1,4 @@ -.PHONY: bindings clean +.PHONY: bindings clean workspace_status.cc include ../definitions.mk @@ -23,15 +23,22 @@ clean-keep-common: clean: clean-keep-common rm -f common.a + rm workspace_status.cc + rm -fr bazel-out/ -bindings: clean-keep-common +workspace_status.cc: + mkdir -p bazel-out/ && \ + ../bazel_workspace_status_cmd.sh > bazel-out/stable-status.txt && \ + ../gen_workspace_status.sh > $@ + +bindings: clean-keep-common workspace_status.cc pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) find temp_build -type f -name "*.o" -delete AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) rm -rf temp_build -bindings-debug: clean-keep-common +bindings-debug: clean-keep-common workspace_status.cc pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) $(GENERATE_DEBUG_SYMS) diff --git a/native_client/ctcdecode/__init__.py b/native_client/ctcdecode/__init__.py index 41fb3002..e33eeb5c 100644 --- a/native_client/ctcdecode/__init__.py +++ b/native_client/ctcdecode/__init__.py @@ -1,7 +1,8 @@ from __future__ import absolute_import, division, print_function -from . import swigwrapper +from . import swigwrapper # pylint: disable=import-self +__version__ = swigwrapper.__version__ class Scorer(swigwrapper.Scorer): """Wrapper for Scorer. diff --git a/native_client/ctcdecode/setup.py b/native_client/ctcdecode/setup.py index 80d5ca79..8a77ac31 100644 --- a/native_client/ctcdecode/setup.py +++ b/native_client/ctcdecode/setup.py @@ -60,7 +60,8 @@ decoder_module = Extension( 'ctc_beam_search_decoder.cpp', 'scorer.cpp', 'path_trie.cpp', - 'decoder_utils.cpp'], + 'decoder_utils.cpp', + 'workspace_status.cc'], swig_opts=['-c++', '-extranative'], language='c++', include_dirs=INCLUDES + [numpy_include], diff --git a/native_client/ctcdecode/swigwrapper.i b/native_client/ctcdecode/swigwrapper.i index b7ac8b11..d3e70898 100644 --- a/native_client/ctcdecode/swigwrapper.i +++ b/native_client/ctcdecode/swigwrapper.i @@ -4,6 +4,7 @@ #include "ctc_beam_search_decoder.h" #define SWIG_FILE_WITH_INIT #define SWIG_PYTHON_STRICT_BYTE_CHAR +#include "workspace_status.h" %} %include "pyabc.i" @@ -27,6 +28,9 @@ import_array(); %include "scorer.h" %include "ctc_beam_search_decoder.h" +%constant const char* __version__ = ds_version(); +%constant const char* __git_version__ = ds_git_version(); + %template(IntVector) std::vector; %template(OutputVector) std::vector; %template(OutputVectorVector) std::vector>; diff --git a/native_client/gen_workspace_status.sh b/native_client/gen_workspace_status.sh index 8b2755d8..298905bd 100755 --- a/native_client/gen_workspace_status.sh +++ b/native_client/gen_workspace_status.sh @@ -3,6 +3,7 @@ set -x tf_git_version=$(grep "STABLE_TF_GIT_VERSION" "bazel-out/stable-status.txt" | cut -d' ' -f2) +ds_version=$(grep "STABLE_DS_VERSION" "bazel-out/stable-status.txt" | cut -d' ' -f2) ds_git_version=$(grep "STABLE_DS_GIT_VERSION" "bazel-out/stable-status.txt" | cut -d' ' -f2) ds_graph_version=$(grep "STABLE_DS_GRAPH_VERSION" "bazel-out/stable-status.txt" | cut -d' ' -f2) @@ -10,6 +11,9 @@ cat < 0: + print("DeepSpeech version ({ds_version}) requires CTC decoder to expose __version__. Please upgrade the ds_ctcdecoder package to version {ds_version}".format(ds_version=ds_version_s)) + sys.exit(1) + raise e + + decoder_version_s = decoder_version.decode() + + rv = semver.compare(ds_version_s, decoder_version_s) + if rv != 0: + print("DeepSpeech version ({}) and CTC decoder version ({}) do not match. Please ensure matching versions are in use.".format(ds_version_s, decoder_version_s)) + sys.exit(1) + + return rv