experiments/STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove extra installs from Dockerfile

Browse Source
This commit is contained in:
Josh Meyer 2021-07-16 11:14:00 -04:00
parent f062f75e17
commit 649bc53536
3 changed files with 13 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
Dockerfile.train.jupyter
View File

@ -40,6 +40,7 @@ RUN wget --no-check-certificate https://github.com/coqui-ai/STT/releases/downloa
rm temp.tar.xz rm temp.tar.xz
#FROM gcr.io/kaggle-gpu-images/python:v98
FROM nvcr.io/nvidia/tensorflow:21.05-tf1-py3 FROM nvcr.io/nvidia/tensorflow:21.05-tf1-py3
ENV DEBIAN_FRONTEND=noninteractive ENV DEBIAN_FRONTEND=noninteractive
@ -97,7 +98,7 @@ RUN rm -rf ~/.local/share/stt
## ##
# #
# FROM gcr.io/kaggle-gpu-images/python:v98 #FROM gcr.io/kaggle-gpu-images/python:v98
RUN chsh -s /bin/bash RUN chsh -s /bin/bash
ENV SHELL=/bin/bash ENV SHELL=/bin/bash
@ -119,74 +120,22 @@ RUN apt-get update && apt-get install -y \
supervisor \ supervisor \
gettext-base \ gettext-base \
less \ less \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
# install nvm RUN pip3 install jupyterlab ipywidgets
# https://github.com/creationix/nvm#install-script
RUN curl --silent -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash
#ENV NVM_DIR /root/.nvm
ENV NVM_DIR /usr/local/nvm
ENV NODE_VERSION v12.20.1
# install node and npm
RUN source $NVM_DIR/nvm.sh \
&& nvm install $NODE_VERSION \
&& nvm alias default $NODE_VERSION \
&& nvm use default
# add node and npm to path so the commands are available
ENV NODE_PATH $NVM_DIR/versions/node/$NODE_VERSION/bin
ENV PATH $NODE_PATH:$PATH
RUN pip install pip==20.3.4
RUN pip install jupyterlab==2.2.9 ipywidgets==7.6.3
RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager
RUN jupyter nbextension enable --py widgetsnbextension #enable ipywidgets RUN jupyter nbextension enable --py widgetsnbextension #enable ipywidgets
RUN jupyter labextension install jupyterlab-plotly@4.14.3 RUN jupyter labextension install jupyterlab-plotly
#install xlrd python module # #### tensorflow 1
RUN pip install pyradiomics # # Create the environment:
RUN pip install xlrd==1.2.0 # SHELL ["/bin/bash", "-c"]
RUN pip install zarr # COPY tfenv.yml .
RUN pip install imbalanced-learn # RUN conda env create -f tfenv.yml
RUN pip install openpyxl # SHELL ["conda", "run", "-n", "tf1", "/bin/bash", "-c"]
RUN pip install efficientnet-pytorch # RUN python -m ipykernel install --name=tensorflow1
RUN pip install monai
RUN pip install prince
RUN pip install vit-pytorch
RUN pip install lifelines==0.25.11
RUN pip install timm==0.3.2
RUN pip install keras-retinanet==1.0.0
RUN python -m pip install histomicstk --find-links https://girder.github.io/large_image_wheels
RUN pip install luminoth ipympl pysurvival missingpy pyinform pingouin pyAgrum missingno autoimpute networkx community yellowbrick factor_analyzer hdbscan pyitlib
RUN pip install eli5 dtreeviz gower batchgenerators mlinsights efficientnet-pytorch pretrainedmodels
# Add R to Jupyter Kernel
RUN conda install -y -c r r-irkernel
#Install survival, sm, ggplot2, Hmisc, mixOmics (ce dernier est en repositoire Bioconductor)
RUN conda install -y -c cran r-survival
RUN conda install -y -c cran r-sm
RUN conda install -y -c cran r-ggplot2
RUN conda install -y -c cran r-hmisc
RUN conda install -y -c cran r-mixomics
RUN conda install -y -c cran r-caret
RUN conda install -y -c cran r-survminer
RUN conda install -y -c cran r-ggfortify
RUN conda install -y -c cran r-wordcloud
RUN conda install -y -c cran r-tm
RUN conda install -y -c cran r-prioritylasso
RUN conda install -y -c cran r-blockforest
RUN conda install -y -c cran r-mice
#### tensorflow 1
# Create the environment:
SHELL ["/bin/bash", "-c"]
COPY tfenv.yml .
RUN conda env create -f tfenv.yml
SHELL ["conda", "run", "-n", "tf1", "/bin/bash", "-c"]
RUN python -m ipykernel install --name=tensorflow1
EXPOSE 8080 EXPOSE 8080

97
Dockerfile.train.jupyter.simple
View File

@ -1,97 +0,0 @@
# This is a Dockerfile useful for training models with Coqui STT.
# You can train "acoustic models" with audio + Tensorflow, and
# you can create "scorers" with text + KenLM.
FROM ubuntu:20.04 AS kenlm-build
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
apt-get install -y --no-install-recommends \
build-essential cmake libboost-system-dev \
libboost-thread-dev libboost-program-options-dev \
libboost-test-dev libeigen3-dev zlib1g-dev \
libbz2-dev liblzma-dev && \
rm -rf /var/lib/apt/lists/*
# Build KenLM to generate new scorers
WORKDIR /code
COPY kenlm /code/kenlm
RUN cd /code/kenlm && \
mkdir -p build && \
cd build && \
cmake .. && \
make -j $(nproc) || \
( echo "ERROR: Failed to build KenLM."; \
echo "ERROR: Make sure you update the kenlm submodule on host before building this Dockerfile."; \
echo "ERROR: $ cd STT; git submodule update --init kenlm"; \
exit 1; )
FROM ubuntu:20.04 AS wget-binaries
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends wget unzip xz-utils
# Tool to convert output graph for inference
RUN wget --no-check-certificate https://github.com/coqui-ai/STT/releases/download/v0.9.3/convert_graphdef_memmapped_format.linux.amd64.zip -O temp.zip && \
unzip temp.zip && \
rm temp.zip
RUN wget --no-check-certificate https://github.com/coqui-ai/STT/releases/download/v0.9.3/native_client.tf.Linux.tar.xz -O temp.tar.xz && \
tar -xf temp.tar.xz && \
rm temp.tar.xz
FROM jupyter/tensorflow-notebook
ENV DEBIAN_FRONTEND=noninteractive
USER root
# We need to purge python3-xdg because
# it's breaking STT install later with
# errors about setuptools
#
RUN apt-get update && \
apt-get install -y --no-install-recommends \
git \
wget \
libopus0 \
libopusfile0 \
libsndfile1 \
sox \
libsox-fmt-mp3 && \
apt-get purge -y python3-xdg && \
rm -rf /var/lib/apt/lists/
# Make sure pip and its dependencies are up-to-date
RUN pip3 install --upgrade pip wheel setuptools
RUN pip3 uninstall -y tensorflow && pip3 install -y 'tensorflow-gpu==1.15.4'
WORKDIR /code
COPY native_client /code/native_client
COPY .git /code/.git
COPY training/coqui_stt_training/VERSION /code/training/coqui_stt_training/VERSION
COPY training/coqui_stt_training/GRAPH_VERSION /code/training/coqui_stt_training/GRAPH_VERSION
# Build CTC decoder first, to avoid clashes on incompatible versions upgrades
RUN cd native_client/ctcdecode && make NUM_PROCESSES=$(nproc) bindings
RUN pip3 install --upgrade native_client/ctcdecode/dist/*.whl
COPY setup.py /code/setup.py
COPY VERSION /code/VERSION
COPY training /code/training
# Copy files from previous build stages
RUN mkdir -p /code/kenlm/build/
COPY --from=kenlm-build /code/kenlm/build/bin /code/kenlm/build/bin
COPY --from=wget-binaries /convert_graphdef_memmapped_format /code/convert_graphdef_memmapped_format
COPY --from=wget-binaries /generate_scorer_package /code/generate_scorer_package
# Install STT
# No need for the decoder since we did it earlier
# TensorFlow GPU should already be installed on the base image,
# and we don't want to break that
RUN DS_NODECODER=y DS_NOTENSORFLOW=y pip3 install --upgrade -e .
# Copy rest of the code and test training
COPY . /code
RUN ./bin/run-ldc93s1.sh && rm -rf ~/.local/share/stt

0
start.sh Normal file → Executable file
View File