diff --git a/Dockerfile.train.tmpl b/Dockerfile.train.tmpl
index 25afc455..6243c575 100644
--- a/Dockerfile.train.tmpl
+++ b/Dockerfile.train.tmpl
@@ -18,6 +18,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         unzip \
         wget
 
+# We need to remove it because it's breaking deepspeech install later with
+# weird errors about setuptools
+RUN apt-get purge -y python3-xdg
+
 WORKDIR /
 RUN git lfs install
 RUN git clone $DEEPSPEECH_REPO
@@ -25,12 +29,6 @@ RUN git clone $DEEPSPEECH_REPO
 WORKDIR /DeepSpeech
 RUN git checkout $DEEPSPEECH_SHA
 
-# Setup a virtualenv otherwise we mess with the system and this is BAD.
-RUN python3 -m venv venv/
-
-ENV VIRTUAL_ENV=/DeepSpeech/venv
-ENV PATH=$VIRTUAL_ENV/bin:$PATH
-
 # Build CTC decoder first, to avoid clashes on incompatible versions upgrades
 RUN cd native_client/ctcdecode && make NUM_PROCESSES=$(nproc) bindings
 RUN pip3 install --upgrade native_client/ctcdecode/dist/*.whl
@@ -38,7 +36,10 @@ RUN pip3 install --upgrade native_client/ctcdecode/dist/*.whl
 # Prepare deps
 RUN pip3 install --upgrade pip==20.0.2 wheel==0.34.2 setuptools==46.1.3
 
-# Install DeepSpeech, no need for the decoder since we did it earlier
-RUN DS_NODECODER=y pip3 install --upgrade --force-reinstall -e .
+# Install DeepSpeech
+#  - No need for the decoder since we did it earlier
+#  - There is already correct TensorFlow GPU installed on the base image,
+#    we don't want to break that
+RUN DS_NODECODER=y DS_NOTENSORFLOW=y pip3 install --upgrade --force-reinstall -e .
 
 RUN ./bin/run-ldc93s1.sh
diff --git a/setup.py b/setup.py
index 6811b717..91a9af5b 100644
--- a/setup.py
+++ b/setup.py
@@ -50,7 +50,6 @@ def main():
         version = fin.read().strip()
 
     install_requires_base = [
-        'tensorflow == 1.15.2',
         'numpy',
         'progressbar2',
         'six',
@@ -74,6 +73,10 @@ def main():
         'ds_ctcdecoder == {}'.format(version)
     ]
 
+    tensorflow_pypi_dep = [
+        'tensorflow == 1.15.2'
+    ]
+
     # Due to pip craziness environment variables are the only consistent way to
     # get options into this script when doing `pip install`.
     tc_decoder_artifacts_root = os.environ.get('DECODER_ARTIFACTS_ROOT', '')
@@ -87,6 +90,11 @@ def main():
     else:
         install_requires = install_requires_base + decoder_pypi_dep
 
+    if os.environ.get('DS_NOTENSORFLOW', ''):
+        install_requires = install_requires
+    else:
+        install_requires = install_requires + tensorflow_pypi_dep
+
     setup(
         name='deepspeech_training',
         version=version,