diff --git a/docker/pytorch/Dockerfile b/docker/pytorch/Dockerfile
index 368d5e71a169..3c7ede203eef 100644
--- a/docker/pytorch/Dockerfile
+++ b/docker/pytorch/Dockerfile
@@ -5,10 +5,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
          cmake \
          git \
          curl \
-         vim \
          ca-certificates \
          libjpeg-dev \
-         libpng-dev &&\
+         libpng-dev && \
      rm -rf /var/lib/apt/lists/*
 
 
@@ -16,11 +15,10 @@ RUN curl -o ~/miniconda.sh -O  https://repo.continuum.io/miniconda/Miniconda3-la
      chmod +x ~/miniconda.sh && \
      ~/miniconda.sh -b -p /opt/conda && \
      rm ~/miniconda.sh && \
-     /opt/conda/bin/conda install -y python=$PYTHON_VERSION numpy pyyaml scipy ipython mkl mkl-include cython typing && \
+     /opt/conda/bin/conda install -y python=$PYTHON_VERSION numpy pyyaml scipy ipython mkl mkl-include ninja cython typing && \
      /opt/conda/bin/conda install -y -c pytorch magma-cuda100 && \
      /opt/conda/bin/conda clean -ya
 ENV PATH /opt/conda/bin:$PATH
-RUN pip install ninja
 # This must be done before pip so that requirements.txt is available
 WORKDIR /opt/pytorch
 COPY . .
@@ -33,4 +31,4 @@ RUN TORCH_CUDA_ARCH_LIST="3.5 5.2 6.0 6.1 7.0+PTX" TORCH_NVCC_FLAGS="-Xfatbin -c
 RUN git clone https://github.com/pytorch/vision.git && cd vision && pip install -v .
 
 WORKDIR /workspace
-RUN chmod -R a+w /workspace
+RUN chmod -R a+w .