Backport transposes optimization to v0.3.0 (#3994)

* Optimizer: optimize transposes in variety of circumstances (#3509)

* Optimizer: Optimize transposes in variety of circumstances

- No-op transposes
- Consecutive transposes (fuse them)
- Transposes into Gemm (fuse them into transA/transB parameter)

* touch up out of date comment

* Backporting optimizer changes

.dockerignore

@@ -0,0 +1 @@
+.gitignore

.gitignore

@@ -2,19 +2,32 @@ build/
 dist/
 torch.egg-info/
 */**/__pycache__
+torch/version.py
 torch/csrc/generic/TensorMethods.cpp
 torch/lib/*.so*
+torch/lib/*.a*
 torch/lib/*.dylib*
 torch/lib/*.h
 torch/lib/build
 torch/lib/tmp_install
 torch/lib/include
 torch/lib/torch_shm_manager
+torch/csrc/jit/generated/*
+torch/csrc/autograd/generated/*
 torch/csrc/cudnn/cuDNN.cpp
 torch/csrc/nn/THNN.cwrap
 torch/csrc/nn/THNN.cpp
 torch/csrc/nn/THCUNN.cwrap
 torch/csrc/nn/THCUNN.cpp
+torch/csrc/nn/THNN_generic.cwrap
+torch/csrc/nn/THNN_generic.cpp
+torch/csrc/nn/THNN_generic.h
+torch/csrc/generated
+docs/src/**/*
+test/data/legacy_modules.t7
+test/data/gpu_tensors.pt
+test/htmlcov
+test/.coverage
 */*.pyc
 */**/*.pyc
 */**/**/*.pyc
@@ -23,3 +36,18 @@ torch/csrc/nn/THCUNN.cpp
 */*.so*
 */**/*.so*
 */**/*.dylib*
+test/data/legacy_serialized.pt
+test/data/linear.pt
+
+# IPython notebook checkpoints
+.ipynb_checkpoints
+
+# Editor temporaries
+*.swn
+*.swo
+*.swp
+*~
+
+# OSX dir files
+.DS_Store
+

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "torch/lib/gloo"]
+	path = torch/lib/gloo
+	url = https://github.com/facebookincubator/gloo
+[submodule "torch/lib/pybind11"]
+	path = torch/lib/pybind11
+	url = https://github.com/pybind/pybind11
+[submodule "torch/lib/nanopb"]
+	path = torch/lib/nanopb
+	url = https://github.com/nanopb/nanopb.git

.travis.yml

@@ -1,29 +1,40 @@
 # https://travis-ci.org/pytorch/pytorch
 language: python
+dist: trusty
+git:
+  submodules: false
 python:
-    - 2.7.8
+    - 2.7.9
     - 2.7
-    - 3.3
-    - 3.4
     - 3.5
+    - 3.6
     - nightly
 
-install:
-    - export CC="gcc-4.8"
-    - export CXX="g++-4.8"
-    - travis_retry pip install -r requirements.txt
-    - travis_retry pip install .
+cache:
+    - ccache
+    - directories:
+        - $HOME/.ccache
 
-script:
-    - ./test/run_test.sh
+install:
+    - unset CCACHE_DISABLE
+    - export CCACHE_DIR=$HOME/.ccache
+    - export CC="ccache gcc-5"
+    - export CXX="ccache g++-5"
+    - ccache --show-stats
+    - travis_retry pip install --upgrade pip setuptools wheel
+    - travis_retry pip install -r requirements.txt --only-binary=scipy
+    - git submodule update --init --recursive
+    - MAX_JOBS=8 python setup.py install
 
 addons:
     apt:
         sources:
             - ubuntu-toolchain-r-test
         packages:
-            - gcc-4.8
-            - g++-4.8
+            - g++-5
+
+script:
+    - OMP_NUM_THREADS=2 ./test/run_test.sh
 
 # This reportedly works around an issue downloading packages from pypi on
 # travis.  Consider removing this after the underlying issue is fixed.
@@ -32,3 +43,9 @@ sudo: false
 
 matrix:
     fast_finish: true
+    include:
+        env: LINT_CHECK
+        python: "2.7"
+        addons: true
+        install: pip install flake8
+        script: flake8

CONTRIBUTING.md

@@ -0,0 +1,195 @@
+## Contributing to PyTorch
+
+If you are interested in contributing to PyTorch, your contributions will fall
+into two categories:
+1. You want to propose a new Feature and implement it
+    - post about your intended feature, and we shall discuss the design and
+    implementation. Once we agree that the plan looks good, go ahead and implement it.
+2. You want to implement a feature or bug-fix for an outstanding issue
+    - Look at the outstanding issues here: https://github.com/pytorch/pytorch/issues
+    - Especially look at the Low Priority and Medium Priority issues
+    - Pick an issue and comment on the task that you want to work on this feature
+    - If you need more context on a particular issue, please ask and we shall provide.
+
+Once you finish implementing a feature or bugfix, please send a Pull Request to
+https://github.com/pytorch/pytorch
+
+If you are not familiar with creating a Pull Request, here are some guides:
+- http://stackoverflow.com/questions/14680711/how-to-do-a-github-pull-request
+- https://help.github.com/articles/creating-a-pull-request/
+
+
+## Developing locally with PyTorch
+
+To locally develop with PyTorch, here are some tips:
+
+1. Uninstall all existing pytorch installs
+```
+conda uninstall pytorch
+pip uninstall torch
+pip uninstall torch # run this command twice
+```
+
+2. Locally clone a copy of PyTorch from source:
+
+```
+git clone https://github.com/pytorch/pytorch
+cd pytorch
+```
+
+3. Install PyTorch in `build develop` mode:
+
+A full set of instructions on installing PyTorch from Source are here:
+https://github.com/pytorch/pytorch#from-source
+
+The change you have to make is to replace
+
+```
+python setup.py install
+```
+
+with
+
+```
+python setup.py build develop
+```
+
+This is especially useful if you are only changing Python files.
+
+This mode will symlink the python files from the current local source tree into the
+python install.
+
+Hence, if you modify a python file, you do not need to reinstall pytorch again and again.
+
+For example:
+- Install local pytorch in `build develop` mode
+- modify your python file `torch/__init__.py` (for example)
+- test functionality
+- modify your python file `torch/__init__.py`
+- test functionality
+- modify your python file `torch/__init__.py`
+- test functionality
+
+You do not need to repeatedly install after modifying python files.
+
+
+## Writing documentation
+
+PyTorch uses [Google style](http://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html)
+for formatting docstrings. Length of line inside docstrings block must be limited to 80 characters to
+fit into Jupyter documentation popups.
+
+
+## Managing multiple build trees
+
+One downside to using `python setup.py develop` is that your development
+version of pytorch will be installed globally on your account (e.g., if
+you run `import torch` anywhere else, the development version will be
+used.
+
+If you want to manage multiple builds of PyTorch, you can make use of
+[conda environments](https://conda.io/docs/using/envs.html) to maintain
+separate Python package environments, each of which can be tied to a
+specific build of PyTorch.  To set one up:
+
+```
+conda create -n pytorch-myfeature
+source activate pytorch-myfeature
+# if you run python now, torch will NOT be installed
+python setup.py build develop
+```
+
+## C++ Development tips
+
+If you are working on the C++ code, there are a few important things that you
+will want to keep in mind:
+
+1. How to rebuild only the code you are working on, and
+2. How to make rebuilds in the absence of changes go faster.
+
+### Build only what you need.
+
+`python setup.py build` will build everything, but since our build system is
+not very optimized for incremental rebuilds, this will actually be very slow.
+Far better is to only request rebuilds of the parts of the project you are
+working on:
+
+- Working on `torch/csrc`?  Run `python setup.py develop` to rebuild
+  (NB: no `build` here!)
+
+- Working on `torch/lib/TH`, did not make any cmake changes, and just want to
+  see if it compiles?  Run `(cd torch/lib/build/TH && make install -j$(getconf _NPROCESSORS_ONLN))`.  This
+  applies for any other subdirectory of `torch/lib`.  **Warning: Changes you
+  make here will not be visible from Python.**  See below.
+
+- Working on `torch/lib` and want to run your changes / rerun cmake?  Run
+  `python setup.py build_deps`.  Note that this will rerun cmake for
+  every subdirectory in TH; if you are only working on one project,
+  consider editing `torch/lib/build_all.sh` and commenting out the
+  `build` lines of libraries you are not working on.
+
+On the initial build, you can also speed things up with the environment
+variables `DEBUG` and `NO_CUDA`.
+
+- `DEBUG=1` will enable debug builds (-g -O0)
+- `NO_CUDA=1` will disable compiling CUDA (in case you are developing on something not CUDA related), to save compile time.
+
+For example:
+```
+NO_CUDA=1 DEBUG=1 python setup.py build develop
+```
+
+Make sure you continue to pass these flags on subsequent builds.
+
+### Make no-op build fast.
+
+Python `setuptools` is pretty dumb, and always rebuilds every C file in a
+project. Using ccache in a situation like this is a real time-saver. However, by
+default, ccache does not properly support CUDA stuff, so here are the
+instructions for installing a custom `ccache` fork that has CUDA support:
+
+```
+# install and export ccache
+if ! ls ~/ccache/bin/ccache
+then
+    sudo apt-get update
+    sudo apt-get install -y automake autoconf
+    sudo apt-get install -y asciidoc
+    mkdir -p ~/ccache
+    pushd /tmp
+    rm -rf ccache
+    git clone https://github.com/colesbury/ccache -b ccbin
+    pushd ccache
+    ./autogen.sh
+    ./configure
+    make install prefix=~/ccache
+    popd
+    popd
+
+    mkdir -p ~/ccache/lib
+    mkdir -p ~/ccache/cuda
+    ln -s ~/ccache/bin/ccache ~/ccache/lib/cc
+    ln -s ~/ccache/bin/ccache ~/ccache/lib/c++
+    ln -s ~/ccache/bin/ccache ~/ccache/lib/gcc
+    ln -s ~/ccache/bin/ccache ~/ccache/lib/g++
+    ln -s ~/ccache/bin/ccache ~/ccache/cuda/nvcc
+
+    ~/ccache/bin/ccache -M 25Gi
+fi
+
+export PATH=~/ccache/lib:$PATH
+export CUDA_NVCC_EXECUTABLE=~/ccache/cuda/nvcc
+```
+
+## CUDA Development tips
+
+If you are working on the CUDA code, here are some useful CUDA debugging tips:
+
+1. `CUDA_DEBUG=1` will enable CUDA debugging symbols (-g -G). This is particularly
+    helpful in debugging device code. However, it will slow down the build process,
+    so use wisely.
+2. `cuda-gdb` and `cuda-memcheck` are your best CUDA debuging friends. Unlike`gdb`,
+   `cuda-gdb` can display actual values in a CUDA tensor (rather than all zeros).
+
+
+Hope this helps, and thanks for considering to contribute.

Dockerfile

@@ -0,0 +1,41 @@
+FROM nvidia/cuda:8.0-cudnn6-devel-ubuntu16.04 
+
+RUN echo "deb http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+         build-essential \
+         cmake \
+         git \
+         curl \
+         vim \
+         ca-certificates \
+         libnccl2=2.0.5-2+cuda8.0 \
+         libnccl-dev=2.0.5-2+cuda8.0 \
+         libjpeg-dev \
+         libpng-dev &&\
+     rm -rf /var/lib/apt/lists/*
+
+
+ENV PYTHON_VERSION=3.6
+RUN curl -o ~/miniconda.sh -O  https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh  && \
+     chmod +x ~/miniconda.sh && \
+     ~/miniconda.sh -b -p /opt/conda && \     
+     rm ~/miniconda.sh && \
+#     /opt/conda/bin/conda install conda-build && \
+     /opt/conda/bin/conda create -y --name pytorch-py$PYTHON_VERSION python=$PYTHON_VERSION numpy pyyaml scipy ipython mkl&& \
+     /opt/conda/bin/conda clean -ya 
+ENV PATH /opt/conda/envs/pytorch-py$PYTHON_VERSION/bin:$PATH
+RUN conda install --name pytorch-py$PYTHON_VERSION -c soumith magma-cuda80
+# This must be done before pip so that requirements.txt is available
+WORKDIR /opt/pytorch
+COPY . .
+
+RUN git submodule update --init
+RUN TORCH_CUDA_ARCH_LIST="3.5 5.2 6.0 6.1+PTX" TORCH_NVCC_FLAGS="-Xfatbin -compress-all" \
+    CMAKE_PREFIX_PATH="$(dirname $(which conda))/../" \
+    pip install -v .
+
+RUN git clone https://github.com/pytorch/vision.git && cd vision && pip install -v .
+
+WORKDIR /workspace
+RUN chmod -R a+w /workspace

README.md

@@ -1,257 +1,249 @@
-# pytorch [alpha-4]
+<p align="center"><img width="40%" src="docs/source/_static/img/pytorch-logo-dark.png" /></p>
 
-- [What is PyTorch?](#what-is-pytorch)
-- [Reasons to consider PyTorch](#reasons-to-consider-pytorch)
+--------------------------------------------------------------------------------
+
+PyTorch is a Python package that provides two high-level features:
+- Tensor computation (like NumPy) with strong GPU acceleration
+- Deep neural networks built on a tape-based autograd system
+
+You can reuse your favorite Python packages such as NumPy, SciPy and Cython to extend PyTorch when needed.
+
+We are in an early-release beta. Expect some adventures and rough edges.
+
+- [More about PyTorch](#more-about-pytorch)
 - [Installation](#installation)
   - [Binaries](#binaries)
-  - [From source](#from-source)
+  - [From Source](#from-source)
+  - [Docker Image](#docker-image)
 - [Getting Started](#getting-started)
 - [Communication](#communication)
-- [Timeline](#timeline)
-- [pytorch vs torch: important changes](#pytorch-vs-torch-important-changes)
+- [Releases and Contributing](#releases-and-contributing)
+- [The Team](#the-team)
 
-| Python |  **`Linux CPU`**   |  **`Linux GPU`** |
-|--------|--------------------|------------------|
-| 2.7.8  | [![Build Status](https://travis-ci.com/apaszke/pytorch.svg?token=shqHbUq29zKDxuqzGcjC&branch=master)](https://travis-ci.com/apaszke/pytorch) | |
-| 2.7    | [![Build Status](https://travis-ci.com/apaszke/pytorch.svg?token=shqHbUq29zKDxuqzGcjC&branch=master)](https://travis-ci.com/apaszke/pytorch) | [![Build Status](http://build.pytorch.org:8080/buildStatus/icon?job=pytorch-master-py2)](https://build.pytorch.org/job/pytorch-master-py2)  |
-| 3.3    | [![Build Status](https://travis-ci.com/apaszke/pytorch.svg?token=shqHbUq29zKDxuqzGcjC&branch=master)](https://travis-ci.com/apaszke/pytorch) | |
-| 3.4    | [![Build Status](https://travis-ci.com/apaszke/pytorch.svg?token=shqHbUq29zKDxuqzGcjC&branch=master)](https://travis-ci.com/apaszke/pytorch) | |
-| 3.5    | [![Build Status](https://travis-ci.com/apaszke/pytorch.svg?token=shqHbUq29zKDxuqzGcjC&branch=master)](https://travis-ci.com/apaszke/pytorch) | [![Build Status](http://build.pytorch.org:8080/buildStatus/icon?job=pytorch-master-py3)](https://build.pytorch.org/job/pytorch-master-py3)  |
-| Nightly| [![Build Status](https://travis-ci.com/apaszke/pytorch.svg?token=shqHbUq29zKDxuqzGcjC&branch=master)](https://travis-ci.com/apaszke/pytorch) | |
+| System | 2.7 | 3.5 |
+| --- | --- | --- |
+| Linux CPU | [![Build Status](https://travis-ci.org/pytorch/pytorch.svg?branch=master)](https://travis-ci.org/pytorch/pytorch) | [![Build Status](https://travis-ci.org/pytorch/pytorch.svg?branch=master)](https://travis-ci.org/pytorch/pytorch) |
+| Linux GPU | [![Build Status](http://build.pytorch.org:8080/buildStatus/icon?job=pytorch-master-py2-linux)](https://build.pytorch.org/job/pytorch-master-py2-linux) | [![Build Status](http://build.pytorch.org:8080/buildStatus/icon?job=pytorch-master-py3-linux)](https://build.pytorch.org/job/pytorch-master-py3-linux) |
+| macOS CPU | [![Build Status](http://build.pytorch.org:8080/buildStatus/icon?job=pytorch-master-py2-osx-cpu)](https://build.pytorch.org/job/pytorch-master-py2-osx-cpu) | [![Build Status](http://build.pytorch.org:8080/buildStatus/icon?job=pytorch-master-py3-osx-cpu)](https://build.pytorch.org/job/pytorch-master-py3-osx-cpu) |
 
-The project is still under active development and is likely to drastically change in short periods of time.
-We will be announcing API changes and important developments via a newsletter, github issues and post a link to the issues on slack.
-Please remember that at this stage, this is an invite-only closed alpha, and please don't distribute code further.
-This is done so that we can control development tightly and rapidly during the initial phases with feedback from you.
 
-## What is PyTorch?
+## More about PyTorch
 
-PyTorch is a library that consists of the following components:
+At a granular level, PyTorch is a library that consists of the following components:
 
-| \_                       | \_ |
-| ------------------------ | --- |
-| torch                    | a Tensor library like NumPy, with strong GPU support |
-| torch.autograd           | a tape based automatic differentiation library that supports all differentiable Tensor operations in torch |
-| torch.nn                 | a neural networks library deeply integrated with autograd designed for maximum flexibility |
-| torch.optim              | an optimization package to be used with torch.nn with standard optimization methods such as SGD, RMSProp, LBFGS, Adam etc. |
-| torch.multiprocessing    | python multiprocessing, but with magical memory sharing of torch Tensors across processes. Useful for data loading and hogwild training. |
-| torch.utils              | DataLoader, Trainer and other utility functions for convenience |
-| torch.legacy(.nn/.optim) | legacy code that has been ported over from torch for backward compatibility reasons |
+<table>
+<tr>
+    <td><b> torch </b></td>
+    <td> a Tensor library like NumPy, with strong GPU support </td>
+</tr>
+<tr>
+    <td><b> torch.autograd </b></td>
+    <td> a tape-based automatic differentiation library that supports all differentiable Tensor operations in torch </td>
+</tr>
+<tr>
+    <td><b> torch.nn </b></td>
+    <td> a neural networks library deeply integrated with autograd designed for maximum flexibility </td>
+</tr>
+<tr>
+    <td><b> torch.multiprocessing  </b></td>
+    <td> Python multiprocessing, but with magical memory sharing of torch Tensors across processes. Useful for data loading and Hogwild training. </td>
+</tr>
+<tr>
+    <td><b> torch.utils </b></td>
+    <td> DataLoader, Trainer and other utility functions for convenience </td>
+</tr>
+<tr>
+    <td><b> torch.legacy(.nn/.optim) </b></td>
+    <td> legacy code that has been ported over from torch for backward compatibility reasons </td>
+</tr>
+</table>
 
 Usually one uses PyTorch either as:
 
-- A replacement for numpy to use the power of GPUs.
+- a replacement for NumPy to use the power of GPUs.
 - a deep learning research platform that provides maximum flexibility and speed
 
-## Reasons to consider PyTorch
+Elaborating further:
 
-### Python first
+### A GPU-Ready Tensor Library
 
-PyTorch is not a Python binding into a monolothic C++ framework.  
+If you use NumPy, then you have used Tensors (a.k.a ndarray).
+
+<p align=center><img width="30%" src="docs/source/_static/img/tensor_illustration.png" /></p>
+
+PyTorch provides Tensors that can live either on the CPU or the GPU, and accelerate
+compute by a huge amount.
+
+We provide a wide variety of tensor routines to accelerate and fit your scientific computation needs
+such as slicing, indexing, math operations, linear algebra, reductions.
+And they are fast!
+
+### Dynamic Neural Networks: Tape-Based Autograd
+
+PyTorch has a unique way of building neural networks: using and replaying a tape recorder.
+
+Most frameworks such as TensorFlow, Theano, Caffe and CNTK have a static view of the world.
+One has to build a neural network, and reuse the same structure again and again.
+Changing the way the network behaves means that one has to start from scratch.
+
+With PyTorch, we use a technique called reverse-mode auto-differentiation, which allows you to
+change the way your network behaves arbitrarily with zero lag or overhead. Our inspiration comes
+from several research papers on this topic, as well as current and past work such as
+[torch-autograd](https://github.com/twitter/torch-autograd),
+[autograd](https://github.com/HIPS/autograd),
+[Chainer](http://chainer.org), etc.
+
+While this technique is not unique to PyTorch, it's one of the fastest implementations of it to date.
+You get the best of speed and flexibility for your crazy research.
+
+<p align=center><img width="80%" src="docs/source/_static/img/dynamic_graph.gif" /></p>
+
+### Python First
+
+PyTorch is not a Python binding into a monolithic C++ framework.
 It is built to be deeply integrated into Python.
-You can use it naturally like you would use numpy / scipy / scikit-learn etc.  
-You can write your new neural network layers in Python itself, using your favorite libraries.
+You can use it naturally like you would use NumPy / SciPy / scikit-learn etc.
+You can write your new neural network layers in Python itself, using your favorite libraries
+and use packages such as Cython and Numba.
+Our goal is to not reinvent the wheel where appropriate.
 
-### Imperativeness first. What you see is what you get!
+### Imperative Experiences
 
-PyTorch is designed to be intuitive and easy to use.  
-When you are debugging your program, or receive error messages / stack traces, you are always guaranteed to get
-error messages that are easy to understand and a stack-trace that points to exactly where your code was defined.
-Never spend hours debugging your code because of bad stack traces or asynchronous and opaque execution engines.
+PyTorch is designed to be intuitive, linear in thought and easy to use.
+When you execute a line of code, it gets executed. There isn't an asynchronous view of the world.
+When you drop into a debugger, or receive error messages and stack traces, understanding them is straightforward.
+The stack trace points to exactly where your code was defined.
+We hope you never spend hours debugging your code because of bad stack traces or asynchronous and opaque execution engines.
 
-### Performance and Memory usage
+### Fast and Lean
 
-PyTorch is as fast as the fastest deep learning framework out there. We integrate acceleration frameworks such as Intel MKL and NVIDIA CuDNN for maximum speed.
+PyTorch has minimal framework overhead. We integrate acceleration libraries
+such as Intel MKL and NVIDIA (cuDNN, NCCL) to maximize speed.
+At the core, its CPU and GPU Tensor and neural network backends
+(TH, THC, THNN, THCUNN) are written as independent libraries with a C99 API.
+They are mature and have been tested for years.
 
-The memory usage in PyTorch is extremely efficient, and we've written custom memory allocators for the GPU to make sure that your
-deep learning models are maximally memory efficient. This enables you to train bigger deep learning models than before.
+Hence, PyTorch is quite fast – whether you run small or large neural networks.
 
-### Multi-GPU ready
+The memory usage in PyTorch is extremely efficient compared to Torch or some of the alternatives.
+We've written custom memory allocators for the GPU to make sure that
+your deep learning models are maximally memory efficient.
+This enables you to train bigger deep learning models than before.
 
-PyTorch is fully powered to efficiently use Multiple GPUs for accelerated deep learning.  
-We integrate efficient multi-gpu collectives such as NVIDIA NCCL to make sure that you get the maximal Multi-GPU performance.
+### Extensions without Pain
 
-### Simple Extension API to interface with C
+Writing new neural network modules, or interfacing with PyTorch's Tensor API was designed to be straightforward
+and with minimal abstractions.
+
+You can write new neural network layers in Python using the torch API
+[or your favorite NumPy-based libraries such as SciPy](http://pytorch.org/tutorials/advanced/numpy_extensions_tutorial.html).
+
+If you want to write your layers in C/C++, we provide an extension API based on
+[cffi](http://cffi.readthedocs.io/en/latest/) that is efficient and with minimal boilerplate.
+There is no wrapper code that needs to be written. You can see [a tutorial here](http://pytorch.org/tutorials/advanced/c_extension.html) and [an example here](https://github.com/pytorch/extension-ffi).
 
-Writing new neural network modules, or interfacing with PyTorch's Tensor API is a breeze, thanks to an easy to use
-extension API that is efficient and easy to use.
 
 ## Installation
 
 ### Binaries
-- Anaconda
-```bash
-conda install pytorch -c https://conda.anaconda.org/t/6N-MsQ4WZ7jo/soumith
-```
+Commands to install from binaries via Conda or pip wheels are on our website:
 
-### From source
+[http://pytorch.org](http://pytorch.org)
+
+### From Source
+
+If you are installing from source, we highly recommend installing an [Anaconda](https://www.continuum.io/downloads) environment.
+You will get a high-quality BLAS library (MKL) and you get a controlled compiler version regardless of your Linux distro.
+
+Once you have [Anaconda](https://www.continuum.io/downloads) installed, here are the instructions.
+
+If you want to compile with CUDA support, install
+- [NVIDIA CUDA](https://developer.nvidia.com/cuda-downloads) 7.5 or above
+- [NVIDIA cuDNN](https://developer.nvidia.com/cudnn) v6.x or above
+
+If you want to disable CUDA support, export environment variable `NO_CUDA=1`.
 
 #### Install optional dependencies
 
+On Linux
+```bash
+export CMAKE_PREFIX_PATH="$(dirname $(which conda))/../" # [anaconda root directory]
+
+# Install basic dependencies
+conda install numpy pyyaml mkl setuptools cmake cffi
+
+# Add LAPACK support for the GPU
+conda install -c soumith magma-cuda80 # or magma-cuda75 if CUDA 7.5
+```
+
+On OSX
 ```bash
 export CMAKE_PREFIX_PATH=[anaconda root directory]
-conda install numpy mkl
-conda install -c soumith magma-cuda75# or magma-cuda80
+conda install numpy pyyaml setuptools cmake cffi
+```
+#### Get the PyTorch source
+```bash
+git clone --recursive https://github.com/pytorch/pytorch
 ```
 
 #### Install PyTorch
+On Linux
 ```bash
-export MACOSX_DEPLOYMENT_TARGET=10.9 # for OSX
-pip install -r requirements.txt
-pip install .
+python setup.py install
 ```
 
+On OSX
+```bash
+MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ python setup.py install
+```
+
+### Docker image
+
+Dockerfile is supplied to build images with cuda support and cudnn v6. Build as usual
+```
+docker build -t pytorch .
+```
+Dockerfile to build with cuda 9 and cudnn v7 (with Volta support) is in tools/docker, the build command is
+```
+docker build -t pytorch_cuda9 -f tools/docker/Dockerfile9 .
+```
+Alternatively, if you want to use a runtime image, you can use the pre-built one from Docker Hub and run with nvidia-docker:
+```
+nvidia-docker run --rm -ti --ipc=host pytorch/pytorch:latest
+```
+Please note that PyTorch uses shared memory to share data between processes, so if torch multiprocessing is used (e.g.
+for multithreaded data loaders) the default shared memory segment size that container runs with is not enough, and you
+should increase shared memory size either with `--ipc=host` or `--shm-size` command line options to `nvidia-docker run`.
+
+
 ## Getting Started
+
 Three pointers to get you started:
-- [Tutorials: notebooks to get you started with understanding and using PyTorch](https://github.com/pytorch/tutorials)
+- [Tutorials: get you started with understanding and using PyTorch](http://pytorch.org/tutorials/)
 - [Examples: easy to understand pytorch code across all domains](https://github.com/pytorch/examples)
-- The API Reference: [http://pytorch.org/api/](http://pytorch.org/api/)
+- [The API Reference](http://pytorch.org/docs/)
 
 ## Communication
-* github issues: bug reports, feature requests, install issues, RFCs, thoughts, etc.
-* slack: general chat, online discussions, collaboration etc. https://pytorch.slack.com/ . If you need a slack invite, ping me at soumith@pytorch.org
+* forums: discuss implementations, research, etc. http://discuss.pytorch.org
+* GitHub issues: bug reports, feature requests, install issues, RFCs, thoughts, etc.
+* Slack: general chat, online discussions, collaboration etc. https://pytorch.slack.com/ . Our slack channel is invite-only to promote a healthy balance between power-users and beginners. If you need a slack invite, ping us at soumith@pytorch.org
 * newsletter: no-noise, one-way email newsletter with important announcements about pytorch. You can sign-up here: http://eepurl.com/cbG0rv
 
-## Timeline
+## Releases and Contributing
 
-We will run the alpha releases weekly for 6 weeks.
-After that, we will reevaluate progress, and if we are ready, we will hit beta-0. If not, we will do another two weeks of alpha.
+PyTorch has a 90 day release cycle (major releases).
+It's current state is Beta, we expect no obvious bugs. Please let us know if you encounter a bug by [filing an issue](https://github.com/pytorch/pytorch/issues).
 
-* ~~alpha-0: Working versions of torch, cutorch, nn, cunn, optim fully unit tested with seamless numpy conversions~~
-* ~~alpha-1: Serialization to/from disk with sharing intact. initial release of the new neuralnets package based on a Chainer-like design~~
-* ~~alpha-2: sharing tensors across processes for hogwild training or data-loading processes. a rewritten optim package for this new nn.~~
-* ~~alpha-3: binary installs, contbuilds, etc.~~
-* ~~alpha-4: multi-GPU support, cudnn integration, imagenet / resnet example~~
-* alpha-5: a ton of examples across vision, nlp, speech, RL -- this phase might make us rethink parts of the APIs, and hence want to do this in alpha than beta
-* alpha-6: Putting a simple and efficient story around multi-machine training. Probably simplistic like torch-distlearn. Building the website, release scripts, more documentation, etc.
-* beta-0: First public release
+We appreciate all contributions. If you are planning to contribute back bug-fixes, please do so without any further discussion.
 
-The beta phases will be leaning more towards working with all of you, convering your use-cases, active development on non-core aspects.
+If you plan to contribute new features, utility functions or extensions to the core, please first open an issue and discuss the feature with us.
+Sending a PR without discussion might end up resulting in a rejected PR, because we might be taking the core in a different direction than you might be aware of.
 
-## pytorch vs torch: important changes
+## The Team
 
-We've decided that it's time to rewrite/update parts of the old torch API, even if it means losing some of backward compatibility.
+PyTorch is a community driven project with several skillful engineers and researchers contributing to it.
 
-**[This tutorial](https://github.com/pytorch/tutorials/blob/master/Introduction%20to%20PyTorch%20for%20former%20Torchies.ipynb) takes you through the biggest changes**
-and walks you through PyTorch
-
-For brevity,
-
-#### Tensors:
-- clear separation of in-place and out-of-place operations
-- zero-indexing
-- no camel casing for Tensor functions
-- an efficient Numpy bridge (with zero memory copy)
-- CUDA tensors have clear and intuitive semantics
-
-#### New neural network module (Combines nn, nngraph, autograd): 
-
-1. Design inspired from Chainer
-2. Modules no longer hold state. State is held in the graph
-    1. Access state via hooks
-	2. Execution engine
-	    1. imperative execution engine (default)
-		2. lazy execution engine
-		   1. allows graph optimizations and automatic in-place / fusing operations
-	4. Model structure is defined by its code
-	    1. You can use loops and arbitrarily complicated conditional statements
-
-**To reiterate, we recommend that you go through [This tutorial](https://github.com/pytorch/tutorials/blob/master/Introduction%20to%20PyTorch%20for%20former%20Torchies.ipynb)**
-
-### Serialization
-
-Pickling tensors is supported, but requires making a temporary copy of all data in memory and breaks sharing.
-
-For this reason we're providing `torch.load` and `torch.save`, that are free of these problems.
-
-They have the same interfaces as `pickle.load` (file object) and `pickle.dump` (serialized object, file object) respectively.
-
-For now the only requirement is that the file should have a `fileno` method, which returns a file descriptor number (this is already implemented by objects returned by `open`).
-
-Objects are serialized in a tar archive consisting of four files:
-- `sys_info` - protocol version, byte order, long size, etc.
-- `pickle` - pickled object
-- `tensors` - tensor metadata
-- `storages` - serialized data
-
-### Multiprocessing with Tensor sharing
-
-We made PyTorch to seamlessly integrate with python multiprocessing.
-What we've added specially in torch.multiprocessing is the seamless ability to efficiently share and send
-tensors over from one process to another. ([technical details of implementation](http://github.com/pytorch/pytorch/wiki/Multiprocessing-Technical-Notes))
-This is very useful for example in:
-- Writing parallelized data loaders
-- Training models "hogwild", where several models are trained in parallel, sharing the same set of parameters.
-
-Here are a couple of examples for torch.multiprocessing
-
-```python
-# loaders.py
-# Functions from this file run in the workers
-
-def fill(queue):
-  while True:
-      tensor = queue.get()
-	  tensor.fill_(10)
-	  queue.put(tensor)
-
-def fill_pool(tensor):
-  tensor.fill_(10)
-```
-
-```python
-# Example 1: Using multiple persistent processes and a Queue
-# process.py
-
-import torch
-import torch.multiprocessing as multiprocessing
-from loaders import fill
-
-# torch.multiprocessing.Queue automatically moves Tensor data to shared memory
-# So the main process and worker share the data
-queue = multiprocessing.Queue()
-buffers = [torch.Tensor(2, 2) for i in range(4)]
-for b in buffers:
-  queue.put(b)
-  processes = [multiprocessing.Process(target=fill, args=(queue,)).start() for i in range(10)]
-```
-
-```python
-# Example 2: Using a process pool
-# pool.py
-
-import torch
-from torch.multiprocessing import Pool
-from loaders import fill_pool
-
-tensors = [torch.Tensor(2, 2) for i in range(100)]
-pool = Pool(10)
-pool.map(fill_pool, tensors)
-```
-
-#### Some notes on new nn implementation
-
-As shown above, structure of the networks is fully defined by control-flow embedded in the code. There are no rigid containers known from Lua. You can put an `if` in the middle of your model and freely branch depending on any condition you can come up with. All operations are registered in the computational graph history.
-
-There are two main objects that make this possible - variables and functions. They will be denoted as squares and circles respectively.
-
-![Variable and function symbols](http://students.mimuw.edu.pl/~ap360585/__torch_img/variable_function.png)
-
-Variables are the objects that hold a reference to a tensor (and optionally to gradient w.r.t. that tensor), and to the function in the computational graph that created it. Variables created explicitly by the user (`Variable(tensor)`) have a Leaf function node associated with them.
-
-![Variable and leaf function](http://students.mimuw.edu.pl/~ap360585/__torch_img/variable_leaf.png)
-
-Functions are simple classes that define a function from a tuple of inputs to a tuple of outputs, and a formula for computing gradient w.r.t. it's inputs. Function objects are instantiated to hold references to other functions, and these references allow to reconstruct the history of a computation. An example graph for a linear layer (`Wx + b`) is shown below.
-
-![Linear layer](http://students.mimuw.edu.pl/~ap360585/__torch_img/linear.png)
-
-Please note that function objects never hold references to Variable objects, except for when they're necessary in the backward pass. This allows to free all the unnecessary intermediate values. A good example for this is addition when computing e.g. (`y = Wx + My`):
-
-![Freeing intermediate values](http://students.mimuw.edu.pl/~ap360585/__torch_img/intermediate_free.png)
-
-Matrix multiplication operation keeps references to it's inputs because it will need them, but addition doesn't need `Wx` and `My` after it computes the result, so as soon as they go out of scope they are freed. To access intermediate values in the forward pass you can either copy them when you still have a reference, or you can use a system of hooks that can be attached to any function. Hooks also allow to access and inspect gradients inside the graph.
-
-Another nice thing about this is that a single layer doesn't hold any state other than it's parameters (all intermediate values are alive as long as the graph references them), so it can be used multiple times before calling backward. This is especially convenient when training RNNs. You can use the same network for all timesteps and the gradients will sum up automatically.
-
-To compute backward pass you can call `.backward()` on a variable if it's a scalar (a 1-element Variable), or you can provide a gradient tensor of matching shape if it's not. This creates an execution engine object that manages the whole backward pass. It's been introduced, so that the code for analyzing the graph and scheduling node processing order is decoupled from other parts, and can be easily replaced. Right now it's simply processing the nodes in topological order, without any prioritization, but in the future we can implement algorithms and heuristics for scheduling independent nodes on different GPU streams, deciding which branches to compute first, etc.
+PyTorch is currently maintained by [Adam Paszke](https://apaszke.github.io/), [Sam Gross](https://github.com/colesbury), [Soumith Chintala](http://soumith.ch) and [Gregory Chanan](https://github.com/gchanan) with major contributions coming from 10s of talented individuals in various forms and means.
+A non-exhaustive but growing list needs to mention: Trevor Killeen, Sasank Chilamkurthy, Sergey Zagoruyko, Adam Lerer, Francisco Massa, Alykhan Tejani, Luca Antiga, Alban Desmaison, Andreas Kopf, James Bradbury, Zeming Lin, Yuandong Tian, Guillaume Lample, Marat Dukhan, Natalia Gimelshein, Christian Sarofeen, Martin Raison, Edward Yang, Zachary Devito.
 
+Note: this project is unrelated to [hughperkins/pytorch](https://github.com/hughperkins/pytorch) with the same name. Hugh is a valuable contributor in the Torch community and has helped with many things Torch and PyTorch.

cmake/FindCUDA/FindCUDA/select_compute_arch.cmake

@@ -63,11 +63,16 @@ function(CUDA_DETECT_INSTALLED_GPUS OUT_VARIABLE)
       "}\n")
 
     execute_process(COMMAND "${CUDA_NVCC_EXECUTABLE}" "--run" "${cufile}"
+                    "-ccbin" ${CMAKE_CXX_COMPILER}
                     WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/CMakeFiles/"
                     RESULT_VARIABLE nvcc_res OUTPUT_VARIABLE nvcc_out
                     ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
 
     if(nvcc_res EQUAL 0)
+      # only keep the last line of nvcc_out
+      STRING(REGEX REPLACE ";" "\\\\;" nvcc_out "${nvcc_out}")
+      STRING(REGEX REPLACE "\n" ";" nvcc_out "${nvcc_out}")
+      list(GET nvcc_out -1 nvcc_out)
       string(REPLACE "2.1" "2.1(2.0)" nvcc_out "${nvcc_out}")
       set(CUDA_GPU_DETECT_OUTPUT ${nvcc_out} CACHE INTERNAL "Returned GPU architetures from detect_gpus tool" FORCE)
     endif()
@@ -116,13 +121,13 @@ function(CUDA_SELECT_NVCC_ARCH_FLAGS out_variable)
       set(add_ptx TRUE)
       set(arch_name ${CMAKE_MATCH_1})
     endif()
-    if(arch_name MATCHES "([0-9]\\.[0-9])$")
+    if(arch_name MATCHES "(^[0-9]\\.[0-9](\\([0-9]\\.[0-9]\\))?)$")
       set(arch_bin ${CMAKE_MATCH_1})
       set(arch_ptx ${arch_bin})
     else()
       # Look for it in our list of known architectures
       if(${arch_name} STREQUAL "Fermi")
-        set(arch_bin 2.0 "2.1(2.0)")
+        set(arch_bin "2.0 2.1(2.0)")
       elseif(${arch_name} STREQUAL "Kepler+Tegra")
         set(arch_bin 3.2)
       elseif(${arch_name} STREQUAL "Kepler+Tesla")
@@ -173,11 +178,11 @@ function(CUDA_SELECT_NVCC_ARCH_FLAGS out_variable)
   # Tell NVCC to add binaries for the specified GPUs
   foreach(arch ${cuda_arch_bin})
     if(arch MATCHES "([0-9]+)\\(([0-9]+)\\)")
-      # User explicitly specified PTX for the concrete BIN
+      # User explicitly specified ARCH for the concrete CODE
       list(APPEND nvcc_flags -gencode arch=compute_${CMAKE_MATCH_2},code=sm_${CMAKE_MATCH_1})
       list(APPEND nvcc_archs_readable sm_${CMAKE_MATCH_1})
     else()
-      # User didn't explicitly specify PTX for the concrete BIN, we assume PTX=BIN
+      # User didn't explicitly specify ARCH for the concrete CODE, we assume ARCH=CODE
       list(APPEND nvcc_flags -gencode arch=compute_${arch},code=sm_${arch})
       list(APPEND nvcc_archs_readable sm_${arch})
     endif()

docs/Makefile

@@ -0,0 +1,27 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = PyTorch
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+docset: html
+	doc2dash --name $(SPHINXPROJ) --icon $(SOURCEDIR)/_static/img/pytorch-logo-flame.png --enable-js --online-redirect-url http://pytorch.org/docs/ --force $(BUILDDIR)/html/
+
+	# Manually fix because Zeal doesn't deal well with `icon.png`-only at 2x resolution.
+	cp $(SPHINXPROJ).docset/icon.png $(SPHINXPROJ).docset/icon@2x.png
+	convert $(SPHINXPROJ).docset/icon@2x.png -resize 16x16 $(SPHINXPROJ).docset/icon.png
+
+.PHONY: help Makefile docset
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/docutils/doc2md.py

@@ -1,534 +0,0 @@
-#! /usr/bin/env python
-# encoding: utf-8
-"""
-Very lightweight docstring to Markdown converter. Modified for use in pytorch
-
-
-### License
-
-Copyright © 2013 Thomas Gläßle <t_glaessle@gmx.de>
-
-This work  is free. You can  redistribute it and/or modify  it under the
-terms of the Do What The Fuck  You Want To Public License, Version 2, as
-published by Sam Hocevar. See the COPYING file for more details.
-
-This program  is free software.  It comes  without any warranty,  to the
-extent permitted by applicable law.
-
-
-### Description
-
-Little convenience tool to extract docstrings from a module or class and
-convert them to GitHub Flavoured Markdown:
-
-https://help.github.com/articles/github-flavored-markdown
-
-Its purpose is to quickly generate `README.md` files for small projects.
-
-
-### API
-
-The interface consists of the following functions:
-
- - `doctrim(docstring)`
- - `doc2md(docstring, title)`
-
-You can run this script from the command line like:
-
-$ doc2md.py [-a] [--no-toc] [-t title] module-name [class-name] > README.md
-
-
-### Limitations
-
-At the moment  this is suited only  for a very specific use  case. It is
-hardly forseeable, if I will decide to improve on it in the near future.
-
-"""
-import re
-import sys
-import inspect
-
-__all__ = ['doctrim', 'doc2md']
-
-doctrim = inspect.cleandoc
-
-def unindent(lines):
-    """
-    Remove common indentation from string.
-
-    Unlike doctrim there is no special treatment of the first line.
-
-    """
-    try:
-        # Determine minimum indentation:
-        indent = min(len(line) - len(line.lstrip())
-                     for line in lines if line)
-    except ValueError:
-        return lines
-    else:
-        return [line[indent:] for line in lines]
-
-def escape_markdown(line):
-    line = line.replace('[', '\[').replace(']', '\]')
-    line = line.replace('(', '\(').replace(')', '\)')
-    line = line.replace('{', '\{').replace('}', '\}')
-    line = line.replace('\\', '\\\\')
-    line = line.replace('`', '\`')
-    line = line.replace('*', '\*')
-    line = line.replace('_', '\_')
-    line = line.replace('#', '\#')
-    line = line.replace('+', '\+')
-    line = line.replace('-', '\-')
-    line = line.replace('.', '\.')
-    line = line.replace('!', '\!')
-    return line
-
-def code_block(lines, language=''):
-    """
-    Mark the code segment for syntax highlighting.
-    """
-    return ['```' + language] + lines + ['```']
-
-def doctest2md(lines):
-    """
-    Convert the given doctest to a syntax highlighted markdown segment.
-    """
-    is_only_code = True
-    lines = unindent(lines)
-    for line in lines:
-        if not line.startswith('>>> ') and not line.startswith('... ') and line not in ['>>>', '...']:
-            is_only_code = False
-            break
-    if is_only_code:
-        orig = lines
-        lines = []
-        for line in orig:
-            lines.append(line[4:])
-    return lines
-
-def doc_code_block(lines, language):
-    if language == 'python':
-        lines = doctest2md(lines)
-    return code_block(lines, language)
-
-_args_section = re.compile('^\s*Args:\s*')
-def is_args_check(line):
-    return _args_section.match(line)
-
-def args_block(lines):
-    out = ['']
-    out += ['Parameter | Default | Description']
-    out += ['--------- | ------- | -----------']
-    for line in lines:
-        matches = re.findall(r'\s*([^:]+):\s*(.*?)\s*(Default:\s(.*))?\s*$', line)
-        assert matches != None
-        name = matches[0][0]
-        description = matches[0][1]
-        default = matches[0][3]
-        out += [name + ' | ' + default + ' | ' + description]
-    return out
-
-# Inputs
-_inputs_section = re.compile('^\s*Inputs:\s*(.*)\s*')
-def is_inputs_check(line):
-    return _inputs_section.match(line)
-
-def inputs_block(lines):
-    out = ['']
-    out += ['Parameter | Default | Description']
-    out += ['--------- | ------- | -----------']
-    for line in lines:
-        matches = re.findall(r'\s*([^:]+):\s*(.*?)\s*(Default:\s(.*))?\s*$', line)
-        assert matches != None
-        name = matches[0][0]
-        description = matches[0][1]
-        default = matches[0][3]
-        out += [name + ' | ' + default + ' | ' + description]
-    return out
-
-# Outputs
-_outputs_section = re.compile('^\s*Outputs:\s*(.*)\s*')
-def is_outputs_check(line):
-    return _outputs_section.match(line)
-
-def outputs_block(lines):
-    out = ['']
-    out += ['Parameter |  Description']
-    out += ['--------- |  -----------']
-    for line in lines:
-        matches = re.findall(r'\s*([^:]+):\s*(.*?)\s*(Default:\s(.*))?\s*$', line)
-        assert matches != None
-        name = matches[0][0]
-        description = matches[0][1]
-        default = matches[0][3]
-        out += [name + ' | ' + description]
-    return out
-
-# Members
-_members_section = re.compile('^\s*Members:\s*(.*)\s*')
-def is_members_check(line):
-    return _members_section.match(line)
-
-def members_block(lines):
-    out = ['']
-    out += ['Parameter | Description']
-    out += ['--------- | -----------']
-    for line in lines:
-        matches = re.findall(r'\s*([^:]+):\s*(.*?)\s*(Default:\s(.*))?\s*$', line)
-        assert matches != None
-        name = matches[0][0]
-        description = matches[0][1]
-        default = matches[0][3]
-        out += [name + ' | ' + description]
-    return out
-
-_returns_section = re.compile('^\s*Returns:\s*')
-def is_returns_check(line):
-    return _returns_section.match(line)
-
-_image_section = re.compile('^\s*Image:\s*')
-def is_image_check(line):
-    return _image_section.match(line)
-
-_example_section = re.compile('^\s*Returns:\s*|^\s*Examples:\s*')
-def is_example_check(line):
-    return _example_section.match(line)
-
-_inputshape_section = re.compile('^\s*Returns:\s*|^\s*Input Shape:\s*')
-def is_inputshape_check(line):
-    return _inputshape_section.match(line)
-
-_outputshape_section = re.compile('^\s*Returns:\s*|^\s*Output Shape:\s*')
-def is_outputshape_check(line):
-    return _outputshape_section.match(line)
-###############################################
-_reg_section = re.compile('^#+ ')
-def is_heading(line):
-    return _reg_section.match(line)
-
-def get_heading(line):
-    assert is_heading(line)
-    part = line.partition(' ')
-    return len(part[0]), part[2]
-
-def make_heading(level, title):
-    return '#'*max(level, 1) + ' ' + title
-
-def find_sections(lines):
-    """
-    Find all section names and return a list with their names.
-    """
-    sections = []
-    for line in lines:
-        if is_heading(line):
-            sections.append(get_heading(line))
-    return sections
-
-def make_toc(sections):
-    """
-    Generate table of contents for array of section names.
-    """
-    if not sections:
-        return []
-    outer = min(n for n,t in sections)
-    refs = []
-    for ind,sec in sections:
-        ref = sec.lower()
-        ref = ref.replace(' ', '-')
-        ref = ref.replace('?', '')
-        refs.append("    "*(ind-outer) + "- [%s](#%s)" % (sec, ref))
-    return refs
-
-def _doc2md(lines, shiftlevel=0):
-    _doc2md.md = []
-    _doc2md.is_code = False
-    _doc2md.is_code_block = False
-    _doc2md.is_args = False
-    _doc2md.is_inputs = False
-    _doc2md.is_outputs = False
-    _doc2md.is_members = False
-    _doc2md.is_returns = False
-    _doc2md.is_inputshape = False
-    _doc2md.is_outputshape = False
-    _doc2md.code = []
-    def reset():
-        if _doc2md.is_code:
-            _doc2md.is_code = False
-            _doc2md.code += doc_code_block(code, 'python')
-            _doc2md.code += ['']
-        if _doc2md.is_code_block:
-            _doc2md.is_code_block = False
-            _doc2md.code += doc_code_block(code_block, 'python')
-            _doc2md.code += ['']
-
-        if _doc2md.is_args:
-            _doc2md.is_args = False
-            _doc2md.md += args_block(args)
-
-        if _doc2md.is_inputs:
-            _doc2md.is_inputs = False
-            _doc2md.md += inputs_block(inputs)
-
-        if _doc2md.is_outputs:
-            _doc2md.is_outputs = False
-            _doc2md.md += outputs_block(outputs)
-
-        if _doc2md.is_members:
-            _doc2md.is_members = False
-            _doc2md.md += members_block(members)
-
-        if _doc2md.is_returns:
-            _doc2md.is_returns = False
-            _doc2md.md += returns
-
-        _doc2md.is_inputshape = False
-        _doc2md.is_outputshape = False
-
-    for line in lines:
-        trimmed = line.lstrip()
-        if is_args_check(line):
-            reset()
-            _doc2md.is_args = True
-            _doc2md.md += ['']
-            _doc2md.md += ['#' * (shiftlevel+2) + ' Constructor Arguments']
-            args = []
-        elif is_inputs_check(line):
-            reset()
-            _doc2md.is_inputs = True
-            _doc2md.md += ['']
-            _doc2md.md += ['#' * (shiftlevel+2) + ' Inputs']
-            inputs = []
-        elif is_outputs_check(line):
-            reset()
-            _doc2md.is_outputs = True
-            _doc2md.md += ['']
-            _doc2md.md += ['#' * (shiftlevel+2) + ' Outputs']
-            outputs = []
-        elif is_members_check(line):
-            reset()
-            _doc2md.is_members = True
-            _doc2md.md += ['']
-            _doc2md.md += ['#' * (shiftlevel+2) + ' Members']
-            members = []
-        elif is_returns_check(line):
-            reset()
-            _doc2md.is_returns = True
-            _doc2md.md += ['']
-            _doc2md.md += ['#' * (shiftlevel+2) + ' Returns']
-            returns = []
-        elif is_example_check(line):
-            reset()
-        elif is_inputshape_check(line):
-            reset()
-            inputshape = re.findall(r'\s*Input\sShape:\s*(.*)\s*:\s*(.*)\s*$', line)[0]
-        elif is_outputshape_check(line):
-            reset()
-            outputshape = re.findall(r'\s*Output\sShape:\s*(.*)\s*:\s*(.*)\s*$', line)[0]
-            _doc2md.md += ['']
-            _doc2md.md += ['#' * (shiftlevel+2) + ' Expected Shape']
-            _doc2md.md += ['       | Shape | Description ']
-            _doc2md.md += ['------ | ----- | ------------']
-            _doc2md.md += [' input | ' + inputshape[0] + ' | ' + inputshape[1]]
-            _doc2md.md += ['output | ' + outputshape[0] + ' | ' + outputshape[1]]
-        elif is_image_check(line):
-            reset()
-            _doc2md.md += ['']
-            filename = re.findall(r'\s*Image:\s*(.*?)\s*$', line)
-            _doc2md.md += ['<img src="image/' + filename[0] + '" >']
-        elif _doc2md.is_code == False and trimmed.startswith('>>> '):
-            reset()
-            _doc2md.is_code = True
-            code = [line]
-        elif _doc2md.is_code_block == False and trimmed.startswith('```'):
-            reset()
-            _doc2md.is_code_block = True
-            code_block = []
-        elif _doc2md.is_code_block == True and trimmed.startswith('```'):
-            # end of code block
-            reset()
-        elif _doc2md.is_code_block:
-            if line:
-                code_block.append(line)
-            else:
-                reset()
-        elif shiftlevel != 0 and is_heading(line):
-            reset()
-            level, title = get_heading(line)
-            _doc2md.md += [make_heading(level + shiftlevel, title)]
-        elif _doc2md.is_args:
-            if line:
-                args.append(line)
-            else:
-                reset()
-        elif _doc2md.is_inputs:
-            if line:
-                inputs.append(line)
-            else:
-                reset()
-        elif _doc2md.is_outputs:
-            if line:
-                outputs.append(line)
-            else:
-                reset()
-        elif _doc2md.is_members:
-            if line:
-                members.append(line)
-            else:
-                reset()
-        elif _doc2md.is_returns:
-            if line:
-                returns.append(line)
-            else:
-                reset()
-        elif _doc2md.is_code:
-            if line:
-                code.append(line)
-            else:
-                reset()
-        else:
-            reset()
-            _doc2md.md += [line]
-    reset()
-    _doc2md.code += _doc2md.md
-    return _doc2md.code
-
-def doc2md(docstr, title, min_level=3, more_info=False, toc=True):
-    """
-    Convert a docstring to a markdown text.
-    """
-    text = doctrim(docstr)
-    lines = text.split('\n')
-
-    sections = find_sections(lines)
-    if sections:
-        level = min(n for n,t in sections) - 1
-    else:
-        level = 1
-
-    shiftlevel = 0
-    if level < min_level:
-        shiftlevel = min_level - level
-        level = min_level
-        sections = [(lev+shiftlevel, tit) for lev,tit in sections]
-
-    md = [
-        make_heading(level, title),
-        "",
-        lines.pop(0),
-        ""
-    ]
-    if toc:
-        md += make_toc(sections)
-    md += _doc2md(lines, shiftlevel)
-    if more_info:
-        return (md, sections)
-    else:
-        return "\n".join(md)
-
-def mod2md(module, title, title_api_section, toc=True):
-    """
-    Generate markdown document from module, including API section.
-    """
-    docstr = module.__doc__  or " "
-
-    text = doctrim(docstr)
-    lines = text.split('\n')
-
-    sections = find_sections(lines)
-    if sections:
-        level = min(n for n,t in sections) - 1
-    else:
-        level = 1
-
-    api_md = []
-    api_sec = []
-    if title_api_section :
-        # sections.append((level+1, title_api_section))
-        for name, entry in iter(module.__dict__.items()):
-            if name[0] != '_' and entry.__doc__:
-                #api_sec.append((level+1, name))
-                #api_md += ['', '']
-                if entry.__doc__:
-                    md, sec = doc2md(entry.__doc__, name,
-                                     min_level=level+1, more_info=True, toc=False)
-                    api_sec += sec
-                    api_md += md
-
-    sections += api_sec
-
-    # headline
-    md = [
-        make_heading(level, title),
-        "",
-        lines.pop(0),
-        ""
-    ]
-
-    # main sections
-    if toc:
-        md += make_toc(sections)
-    md += _doc2md(lines)
-
-    if toc:
-        md += ['']
-        md += make_toc(api_sec)
-    md += api_md
-
-    return "\n".join(md)
-
-def main(args=None):
-    # parse the program arguments
-    import argparse
-    parser = argparse.ArgumentParser(
-            description='Convert docstrings to markdown.')
-
-    parser.add_argument(
-            'module', help='The module containing the docstring.')
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument(
-            'entry', nargs='?',
-            help='Convert only docstring of this entry in module.')
-    group.add_argument(
-            '-a', '--all', dest='all', action='store_true',
-            help='Create an API section with the contents of module.__all__.')
-    parser.add_argument(
-            '-t', '--title', dest='title',
-            help='Document title (default is module name)')
-    parser.add_argument(
-            '--no-toc', dest='toc', action='store_false', default=True,
-            help='Do not automatically generate the TOC')
-    args = parser.parse_args(args)
-
-    import importlib
-    import inspect
-    import os
-
-    def add_path(*pathes):
-        for path in reversed(pathes):
-            if path not in sys.path:
-                sys.path.insert(0, path)
-
-    file = inspect.getfile(inspect.currentframe())
-    add_path(os.path.realpath(os.path.abspath(os.path.dirname(file))))
-    add_path(os.getcwd())
-
-    mod_name = args.module
-    if mod_name.endswith('.py'):
-        mod_name = mod_name.rsplit('.py', 1)[0]
-    title = args.title or mod_name.replace('_', '-')
-
-    module = importlib.import_module(mod_name)
-
-    if args.all:
-        print(mod2md(module, title, 'API', toc=args.toc))
-
-    else:
-        if args.entry:
-            docstr = module.__dict__[args.entry].__doc__ or ''
-        else:
-            docstr = module.__doc__ or ''
-
-        print(doc2md(docstr, title, toc=args.toc))
-
-if __name__ == "__main__":
-    main()

docs/docutils/gendocs.sh

@@ -1,100 +0,0 @@
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-pushd $SCRIPT_DIR
-
-# module
-#python doc2md.py torch.nn Module --title Module --no-toc >../nn_module.md
-
-# containers
-echo "## Containers" > ../nn_container.md
-python doc2md.py torch.nn Container --title Container --no-toc    >>../nn_container.md
-python doc2md.py torch.nn Sequential --title Sequential --no-toc >>../nn_container.md
-
-# convolution
-echo "## Convolution Layers" > ../nn_convolution.md
-echo Conv1d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc          >>../nn_convolution.md
-echo Conv2d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc          >>../nn_convolution.md
-echo ConvTranspose2d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc >>../nn_convolution.md
-echo Conv3d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc          >>../nn_convolution.md
-echo ConvTranspose3d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc >>../nn_convolution.md
-
-# pooling
-echo "## Pooling Layers" > ../nn_pooling.md
-echo MaxPool1d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc           >>../nn_pooling.md
-echo MaxPool2d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc           >>../nn_pooling.md
-echo MaxPool3d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc           >>../nn_pooling.md
-echo MaxUnpool2d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc         >>../nn_pooling.md
-echo MaxUnpool3d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc         >>../nn_pooling.md
-echo AvgPool2d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc           >>../nn_pooling.md
-echo AvgPool3d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc           >>../nn_pooling.md
-echo FractionalMaxPool2d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc >>../nn_pooling.md
-echo LPPool2d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc            >>../nn_pooling.md
-
-# activations
-echo "## Non-linearities" > ../nn_activation.md
-echo ReLU | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc            >>../nn_activation.md
-echo ReLU6 | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc           >>../nn_activation.md
-echo Threshold | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc       >>../nn_activation.md
-echo Hardtanh | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc        >>../nn_activation.md
-echo Sigmoid | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc         >>../nn_activation.md
-echo Tanh | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc            >>../nn_activation.md
-echo ELU | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc             >>../nn_activation.md
-echo LeakyReLU | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc       >>../nn_activation.md
-echo LogSigmoid | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc      >>../nn_activation.md
-echo Softplus | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc        >>../nn_activation.md
-echo Softshrink | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc      >>../nn_activation.md
-echo PReLU | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc           >>../nn_activation.md
-echo Softsign | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc        >>../nn_activation.md
-echo Tanhshrink | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc      >>../nn_activation.md
-echo Softmin | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc         >>../nn_activation.md
-echo Softmax | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc         >>../nn_activation.md
-echo Softmax2d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc       >>../nn_activation.md
-echo LogSoftmax | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc      >>../nn_activation.md
-
-# normalization
-echo "## Normalization layers" > ../nn_normalization.md
-echo BatchNorm1d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc >>../nn_normalization.md
-echo BatchNorm2d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc >>../nn_normalization.md
-echo BatchNorm3d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc >>../nn_normalization.md
-
-# recurrentnet
-echo "## Recurrent layers" > ../nn_recurrent.md
-echo RNN | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc             >>../nn_recurrent.md
-echo LSTM | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc            >>../nn_recurrent.md
-echo GRU | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc             >>../nn_recurrent.md
-echo RNNCell | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc         >>../nn_recurrent.md
-echo LSTMCell | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc        >>../nn_recurrent.md
-echo GRUCell | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc         >>../nn_recurrent.md
-
-# linear
-echo "## Linear layers" > ../nn_linear.md
-echo Linear | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc      >>../nn_linear.md
-
-# dropout
-echo "## Dropout layers" > ../nn_dropout.md
-echo Dropout | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc   >>../nn_dropout.md
-echo Dropout2d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc >>../nn_dropout.md
-echo Dropout3d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc >>../nn_dropout.md
-
-# Sparse
-echo "## Sparse layers" > ../nn_sparse.md
-echo Embedding | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc              >>../nn_sparse.md
-
-# loss_functions
-echo "## Loss functions" > ../nn_loss.md
-echo L1Loss | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc                        >>../nn_loss.md
-echo MSELoss | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc                       >>../nn_loss.md
-echo CrossEntropyLoss | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc              >>../nn_loss.md
-echo NLLLoss | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc                       >>../nn_loss.md
-echo NLLLoss2d | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc                     >>../nn_loss.md
-echo KLDivLoss | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc                     >>../nn_loss.md
-echo BCELoss | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc                       >>../nn_loss.md
-echo MarginRankingLoss | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc             >>../nn_loss.md
-echo HingeEmbeddingLoss | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc            >>../nn_loss.md
-echo MultiLabelMarginLoss | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc          >>../nn_loss.md
-echo SmoothL1Loss | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc                  >>../nn_loss.md
-echo SoftMarginLoss | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc                >>../nn_loss.md
-echo MultiLabelSoftMarginLoss | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc      >>../nn_loss.md
-echo CosineEmbeddingLoss | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc           >>../nn_loss.md
-echo MultiMarginLoss | xargs -I {} python doc2md.py torch.nn {} --title {} --no-toc               >>../nn_loss.md
-
-popd

docs/docutils/torch-cwrap-gen.py

@@ -1,143 +0,0 @@
-import sys
-from tools.cwrap import cwrap
-from tools.cwrap.plugins import CWrapPlugin
-from string import Template
-import sys
-import torch
-from torch.autograd import Variable
-
-def transform_defined_if(defined_if):
-    if defined_if != None:
-        defined_if = defined_if.replace('defined(TH_REAL_IS_FLOAT)', 'Float')
-        defined_if = defined_if.replace('defined(TH_REAL_IS_DOUBLE)', 'Double')
-        defined_if = defined_if.replace('defined(TH_REAL_IS_BYTE)', 'Byte')
-        defined_if = defined_if.replace('defined(TH_REAL_IS_CHAR)', 'Char')
-        defined_if = defined_if.replace('defined(TH_REAL_IS_INT)', 'Int')
-        defined_if = defined_if.replace('defined(TH_REAL_IS_LONG)', 'Long')
-        defined_if = defined_if.replace('defined(NUMPY_TYPE_ENUM)', 
-                                        'Byte // Short // Int // Long // Float // Double')
-        defined_if = defined_if.replace('CUDA_INT', 'Cuda_Int')
-        defined_if = defined_if.replace('CUDA_LONG', 'Cuda_Long')
-        defined_if = defined_if.replace('CUDA_FLOAT', 'Cuda_Float')
-        defined_if = defined_if.replace('CUDA_DOUBLE', 'Cuda_Double')
-        defined_if = defined_if.replace('CUDA_HALF', 'Cuda_Half')
-        defined_if = defined_if.replace('!IS_CUDA', 'All CPU Types')
-    else:
-        defined_if = "All Types (CPU and CUDA)"
-    defined_if = defined_if.replace('||', '//')
-    return defined_if
-
-class DocGen(CWrapPlugin):
-    def __init__(self):
-        self.declarations = {}
-
-    def process_declarations(self, declarations):
-        self.declarations.update({declaration['name']: declaration for declaration in declarations})
-        # self.declarations += declarations
-        return declarations
-
-    def get_wrapper_template(self, declaration):
-        return Template("")
-
-    def get_type_check(self, arg, option):
-        return Template("")
-
-    def get_type_unpack(self, arg, option):
-        return Template("")
-
-    def get_return_wrapper(self, option):
-        return Template("")
-
-    def print_declarations(self):  
-        print("# torch.Tensor")
-        for name, declarations in sorted(self.declarations.items()):
-            if name.endswith('_') and name[:-1] in self.declarations:
-                continue
-            if not name.endswith('_') and name + '_' in self.declarations:
-                inplace = True
-            else:
-                inplace = False
-
-            pname = declarations['options'][0].get('python_name', None)
-            if pname != None:
-                name = pname
-            if name.startswith('_'):
-                continue
-
-            # START PRINTING MARKDOWN
-            print("## " + name + " \n")
-            print("|    %-25s |    %-8s |    %-25s |" % ("Name", "Autograd", "defined if"))
-            print("| " + ('-' * 28) + " | " + ('-' * 11) + " | "+ ('-' * 28) + " |")
-            if inplace:
-                sys.stdout.write("|    %-25s" % (name + '  //  ' + name + "_"))
-            else:
-                sys.stdout.write("|    %-25s" % name)
-            sys.stdout.write(' | ')
-            if hasattr(Variable(torch.randn(10)), name):
-                sys.stdout.write(' %9s ' % 'yes') # + '   ' + name)
-            else:
-                sys.stdout.write(' %9s ' % 'no') # + '   ' + name)
-            defined_if = declarations.get('defined_if', None)
-            defined_if = transform_defined_if(defined_if)
-            sys.stdout.write(' | ')
-            sys.stdout.write(defined_if)
-            sys.stdout.write(' |')
-            sys.stdout.write('\n\n')
-            #if inplace:
-            #    print('Inplace Exists : True')
-            #sys.stdout.write('Arguments  : ')
-
-            args = declarations['options'][0]['arguments']
-            if len(args) == 0:
-                print(    '**No Arguments**\n' )
-            else:
-                print(    '**Arguments**\n' )
-                print("|    %-15s |    %-12s |    %-15s |" % ("Name", "Type", "Default"))
-                print("| " + ('-' * 18) + " | " + ('-' * 15) + " | "+ ('-' * 18) + " |")
-
-                for arg in args:
-                    type_ = arg['type']
-                    if type_ == 'THGenerator*':
-                        continue
-                    if type_ == 'THTensor*':
-                        type_ = 'Tensor'
-                    if type_ == 'THIndexTensor*':
-                        type_ = 'LongTensor'
-                    if type_ == 'THBoolTensor*':
-                        type_ = 'ByteTensor'
-                    if type_ == 'THLongTensor*':
-                        type_ = 'LongTensor'
-                    if type_ == 'THLongStorage*':
-                        type_ = 'LongStorage'
-                    default = arg.get('default', None)
-                    allocated = arg.get('allocate', None)
-                    if default == None and allocated == None:
-                        default = "     [required]"
-                    elif allocated != None:
-                        default = "     [optional]"
-                    else:
-                        default = str(default)
-                        import re
-                        m = re.search('\s*AS_REAL\((.+)\)\s*', default)
-                        if m:
-                            default = m.group(1)
-                            default = default
-
-                    print('| %15s    |  %12s   |   %10s |' % (arg['name'], type_, default))
-                    # print(    'Options    : ' )
-                    # print(declarations['options'][0])
-                print('')
-            if declarations['return']:
-                return_ = declarations['return']
-                if return_ == 'THTensor*':
-                    return_ = 'Tensor'
-                if return_ == 'void':
-                    return_ = 'nothing'
-                print(    '**Returns        : ' + return_ + '**')
-            print('')
-
-
-docs = DocGen()
-cwrap('../../torch/csrc/generic/TensorMethods.cwrap', plugins=[docs])
-
-docs.print_declarations()

docs/make.bat

@@ -0,0 +1,36 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+set SPHINXPROJ=PyTorch
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

docs/nn.md

@@ -1,3 +0,0 @@
-# torch.nn
-
-Neural Networks in PyTorch

docs/nn_activation.md

@@ -1,496 +0,0 @@
-## Non-linearities
-### ReLU
-
-Applies the rectified linear unit function element-wise ReLU(x)= max(0,x)
-
-```python
-m = nn.ReLU()
-input = autograd.Variable(torch.randn(2))
-print(input)
-print(m(input))
-```
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-inplace |  | can optionally do the operation in-place
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | Any  | Tensor of any size and dimension
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input
-
-<img src="image/relu.png" >
-### ReLU6
-
-Applies the element-wise function ReLU6(x) = min( max(0,x), 6)
-
-```python
-m = nn.ReLU6()
-input = autograd.Variable(torch.randn(2))
-print(input)
-print(m(input))
-```
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-inplace |  | can optionally do the operation in-place
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | Any  | Tensor of any size and dimension
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input
-
-<img src="image/relu6.png" >
-### Threshold
-
-Thresholds each element of the input Tensor
-
-```python
-m = nn.Threshold(0.1, 20)
-input = Variable(torch.randn(2))
-print(input)
-print(m(input))
-```
-
-Threshold is defined as:
-     y =  x        if x >= threshold
-          value    if x <  threshold
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-threshold |  | The value to threshold at
-value |  | The value to replace with
-inplace |  | can optionally do the operation in-place
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | Any  | Tensor of any size and dimension
-output | Same  | Output has the same shape as input
-
-#### Returns
-    Tensor of same dimension and shape as the input
-### Hardtanh
-
-Applies the HardTanh function element-wise
-
-```python
-m = nn.HardTanh(-2, 2)
-input = autograd.Variable(torch.randn(2))
-print(input)
-print(m(input))
-```
-
-HardTanh is defined as:
-   f(x) = +1, if x  >  1
-   f(x) = -1, if x  < -1
-   f(x) =  x,  otherwise
-The range of the linear region [-1, 1] can be adjusted
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-min_value |  | minimum value of the linear region range
-max_value |  | maximum value of the linear region range
-inplace |  | can optionally do the operation in-place
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | Any  | Tensor of any size and dimension
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input
-
-<img src="image/htanh.png" >
-### Sigmoid
-
-Applies the element-wise function sigmoid(x) = 1 / ( 1 + exp(-x))
-
-```python
-m = nn.Sigmoid()
-input = autograd.Variable(torch.randn(2))
-print(input)
-print(m(input))
-```
-
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | Any  | Tensor of any size and dimension
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input
-
-<img src="image/sigmoid.png" >
-### Tanh
-
-Applies element-wise, Tanh(x) = (exp(x) - exp(-x)) / (exp(x) + exp(-x))
-
-```python
-m = nn.Tanh()
-input = autograd.Variable(torch.randn(2))
-print(input)
-print(m(input))
-```
-
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | Any  | Tensor of any size and dimension
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input
-
-<img src="image/tanh.png" >
-### ELU
-
-Applies element-wise, ELU(x) = max(0,x) + min(0, alpha * (exp(x) - 1))
-
-```python
-m = nn.ELU()
-input = autograd.Variable(torch.randn(2))
-print(input)
-print(m(input))
-```
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-alpha | 1.0 | the alpha value for the ELU formulation.
-inplace |  | can optionally do the operation in-place
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | Any  | Tensor of any size and dimension
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input
-
-<img src="image/elu.png" >
-### LeakyReLU
-
-Applies element-wise, f(x) = max(0, x) + negative_slope * min(0, x)
-
-```python
-m = nn.LeakyReLU(0.1)
-input = autograd.Variable(torch.randn(2))
-print(input)
-print(m(input))
-```
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-negative_slope | 1e-2 | Controls the angle of the negative slope.
-inplace |  | can optionally do the operation in-place
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | Any  | Tensor of any size and dimension
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input
-### LogSigmoid
-
-Applies element-wise LogSigmoid(x) = log( 1 / (1 + exp(-x_i)))
-
-```python
-m = nn.LogSigmoid()
-input = autograd.Variable(torch.randn(2))
-print(input)
-print(m(input))
-```
-
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | Any  | Tensor of any size and dimension
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input
-
-<img src="image/logsigmoid.png" >
-### Softplus
-
-Applies element-wise SoftPlus(x) = 1/beta * log(1 + exp(beta * x_i))
-
-```python
-m = nn.Softplus()
-input = autograd.Variable(torch.randn(2))
-print(input)
-print(m(input))
-```
-
-SoftPlus is a smooth approximation to the ReLU function and can be used
-to constrain the output of a machine to always be positive.
-For numerical stability the implementation reverts to the linear function
-for inputs above a certain value.
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-beta | 1 | the beta value for the Softplus formulation.
-threshold | 20 | values above this revert to a linear function.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | Any  | Tensor of any size and dimension
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input
-
-<img src="image/softplus.png" >
-### Softshrink
-
-Applies the soft shrinkage function elementwise
-
-```python
-m = nn.Softshrink()
-input = autograd.Variable(torch.randn(2))
-print(input)
-print(m(input))
-```
-
-SoftShrinkage operator is defined as:
-    f(x) = x-lambda, if x > lambda >  f(x) = x+lambda, if x < -lambda
-    f(x) = 0, otherwise
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-lambd | 0.5 | the lambda value for the Softshrink formulation.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | Any  | Tensor of any size and dimension
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input
-
-<img src="image/sshrink.png" >
-### PReLU
-
-Applies element-wise the function PReLU(x) = max(0,x) + a * min(0,x)
-
-```python
-m = nn.PReLU()
-input = autograd.Variable(torch.randn(2))
-print(input)
-print(m(input))
-```
-
-Here "a" is a learnable parameter.
-When called without arguments, nn.PReLU() uses a single parameter "a"
-across all input channels. If called with nn.PReLU(nChannels), a separate
-"a" is used for each input channel.
-Note that weight decay should not be used when learning "a" for good
-performance.
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-num_parameters | 1 | number of "a" to learn.
-init | 0.25 | the initial value of "a".
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | Any  | Tensor of any size and dimension
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input
-
-<img src="image/prelu.png" >
-### Softsign
-
-Applies element-wise, the function Softsign(x) = x / (1 + |x|)
-
-```python
-m = nn.Softsign()
-input = autograd.Variable(torch.randn(2))
-print(input)
-print(m(input))
-```
-
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | Any  | Tensor of any size and dimension
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input
-
-<img src="image/softsign.png" >
-### Tanhshrink
-
-Applies element-wise, Tanhshrink(x) = x - Tanh(x)
-
-```python
-m = nn.Tanhshrink()
-input = autograd.Variable(torch.randn(2))
-print(input)
-print(m(input))
-```
-
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | Any  | Tensor of any size and dimension
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input
-### Softmin
-
-Applies the Softmin function to an n-dimensional input Tensor
-
-```python
-m = nn.Softmin()
-input = autograd.Variable(torch.randn(2, 3))
-print(input)
-print(m(input))
-```
-
-rescaling them so that the elements of the n-dimensional output Tensor
-lie in the range (0,1) and sum to 1
-Softmin(x) = exp(-x_i - shift) / sum_j exp(-x_j - shift)
-             where shift = max_i - x_i
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , * ]  | 2D Tensor of any size
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input, with
-    values in the range [0, 1]
-
-<img src="image/softmin.png" >
-### Softmax
-
-Applies the Softmax function to an n-dimensional input Tensor
-
-```python
-m = nn.Softmax()
-input = autograd.Variable(torch.randn(2, 3))
-print(input)
-print(m(input))
-```
-
-rescaling them so that the elements of the n-dimensional output Tensor
-lie in the range (0,1) and sum to 1
-
-Softmax is defined as f_i(x) = exp(x_i - shift) / sum_j exp(x_j - shift)
-                      where shift = max_i x_i
-
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , * ]  | 2D Tensor of any size
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input with
-    values in the range [0, 1]
-
-<img src="image/softmax.png" >
-Notes:
-    Note that this module doesn't work directly with NLLLoss,
-    which expects the Log to be computed between the Softmax and itself.
-    Use Logsoftmax instead (it's faster).
-### Softmax2d
-
-Applies SoftMax over features to each spatial location
-
-```python
-m = nn.Softmax2d()
-# you softmax over the 2nd dimension
-input = autograd.Variable(torch.randn(2, 3, 12, 13))
-print(input)
-print(m(input))
-```
-
-When given an image of Channels x Height x Width, it will
-apply Softmax to each location [Channels, h_i, w_j]
-
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , * , * , * ]  | 4D Tensor of any size
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input with
-    values in the range [0, 1]
-### LogSoftmax
-
-Applies the Log(Softmax(x)) function to an n-dimensional input Tensor.
-
-```python
-m = nn.LogSoftmax()
-input = autograd.Variable(torch.randn(2, 3))
-print(input)
-print(m(input))
-```
-
-The LogSoftmax formulation can be simplified as
-     f_i(x) = log(1 / a * exp(x_i)) where a = sum_j exp(x_j) .
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , * ]  | 2D Tensor of any size
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a Tensor of the same dimension and shape as the input with
-    values in the range [-inf, 0)
-
-<img src="image/logsoftmax.png" >

docs/nn_container.md

@@ -1,136 +0,0 @@
-## Containers
-### Container
-
-This is the base container class for all neural networks you would define.
-
-```python
-# Example of using Container
- class Net(nn.Container):
-    def __init__(self):
-        super(Net, self).__init__(
-            conv1 = nn.Conv2d(1, 20, 5),
-            relu  = nn.ReLU()
-         )
-    def forward(self, input):
-        output = self.relu(self.conv1(x))
-        return output
- model = Net()
-```
-
-```python
-# one can add modules to the container after construction
-model.add_module('pool1', nn.MaxPool2d(2, 2))
-```
-
-```python
-```
-
-```python
-# .parameters()
-```
-
-```python
->>> for param in model.parameters():
->>>     print(type(param.data), param.size())
-<class 'torch.FloatTensor'> (20L,)
-<class 'torch.FloatTensor'> (20L, 1L, 5L, 5L)
-```
-
-```python
-```
-
-```python
-# .parameter_dict()
-```
-
-```python
->>> pdict = model.parameter_dict()
->>> print(pdict.keys())
-['conv1.bias', 'conv1.weight']
-```
-
-```python
-```
-
-You will subclass your container from this class.
-In the constructor you define the modules that you would want to use,
-and in the "forward" function you use the constructed modules in
-your operations.
-
-To make it easier to understand, given is a small example.
-
-One can also add new modules to a container after construction.
-You can do this with the add_module function 
-or by assigning them as Container attributes.
-
-#### one can also set modules as attributes of the container
-model.conv1 = nn.Conv2d(12, 24, 3)
-The container has some important additional methods: 
-
-**`[generator] parameters()`**
-
-returns a generator over all learnable parameters in the container instance. 
-This can typically be passed to the optimizer API
-
-**`[dict] parameter_dict()`**
-
-returns a dictionary of learnable parameters of the Container.
-For example: ['conv1.weight' : Parameter(torch.FloatTensor(20x1x5x5)),
-              'conv1.bias'   : Parameter(torch.FloatTensor(20)),
-             ]
-
-
-**`load_parameter_dict(dict)`**
-
-Given a parameter dict, sets the parameters of self to be the given dict.
-It loads loads the parameters recursively.
-Excessive or non-matching parameter names are ignored.
-For example, the input dict has an entry 'conv44.weight', but 
-if the container does not have a module named 'conv44', then this entry is ignored.
-
-**`children()`**
-
-Returns a generator over all the children modules of self
-
-**`train()`**
-
-Sets the Container (and all it's child modules) to training mode (for modules such as batchnorm, dropout etc.)
-
-**`eval()`**
-
-Sets the Container (and all it's child modules) to evaluate mode (for modules such as batchnorm, dropout etc.)
-
-**`apply(closure)`**
-
-Applies the given closure to each parameter of the container. 
-
-
-**__Note: Apart from these, the container will define the base functions that it has derived from nn.Module __**
-### Sequential
-
-A sequential Container. It is derived from the base nn.Container class
-
-```python
-# Example of using Sequential
-model = nn.Sequential(
-          nn.Conv2d(1,20,5),
-          nn.ReLU(),
-          nn.Conv2d(20,64,5),
-          nn.ReLU()
-        )
-```
-
-```python
-```
-
-Modules will be added to it in the order they are passed in the constructor.
-Alternatively, an ordered dict of modules can also be passed in.
-
-To make it easier to understand, given is a small example.
-#### Example of using Sequential with OrderedDict
-model = nn.Sequential(OrderedDict([
-          ('conv1', nn.Conv2d(1,20,5)),
-          ('relu1', nn.ReLU()),
-          ('conv2', nn.Conv2d(20,64,5)),
-          ('relu2', nn.ReLU())
-        ]))

docs/nn_convolution.md

@@ -1,236 +0,0 @@
-## Convolution Layers
-### Conv1d
-
-Applies a 1D convolution over an input signal composed of several input
-
-```python
-The output value of the layer with input (b x iC x W) and output (b x oC x oW)
-can be precisely described as:
-output[b_i][oc_i][w_i] = bias[oc_i]
-            + sum_iC sum_{ow = 0, oW-1} sum_{kw = 0 to kW-1}
-                weight[oc_i][ic_i][kw] * input[b_i][ic_i][stride_w * ow + kw)]
-```
-
-```python
-m = nn.Conv1d(16, 33, 3, stride=2)
-input = autograd.Variable(torch.randn(20, 16, 50))
-output = m(input)
-```
-
-planes.
-
-
-Note that depending of the size of your kernel, several (of the last)
-columns of the input might be lost. It is up to the user
-to add proper padding.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-in_channels |  | The number of expected input channels in the image given as input
-out_channels |  | The number of output channels the convolution layer will produce
-kernel_size |  | the size of the convolving kernel.
-stride |  | the stride of the convolving kernel.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , in_channels  , * ]  | Input is minibatch x in_channels x iW
-output | [ * , out_channels , * ]   | Output shape is precisely minibatch x out_channels x floor((iW  + 2*padW - kW) / dW + 1)
-
-#### Members
-
-Parameter | Description
---------- | -----------
-weight | the learnable weights of the module of shape (out_channels x in_channels x kW)
-bias | the learnable bias of the module of shape (out_channels)
-### Conv2d
-
-Applies a 2D convolution over an input image composed of several input
-
-```python
-The output value of the layer with input (b x iC x H x W) and output (b x oC x oH x oW)
-can be precisely described as:
-output[b_i][oc_i][h_i][w_i] = bias[oc_i]
-            + sum_iC sum_{oh = 0, oH-1} sum_{ow = 0, oW-1} sum_{kh = 0 to kH-1} sum_{kw = 0 to kW-1}
-                weight[oc_i][ic_i][kh][kw] * input[b_i][ic_i][stride_h * oh + kh)][stride_w * ow + kw)]
-```
-
-```python
-# With square kernels and equal stride
-m = nn.Conv2d(16, 33, 3, stride=2)
-# non-square kernels and unequal stride and with padding
-m = nn.Conv2d(16, 33, (3, 5), stride=(2, 1), padding=(4, 2))
-# non-square kernels and unequal stride and with padding and dilation
-m = nn.Conv2d(16, 33, (3, 5), stride=(2, 1), padding=(4, 2), dilation=(3, 1))
-input = autograd.Variable(torch.randn(20, 16, 50, 100))
-output = m(input)
-```
-
-planes.
-
-
-Note that depending of the size of your kernel, several (of the last)
-columns or rows of the input image might be lost. It is up to the user
-to add proper padding in images.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-in_channels |  | The number of expected input channels in the image given as input
-out_channels |  | The number of output channels the convolution layer will produce
-kernel_size |  | the size of the convolving kernel. Can be a single number k (for a square kernel of k x k) or a tuple (kh x kw)
-stride | 1 | the stride of the convolving kernel. Can be a single number s or a tuple (sh x sw).
-padding | 0 | implicit zero padding on the input. Can be a single number s or a tuple.
-dilation | None | If given, will do dilated (or atrous) convolutions. Can be a single number s or a tuple.
-bias | True | If set to False, the layer will not learn an additive bias.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , in_channels  , * , * ]  | Input is minibatch x in_channels x iH x iW
-output | [ * , out_channels , * , * ]   | Output shape is precisely minibatch x out_channels x floor((iH  + 2*padH - kH) / dH + 1) x floor((iW  + 2*padW - kW) / dW + 1)
-
-#### Members
-
-Parameter | Description
---------- | -----------
-weight | the learnable weights of the module of shape (out_channels x in_channels x kH x kW)
-bias | the learnable bias of the module of shape (out_channels)
-### ConvTranspose2d
-
-Applies a 2D deconvolution operator over an input image composed of several input
-
-```python
-# With square kernels and equal stride
-m = nn.ConvTranspose2d(16, 33, 3, stride=2)
-# non-square kernels and unequal stride and with padding
-m = nn.ConvTranspose2d(16, 33, (3, 5), stride=(2, 1), padding=(4, 2))
-input = autograd.Variable(torch.randn(20, 16, 50, 100))
-output = m(input)
-# exact output size can be also specified as an argument
-input = autograd.Variable(torch.randn(1, 16, 12, 12))
-downsample = nn.Conv2d(16, 16, 3, stride=2, padding=1)
-upsample = nn.ConvTranspose2d(16, 16, 3, stride=2, padding=1)
-h = downsample(input)
-output = upsample(h, output_size=input.size())
-```
-
-planes.
-The deconvolution operator multiplies each input value element-wise by a learnable kernel,
-and sums over the outputs from all input feature planes.
-This module can be seen as the exact reverse of the Conv2d module.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-in_channels |  | The number of expected input channels in the image given as input
-out_channels |  | The number of output channels the convolution layer will produce
-kernel_size |  | the size of the convolving kernel. Can be a single number k (for a square kernel of k x k) or a tuple (kh x kw)
-stride | 1 | the stride of the convolving kernel. Can be a single number or a tuple (sh x sw).
-padding | 0 | implicit zero padding on the input. Can be a single number or a tuple.
-output_padding | 0 | A zero-padding of 0 <= padding < stride that should be added to the output. Can be a single number or a tuple.
-bias | True | If set to False, the layer will not learn an additive bias.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , in_channels  , * , * ]  | Input is minibatch x in_channels x iH x iW
-output | [ * , out_channels , * , * ]   | Output shape is minibatch x out_channels x (iH - 1) * sH - 2*padH + kH + output_paddingH x (iW - 1) * sW - 2*padW + kW, or as specified in a second argument to the call.
-
-#### Members
-
-Parameter | Description
---------- | -----------
-weight | the learnable weights of the module of shape (in_channels x out_channels x kH x kW)
-bias | the learnable bias of the module of shape (out_channels)
-### Conv3d
-
-Applies a 3D convolution over an input image composed of several input
-
-```python
-# With square kernels and equal stride
-m = nn.Conv3d(16, 33, 3, stride=2)
-# non-square kernels and unequal stride and with padding
-m = nn.Conv3d(16, 33, (3, 5, 2), stride=(2, 1, 1), padding=(4, 2, 0))
-input = autograd.Variable(torch.randn(20, 16, 10, 50, 100))
-output = m(input)
-```
-
-planes.
-
-Note that depending of the size of your kernel, several (of the last)
-columns or rows of the input image might be lost. It is up to the user
-to add proper padding in images.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-in_channels |  | The number of expected input channels in the image given as input
-out_channels |  | The number of output channels the convolution layer will produce
-kernel_size |  | the size of the convolving kernel. Can be a single number k (for a square kernel of k x k x k) or a tuple (kt x kh x kw)
-stride | 1 | the stride of the convolving kernel. Can be a single number s or a tuple (kt x sh x sw).
-padding | 0 | implicit zero padding on the input. Can be a single number s or a tuple.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , in_channels  , * , * , * ]  | Input is minibatch x in_channels x iT x iH x iW
-output | [ * , out_channels , * , * , * ]   | Output shape is precisely minibatch x out_channels x floor((iT  + 2*padT - kT) / dT + 1) x floor((iH  + 2*padH - kH) / dH + 1) x floor((iW  + 2*padW - kW) / dW + 1)
-
-#### Members
-
-Parameter | Description
---------- | -----------
-weight | the learnable weights of the module of shape (out_channels x in_channels x kT x kH x kW)
-bias | the learnable bias of the module of shape (out_channels)
-### ConvTranspose3d
-
-Applies a 3D deconvolution operator over an input image composed of several input
-
-```python
-# With square kernels and equal stride
-m = nn.ConvTranspose3d(16, 33, 3, stride=2)
-# non-square kernels and unequal stride and with padding
-m = nn.Conv3d(16, 33, (3, 5, 2), stride=(2, 1, 1), padding=(0, 4, 2))
-input = autograd.Variable(torch.randn(20, 16, 10, 50, 100))
-output = m(input)
-```
-
-planes.
-The deconvolution operator multiplies each input value element-wise by a learnable kernel,
-and sums over the outputs from all input feature planes.
-This module can be seen as the exact reverse of the Conv3d module.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-in_channels |  | The number of expected input channels in the image given as input
-out_channels |  | The number of output channels the convolution layer will produce
-kernel_size |  | the size of the convolving kernel. Can be a single number k (for a square kernel of k x k x k) or a tuple (kt x kh x kw)
-stride | 1 | the stride of the convolving kernel. Can be a single number or a tuple (st x sh x sw).
-padding | 0 | implicit zero padding on the input. Can be a single number or a tuple.
-output_padding | 0 | A zero-padding of 0 <= padding < stride that should be added to the output. Can be a single number or a tuple.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , in_channels  , * , * , * ]  | Input is minibatch x in_channels x iH x iW
-output | [ * , out_channels , * , * , * ]   | Output shape is precisely minibatch x out_channels x (iT - 1) * sT - 2*padT + kT + output_paddingT x (iH - 1) * sH - 2*padH + kH + output_paddingH x (iW - 1) * sW - 2*padW + kW
-
-#### Members
-
-Parameter | Description
---------- | -----------
-weight | the learnable weights of the module of shape (in_channels x out_channels x kT x kH x kW)
-bias | the learnable bias of the module of shape (out_channels)

docs/nn_core.md

@@ -1,233 +0,0 @@
-# Module
-
-This is the base class for all Modules defined in the nn package.
-
-```python
-# .parameters()
-```
-
-```python
->>> for param in model.parameters():
->>>     print(type(param.data), param.size())
-<class 'torch.FloatTensor'> (20L,)
-<class 'torch.FloatTensor'> (20L, 1L, 5L, 5L)
-```
-
-```python
-```
-
-```python
-# .parameter_dict()
-```
-
-```python
->>> pdict = model.parameter_dict()
->>> print(pdict.keys())
-['bias', 'weight']
-```
-
-```python
-```
-
-Even the Container class derives from this class.
-
-An nn.Module has the following interface:
-
-**Constructor:**
-   nn.Module(**parameters)
-
-All arguments passed in to the constructor need to be of type 
-nn.Parameter or a Tensor.
-
-
-**forward(...)**
-
-This is the function that one defines when subclassing to create
-their own modules.
-It takes in inputs and returns outputs.
-
-**__call__(...)**
-
-This calls the forward function, as well as the hooks
-
-**register_buffer(name, tensor)**
-
-This is typically used to register a buffer that is not a Parameter.
-For example, in BatchNorm, the running_mean is a buffer, so one would
-register it in the constructor of BatchNorm with:
-
-`self.register_buffer('running_mean', torch.zeros(num_features))`
-
-The registered buffers can simply be accessed as class members
-when needed.
-
-**cpu()**
-
-Recursively moves all it's parameters and buffers to the CPU
-
-**cuda(device_id=None)**
-Recursively moves all it's parameters and buffers to the CUDA memory.
-If device_id is given, moves it to GPU number device_id
-
-**float()**
-Typecasts the parameters and buffers to float
-
-**double()**
-Typecasts the parameters and buffers to double
-
-**register_forward_hook(name, hook)**
-
-This will register a user-defined closure on the module.
-Whenever the module finishes it's forward operation,
-the user closure is called.
-The signature of the closure is `def closure(input, output)`
-
-**register_backward_hook(name, hook)**
-
-This will register a user-defined closure on the module.
-Whenever the module finishes it's backward operation,
-the user closure is called.
-The signature of the closure is `def closure(gradOutput, gradInput)`
-
-**remove_forward_hook(name)**
-
-Removes a registered forward hook with the given name
-
-**remove_backward_hook(name)**
-
-Removes a registered backward hook with the given name
-
-**`[generator] parameters()`**
-
-returns a generator over all learnable parameters in the container instance. 
-This can typically be passed to the optimizer API
-
-**`[dict] parameter_dict()`**
-
-returns a dictionary of learnable parameters of the Module.
-For example: ['weight' : Parameter(torch.FloatTensor(20x1x5x5)),
-              'bias'   : Parameter(torch.FloatTensor(20)),
-             ]
-
-**`load_parameter_dict(dict)`**
-
-Given a parameter dict, sets the parameters of self to be the given dict.
-
-**`train()`**
-
-Sets the Container to training mode (for modules such as batchnorm, dropout etc.)
-
-**`eval()`**
-
-Sets the Container to evaluate mode (for modules such as batchnorm, dropout etc.)
-
-**`zero_grad()`**
-
-Zeroes the gradients of each Parameter of the module
-# Container
-
-This is the base container class for all neural networks you would define.
-
-```python
-# Example of using Container
- class Net(nn.Container):
-    def __init__(self):
-        super(Net, self).__init__(
-            conv1 = nn.Conv2d(1, 20, 5),
-            relu  = nn.ReLU()
-         )
-    def forward(self, input):
-        output = self.relu(self.conv1(x))
-        return output
- model = Net()
-```
-
-```python
-# one can add modules to the container after construction
-model.add_module('pool1', nn.MaxPool2d(2, 2))
-```
-
-```python
-```
-
-```python
-# .parameters()
-```
-
-```python
->>> for param in model.parameters():
->>>     print(type(param.data), param.size())
-<class 'torch.FloatTensor'> (20L,)
-<class 'torch.FloatTensor'> (20L, 1L, 5L, 5L)
-```
-
-```python
-```
-
-```python
-# .parameter_dict()
-```
-
-```python
->>> pdict = model.parameter_dict()
->>> print(pdict.keys())
-['conv1.bias', 'conv1.weight']
-```
-
-```python
-```
-
-You will subclass your container from this class.
-In the constructor you define the modules that you would want to use,
-and in the "forward" function you use the constructed modules in
-your operations.
-
-To make it easier to understand, given is a small example.
-
-One can also add new modules to a container after construction.
-You can do this with the add_module function 
-or by assigning them as Container attributes.
-
-## one can also set modules as attributes of the container
-model.conv1 = nn.Conv2d(12, 24, 3)
-The container has some important additional methods: 
-
-**`[generator] parameters()`**
-
-returns a generator over all learnable parameters in the container instance. 
-This can typically be passed to the optimizer API
-
-**`[dict] parameter_dict()`**
-
-returns a dictionary of learnable parameters of the Container.
-For example: ['conv1.weight' : Parameter(torch.FloatTensor(20x1x5x5)),
-              'conv1.bias'   : Parameter(torch.FloatTensor(20)),
-             ]
-
-
-**`load_parameter_dict(dict)`**
-
-Given a parameter dict, sets the parameters of self to be the given dict.
-It loads loads the parameters recursively.
-Excessive or non-matching parameter names are ignored.
-For example, the input dict has an entry 'conv44.weight', but 
-if the container does not have a module named 'conv44', then this entry is ignored.
-
-**`children()`**
-
-Returns a generator over all the children modules of self
-
-**`train()`**
-
-Sets the Container (and all it's child modules) to training mode (for modules such as batchnorm, dropout etc.)
-
-**`eval()`**
-
-Sets the Container (and all it's child modules) to evaluate mode (for modules such as batchnorm, dropout etc.)
-
-**`apply(closure)`**
-
-Applies the given closure to each parameter of the container. 
-
-
-**__Note: Apart from these, the container will define the base functions that it has derived from nn.Module __**

docs/nn_dropout.md

@@ -1,90 +0,0 @@
-## Dropout layers
-### Dropout
-
-Randomly zeroes some of the elements of the input tensor.
-
-```python
-m = nn.Dropout(p=0.2)
-input = autograd.Variable(torch.randn(20, 16))
-output = m(input)
-```
-
-The elements to zero are randomized on every forward call.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-p | 0.5 | probability of an element to be zeroed.
-inplace | false | If set to True, will do this operation in-place.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | Any  | Input can be of any shape
-output | Same   | Output is of the same shape as input
-### Dropout2d
-
-Randomly zeroes whole channels of the input tensor.
-
-```python
-m = nn.Dropout2d(p=0.2)
-input = autograd.Variable(torch.randn(20, 16, 32, 32))
-output = m(input)
-```
-
-The input is 4D (batch x channels, height, width) and each channel
-is of size (1, height, width).
-The channels to zero are randomized on every forward call.
-Usually the input comes from Conv2d modules.
-
-As described in the paper "Efficient Object Localization Using Convolutional
-Networks" (http:arxiv.org/abs/1411.4280), if adjacent pixels within
-feature maps are strongly correlated (as is normally the case in early
-convolution layers) then iid dropout will not regularize the activations
-and will otherwise just result in an effective learning rate decrease.
-In this case, nn.Dropout2d will help promote independence between
-feature maps and should be used instead.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-p | 0.5 | probability of an element to be zeroed.
-inplace | false | If set to True, will do this operation in-place.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [*, *, *, *]  | Input can be of any sizes of 4D shape
-output | Same   | Output is of the same shape as input
-### Dropout3d
-
-Randomly zeroes whole channels of the input tensor.
-
-```python
-m = nn.Dropout3d(p=0.2)
-input = autograd.Variable(torch.randn(20, 16, 4, 32, 32))
-output = m(input)
-```
-
-The input is 5D (batch x channels, depth, height, width) and each channel
-is of size (1, depth, height, width).
-The channels to zero are randomized on every forward call.
-Usually the input comes from Conv3d modules.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-p | 0.5 | probability of an element to be zeroed.
-inplace | false | If set to True, will do this operation in-place.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [*, *, *, *, *]  | Input can be of any sizes of 5D shape
-output | Same   | Output is of the same shape as input

docs/nn_linear.md

@@ -1,36 +0,0 @@
-## Linear layers
-### Linear
-
-Applies a linear transformation to the incoming data, y = Ax + b
-
-```python
-m = nn.Linear(20, 30)
-input = autograd.Variable(torch.randn(128, 20))
-output = m(input)
-print(output.size())
-```
-
-The input is a 2D mini-batch of samples, each of size in_features
-The output will be a 2D Tensor of size mini-batch x out_features
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-in_features |  | size of each input sample
-out_features |  | size of each output sample
-bias | True | If set to False, the layer will not learn an additive bias.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [*, in_features]  | Input can be of shape minibatch x in_features
-output | [*, out_features]   | Output is of shape minibatch x out_features
-
-#### Members
-
-Parameter | Description
---------- | -----------
-weight | the learnable weights of the module of shape (out_features x in_features)
-bias | the learnable bias of the module of shape (out_features)

docs/nn_loss.md

@@ -1,294 +0,0 @@
-## Loss functions
-### L1Loss
-
-Creates a criterion that measures the mean absolute value of the 
-
-element-wise difference between input `x` and target `y`:
-
-loss(x, y)  = 1/n \sum |x_i - y_i|
-
-`x` and `y` arbitrary shapes with a total of `n` elements each
-the sum operation still operates over all the elements, and divides by `n`.
-
-The division by `n` can be avoided if one sets the internal 
-variable `sizeAverage` to `False`
-### MSELoss
-
-Creates a criterion that measures the mean squared error between 
-
-`n` elements in the input `x` and target `y`:
-    loss(x, y) = 1/n \sum |x_i - y_i|^2
-`x` and `y` arbitrary shapes with a total of `n` elements each
-the sum operation still operates over all the elements, and divides by `n`.
-
-The division by `n` can be avoided if one sets the internal variable 
-`sizeAverage` to `False`
-By default, the losses are averaged over observations for each minibatch. 
-However, if the field `sizeAverage = False`, the losses are instead summed.
-### CrossEntropyLoss
-
-This criterion combines `LogSoftMax` and `ClassNLLLoss` in one single class.
-
-
-It is useful when training a classification problem with `n` classes.
-If provided, the optional argument `weights` should be a 1D `Tensor` 
-assigning weight to each of the classes. 
-This is particularly useful when you have an unbalanced training set.
-
-The `input` is expected to contain scores for each class: 
-      `input` has to be a 2D `Tensor` of size `batch x n`.
-This criterion expects a class index (0 to nClasses-1) as the 
-`target` for each value of a 1D tensor of size `n`
-
-The loss can be described as:
-
-loss(x, class) = -log(exp(x[class]) / (\sum_j exp(x[j])))
-               = -x[class] + log(\sum_j exp(x[j]))
-
-or in the case of the `weights` argument being specified:
-
-loss(x, class) = weights[class] * (-x[class] + log(\sum_j exp(x[j])))
-
-The losses are averaged across observations for each minibatch.
-### NLLLoss
-
-The negative log likelihood loss. It is useful to train a classication problem with n classes
-
-```python
-m = nn.LogSoftmax()
-loss = nn.NLLLoss()
-# input is of size nBatch x nClasses = 3 x 5
-input = autograd.Variable(torch.randn(3, 5))
-# each element in target has to have 0 <= value < nclasses 
-target = autograd.Variable(torch.LongTensor([1, 0, 4]))
-output = loss(m(input), target)
-output.backward()
-```
-
-
-If provided, the optional argument `weights` should be a 1D Tensor assigning
-weight to each of the classes.
-This is particularly useful when you have an unbalanced training set.
-
-The input given through a forward call is expected to contain log-probabilities
-of each class: input has to be a 2D Tensor of size minibatch x n
-Obtaining log-probabilities in a neural network is easily achieved by
-adding a  `LogSoftmax`  layer in the last layer.
-You may use `CrossEntropyLoss`  instead, if you prefer not to
-add an extra layer.
-
-The target that this loss expects is a class index (1 to the number of class)
-
-The loss can be described as:
-    loss(x, class) = -x[class]
-
-or in the case of the weights argument it is specified as follows:
-    loss(x, class) = -weights[class] * x[class]
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-weight | None | a manual rescaling weight given to each class. If given, has to be a Tensor of size "nclasses".
-size_average | True | By default, the losses are averaged over observations for each minibatch. However, if the field sizeAverage is set to False, the losses are instead summed for each minibatch.
-Target Shape: [ * ] : Targets of size [minibatch], each value has to be 1 <= targets[i] <= nClasses
-
-#### Members
-
-Parameter | Description
---------- | -----------
-weight | the class-weights given as input to the constructor
-### NLLLoss2d
-
-This is negative log likehood loss, but for image inputs. It computes NLL loss per-pixel.
-
-```python
-m = nn.Conv2d(16, 32, (3, 3)).float()
-loss = nn.NLLLoss2d()
-# input is of size nBatch x nClasses x height x width
-input = autograd.Variable(torch.randn(3, 16, 10, 10))
-# each element in target has to have 0 <= value < nclasses
-target = autograd.Variable(torch.LongTensor(3, 8, 8).random_(0, 4))
-output = loss(m(input), target)
-output.backward()
-```
-
-This loss does not support per-class weights
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-size_average | True | By default, the losses are averaged over observations for each minibatch. However, if the field sizeAverage is set to False, the losses are instead summed for each minibatch.
-Target Shape: [ * , *, *] : Targets of size minibatch x height x width, each value has to be 1 <= targets[i] <= nClasses
-### KLDivLoss
-
-The [Kullback-Leibler divergence](http://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence) Loss
-
-KL divergence is a useful distance measure for continuous distributions 
-and is often useful when performing direct regression over the space of
-(discretely sampled) continuous output distributions.
-As with ClassNLLLoss, the `input` given is expected to contain 
-_log-probabilities_, however unlike ClassNLLLoss, `input` is not 
-restricted to a 2D Tensor, because the criterion is applied element-wise.
-
-This criterion expects a `target` `Tensor` of the same size as the 
-`input` `Tensor`.
-
-The loss can be described as:
-    loss(x, target) = 1/n \sum(target_i * (log(target_i) - x_i))
-
-By default, the losses are averaged for each minibatch over observations 
-*as well as* over dimensions. However, if the field 
-`sizeAverage` is set to `False`, the losses are instead summed.
-### BCELoss
-
-Creates a criterion that measures the Binary Cross Entropy 
-
-between the target and the output:
-    loss(o, t) = - 1/n sum_i (t[i] * log(o[i]) + (1 - t[i]) * log(1 - o[i]))
-
-or in the case of the weights argument being specified:
-    loss(o, t) = - 1/n sum_i weights[i] * (t[i] * log(o[i]) + (1 - t[i]) * log(1 - o[i]))
-
-This is used for measuring the error of a reconstruction in for example 
-an auto-encoder. Note that the targets `t[i]` should be numbers between 0 and 1, 
-for instance, the output of an `nn.Sigmoid` layer.
-
-By default, the losses are averaged for each minibatch over observations 
-*as well as* over dimensions. However, if the field `sizeAverage` is set 
-to `False`, the losses are instead summed.
-### MarginRankingLoss
-
-Creates a criterion that measures the loss given  
-
-inputs `x1`, `x2`, two 1D min-batch `Tensor`s, 
-and a label 1D mini-batch tensor `y` with values (`1` or `-1`).
-
-If `y == 1` then it assumed the first input should be ranked higher 
-(have a larger value) than the second input, and vice-versa for `y == -1`.
-
-The loss function for each sample in the mini-batch is:
-
-    loss(x, y) = max(0, -y * (x1 - x2) + margin)
-
-if the internal variable `sizeAverage = True`, 
-the loss function averages the loss over the batch samples; 
-if `sizeAverage = False`, then the loss function sums over the batch samples. 
-By default, `sizeAverage` equals to `True`.
-### HingeEmbeddingLoss
-
-Measures the loss given an input `x` which is a 2D mini-batch tensor
-
-and a labels `y`, a 1D tensor containg values (`1` or `-1`).
-This is usually used for measuring whether two inputs are similar or dissimilar, 
-e.g. using the L1 pairwise distance, and is typically used for learning 
-nonlinear embeddings or semi-supervised learning.
-
-                     { x_i,                  if y_i ==  1
-    loss(x, y) = 1/n {
-                     { max(0, margin - x_i), if y_i == -1
-
-`x` and `y` arbitrary shapes with a total of `n` elements each
-the sum operation still operates over all the elements, and divides by `n`.
-(the division by `n` can be avoided if one sets the internal variable `sizeAverage=False`). 
-The `margin` has a default value of `1`, or can be set in the constructor.
-### MultiLabelMarginLoss
-
-Creates a criterion that optimizes a multi-class multi-classification 
-
-hinge loss (margin-based loss) between input `x`  (a 2D mini-batch `Tensor`) and 
-output `y` (which is a 2D `Tensor` of target class indices).
-For each sample in the mini-batch:
-
-    loss(x, y) = sum_ij(max(0, 1 - (x[y[j]] - x[i]))) / x:size(1)
-
-where `i == 0` to `x.size(0)`, `j == 0` to `y.size(0)`, 
-      `y[j] != 0`, and `i != y[j]` for all `i` and `j`.
-
-`y` and `x` must have the same size.
-The criterion only considers the first non zero `y[j]` targets.
-This allows for different samples to have variable amounts of target classes
-### SmoothL1Loss
-
-Creates a criterion that uses a squared term if the absolute 
-
-element-wise error falls below 1 and an L1 term otherwise. 
-It is less sensitive to outliers than the `MSELoss` and in some cases 
-prevents exploding gradients (e.g. see "Fast R-CNN" paper by Ross Girshick).
-
-                          { 0.5 * (x_i - y_i)^2, if |x_i - y_i| < 1
-    loss(x, y) = 1/n \sum {
-                          { |x_i - y_i| - 0.5,   otherwise
-
-`x` and `y` arbitrary shapes with a total of `n` elements each
-the sum operation still operates over all the elements, and divides by `n`.
-
-The division by `n` can be avoided if one sets the internal variable 
-`sizeAverage` to `False`
-### SoftMarginLoss
-
-Creates a criterion that optimizes a two-class classification 
-
-logistic loss between input `x` (a 2D mini-batch `Tensor`) and 
-target `y` (which is a tensor containing either `1`s or `-1`s).
-
-    loss(x, y) = sum_i (log(1 + exp(-y[i]*x[i]))) / x:nElement()
-
-The normalization by the number of elements in the input can be disabled by
-setting `self.sizeAverage` to `False`.
-### MultiLabelSoftMarginLoss
-
-Creates a criterion that optimizes a multi-label one-versus-all 
-
-loss based on max-entropy, between input `x`  (a 2D mini-batch `Tensor`) and 
-target `y` (a binary 2D `Tensor`). For each sample in the minibatch:
-
-   loss(x, y) = - sum_i (y[i] log( exp(x[i]) / (1 + exp(x[i]))) 
-                         + (1-y[i]) log(1/(1+exp(x[i])))) / x:nElement()
-
-where `i == 0` to `x.nElement()-1`, `y[i]  in {0,1}`.
-`y` and `x` must have the same size.
-### CosineEmbeddingLoss
-
-Creates a criterion that measures the loss given  an input tensors x1, x2 
-
-and a `Tensor` label `y` with values 1 or -1.
-This is used for measuring whether two inputs are similar or dissimilar, 
-using the cosine distance, and is typically used for learning nonlinear 
-embeddings or semi-supervised learning.
-
-`margin` should be a number from `-1` to `1`, `0` to `0.5` is suggested.
-If `margin` is missing, the default value is `0`.
-
-The loss function for each sample is:
-
-                 { 1 - cos(x1, x2),              if y ==  1
-    loss(x, y) = {
-                 { max(0, cos(x1, x2) - margin), if y == -1
-
-If the internal variable `sizeAverage` is equal to `True`, 
-the loss function averages the loss over the batch samples; 
-if `sizeAverage` is `False`, then the loss function sums over the 
-batch samples. By default, `sizeAverage = True`.
-### MultiMarginLoss
-
-Creates a criterion that optimizes a multi-class classification hinge loss 
-
-(margin-based loss) between input `x` (a 2D mini-batch `Tensor`) and 
-output `y` (which is a 1D tensor of target class indices, `0` <= `y` <= `x.size(1)`):
-
-For each mini-batch sample:
-    loss(x, y) = sum_i(max(0, (margin - x[y] + x[i]))^p) / x.size(0)
-                 where `i == 0` to `x.size(0)` and `i != y`.
-
-Optionally, you can give non-equal weighting on the classes by passing 
-a 1D `weights` tensor into the constructor.
-
-The loss function then becomes:
-    loss(x, y) = sum_i(max(0, w[y] * (margin - x[y] - x[i]))^p) / x.size(0)
-
-By default, the losses are averaged over observations for each minibatch. 
-However, if the field `sizeAverage` is set to `False`, 
-the losses are instead summed.

docs/nn_normalization.md

@@ -1,142 +0,0 @@
-## Normalization layers
-### BatchNorm1d
-
-Applies Batch Normalization over a 2d input that is seen as a mini-batch of 1d inputs
-
-```python
-              x - mean(x)
-y =  ----------------------------- * gamma + beta
-      standard_deviation(x) + eps
-```
-
-```python
-# With Learnable Parameters
-m = nn.BatchNorm1d(100)
-# Without Learnable Parameters
-m = nn.BatchNorm1d(100, affine=False)
-input = autograd.Variable(torch.randn(20, 100))
-output = m(input)
-```
-
-
-
-The mean and standard-deviation are calculated per-dimension over
-the mini-batches and gamma and beta are learnable parameter vectors
-of size N (where N is the input size).
-
-During training, this layer keeps a running estimate of its computed mean
-and variance. The running sum is kept with a default momentum of 0.1
-During evaluation, this running mean/variance is used for normalization.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-num_features |  | the size of each 1D input in the mini-batch
-eps | 1e-5 | a value added to the denominator for numerical stability.
-momentum | 0.1 | the value used for the running_mean and running_var computation.
-affine |  | a boolean value that when set to true, gives the layer learnable affine parameters.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , num_features ]  | 2D Tensor of nBatches x num_features
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a normalized tensor in the batch dimension
-### BatchNorm2d
-
-Applies Batch Normalization over a 4d input that is seen as a mini-batch of 3d inputs
-
-```python
-              x - mean(x)
-y =  ----------------------------- * gamma + beta
-      standard_deviation(x) + eps
-```
-
-```python
-# With Learnable Parameters
-m = nn.BatchNorm2d(100)
-# Without Learnable Parameters
-m = nn.BatchNorm2d(100, affine=False)
-input = autograd.Variable(torch.randn(20, 100, 35, 45))
-output = m(input)
-```
-
-
-
-The mean and standard-deviation are calculated per-dimension over
-the mini-batches and gamma and beta are learnable parameter vectors
-of size N (where N is the input size).
-
-During training, this layer keeps a running estimate of its computed mean
-and variance. The running sum is kept with a default momentum of 0.1
-During evaluation, this running mean/variance is used for normalization.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-num_features |  | num_features from an expected input of size batch_size x num_features x height x width
-eps | 1e-5 | a value added to the denominator for numerical stability.
-momentum | 0.1 | the value used for the running_mean and running_var computation.
-affine |  | a boolean value that when set to true, gives the layer learnable affine parameters.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , num_features , *, * ]  | 4D Tensor of batch_size x num_features x height x width
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a normalized tensor in the batch dimension
-### BatchNorm3d
-
-Applies Batch Normalization over a 5d input that is seen as a mini-batch of 4d inputs
-
-```python
-              x - mean(x)
-y =  ----------------------------- * gamma + beta
-      standard_deviation(x) + eps
-```
-
-```python
-# With Learnable Parameters
-m = nn.BatchNorm3d(100)
-# Without Learnable Parameters
-m = nn.BatchNorm3d(100, affine=False)
-input = autograd.Variable(torch.randn(20, 100, 35, 45, 10))
-output = m(input)
-```
-
-
-
-The mean and standard-deviation are calculated per-dimension over
-the mini-batches and gamma and beta are learnable parameter vectors
-of size N (where N is the input size).
-
-During training, this layer keeps a running estimate of its computed mean
-and variance. The running sum is kept with a default momentum of 0.1
-During evaluation, this running mean/variance is used for normalization.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-num_features |  | num_features from an expected input of size batch_size x num_features x height x width
-eps | 1e-5 | a value added to the denominator for numerical stability.
-momentum | 0.1 | the value used for the running_mean and running_var computation.
-affine |  | a boolean value that when set to true, gives the layer learnable affine parameters.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , num_features , * , * , * ]  | 5D Tensor of batch_size x num_features x depth x height x width
-output | Same  | Output has the same shape as input
-
-#### Returns
-    a normalized tensor in the batch dimension

docs/nn_pooling.md

@@ -1,308 +0,0 @@
-## Pooling Layers
-### MaxPool1d
-
-Applies a 1D max pooling over an input signal composed of several input
-
-```python
-The output value of the layer with input (b x C x W) and output (b x C x oW)
-can be precisely described as:
-output[b_i][c_i][w_i] = max_{k=1, K} input[b_i][c_i][stride_w * w_i + k)]
-```
-
-```python
-# pool of size=3, stride=2
-m = nn.MaxPool1d(3, stride=2)
-input = autograd.Variable(torch.randn(20, 16, 50))
-output = m(input)
-```
-
-planes.
-
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-kernel_size |  | the size of the window to take a max over
-stride |  | the stride of the window
-padding | 0 | implicit padding to be added.
-dilation | kernel_size | a parameter that controls the stride of elements in the window.
-return_indices | False | if True, will return the indices along with the outputs. Useful when Unpooling later.
-ceil_mode |  | when True, will use "ceil" instead of "floor" to compute the output shape
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , * , * ]  | Input is minibatch x channels x iW
-output | [ * , * , * ]   | Output shape = minibatch x channels x floor((iW  + 2*padW - kernel_size) / stride + 1)
-### MaxPool2d
-
-Applies a 2D max pooling over an input signal composed of several input
-
-```python
-The output value of the layer with input (b x C x H x W) and output (b x C x oH x oW)
-can be precisely described as:
-output[b_i][c_i][h_i][w_i] = max_{{kh=1, KH}, {kw=1, kW}} input[b_i][c_i][stride_h * h_i + kH)][stride_w * w_i + kW)]
-```
-
-```python
-# pool of square window of size=3, stride=2
-m = nn.MaxPool2d(3, stride=2)
-# pool of non-square window
-m = nn.MaxPool2d((3, 2), stride=(2, 1))
-input = autograd.Variable(torch.randn(20, 16, 50, 32))
-output = m(input)
-```
-
-planes.
-
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-kernel_size |  | the size of the window to take a max over. Can be a single number k (for a square kernel of k x k) or a tuple (kh x kw)
-stride | kernel_size | the stride of the window. Can be a single number s or a tuple (sh x sw).
-padding | 0 | implicit padding to be added. Can be a single number or a tuple.
-dilation | 1 | a parameter that controls the stride of elements in the window. Can be a single number or a tuple.
-return_indices | False | if True, will return the indices along with the outputs. Useful to pass to nn.MaxUnpool2d .
-ceil_mode |  | when True, will use "ceil" instead of "floor" to compute the output shape
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , * , *, * ]  | Input is minibatch x channels x iH x iW
-output | [ * , * , *, * ]   | Output shape = minibatch x channels x floor((iH  + 2*padH - kH) / sH + 1) x floor((iW  + 2*padW - kW) / sW + 1)
-### MaxPool3d
-
-Applies a 3D max pooling over an input signal composed of several input
-
-```python
-# pool of square window of size=3, stride=2
-m = nn.MaxPool3d(3, stride=2)
-# pool of non-square window
-m = nn.MaxPool3d((3, 2, 2), stride=(2, 1, 2))
-input = autograd.Variable(torch.randn(20, 16, 50,44, 31))
-output = m(input)
-```
-
-planes.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-kernel_size |  | the size of the window to take a max over. Can be a single number k (for a square kernel of k x k x k) or a tuple (kt x kh x kw)
-stride | kernel_size | the stride of the window. Can be a single number s or a tuple (st x sh x sw).
-padding | 0 | implicit padding to be added. Can be a single number or a tuple.
-dilation | 1 | a parameter that controls the stride of elements in the window. Can be a single number or a tuple.
-return_indices | False | if True, will return the indices along with the outputs. Useful to pass to nn.MaxUnpool3d .
-ceil_mode |  | when True, will use "ceil" instead of "floor" to compute the output shape
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , * , *, *, * ]  | Input is minibatch x channels x iT x iH x iW
-output | [ * , * , *, *, * ]   | Output shape = minibatch x channels x floor((iT  + 2*padT - kT) / sT + 1) x floor((iH  + 2*padH - kH) / sH + 1) x floor((iW  + 2*padW - kW) / sW + 1)
-### MaxUnpool2d
-
-Computes the inverse operation of MaxPool2d
-
-```python
-# pool of square window of size=3, stride=2
-m = nn.MaxPool2d(2, stride=2, return_indices = True)
-mu = nn.MaxUnpool2d(2, stride=2)
-input = autograd.Variable(torch.randn(20, 16, 50, 32))
-output, indices = m(input)
-unpooled_output = mu.forward(output, indices)
-# exact output size can be also specified as an argument
-input = autograd.Variable(torch.randn(1, 16, 11, 11))
-downsample = nn.MaxPool2d(3, 3, return_indices=True)
-upsample = nn.MaxUnpool2d(3, 3)
-h, indices = downsample(input)
-output = upsample(h, indices, output_size=input.size())
-```
-
-MaxPool2d is not invertible, as the locations of the max locations are lost.
-MaxUnpool2d takes in as input the output of MaxPool2d and the indices of the Max locations
-and computes the inverse.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-kernel_size |  | the size of the max window. Can be a single number k (for a square kernel of k x k) or a tuple (kh x kw)
-stride | kernel_size | the stride of the window. Can be a single number s or a tuple (sh x sw).
-padding | 0 | implicit padding that was added to the input. Can be a single number or a tuple.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , * , *, * ]  | Input is minibatch x channels x iH x iW
-output | [ * , * , *, * ]   | Output shape is minibatch x channels x padH x (iH - 1) * sH + kH x padW x (iW - 1) * sW + kW, or as specified to the call.
-### MaxUnpool3d
-
-Computes the inverse operation of MaxPool3d
-
-```python
-# pool of square window of size=3, stride=2
-m = nn.MaxPool3d(3, stride=2, return_indices = True)
-mu = nn.MaxUnpool3d(3, stride=2)
-input, indices = autograd.Variable(torch.randn(20, 16, 50, 32, 15))
-output = m(input)
-unpooled_output = m2.forward(output, indices)
-```
-
-MaxPool3d is not invertible, as the locations of the max locations are lost.
-MaxUnpool3d takes in as input the output of MaxPool3d and the indices of the Max locations
-and computes the inverse.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-kernel_size |  | the size of the max window. Can be a single number k (for a square kernel of k x k) or a tuple (kt x kh x kw)
-stride | kernel_size | the stride of the window. Can be a single number s or a tuple (st x sh x sw).
-padding | 0 | implicit padding that was added to the input. Can be a single number or a tuple.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , * , *, *, * ]  | Input is minibatch x channels x iT x iH x iW
-output | [ * , * , *, *, * ]   | Output shape = minibatch x channels x padT x (iT - 1) * sT + kT x padH x (iH - 1) * sH + kH x padW x (iW - 1) * sW + kW
-### AvgPool2d
-
-Applies a 2D average pooling over an input signal composed of several input
-
-```python
-The output value of the layer with input (b x C x H x W) and output (b x C x oH x oW)
-can be precisely described as:
-output[b_i][c_i][h_i][w_i] = (1 / K) * sum_{kh=1, KH} sum_{kw=1, kW}  input[b_i][c_i][stride_h * h_i + kh)][stride_w * w_i + kw)]
-```
-
-```python
-# pool of square window of size=3, stride=2
-m = nn.AvgPool2d(3, stride=2)
-# pool of non-square window
-m = nn.AvgPool2d((3, 2), stride=(2, 1))
-input = autograd.Variable(torch.randn(20, 16, 50, 32))
-output = m(input)
-```
-
-planes.
-
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-kernel_size |  | the size of the window. Can be a single number k (for a square kernel of k x k) or a tuple (kh x kw)
-stride | kernel_size | the stride of the window. Can be a single number s or a tuple (sh x sw).
-padding | 0 | implicit padding to be added. Can be a single number or a tuple.
-ceil_mode |  | when True, will use "ceil" instead of "floor" to compute the output shape
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , * , *, * ]  | Input is minibatch x channels x iH x iW
-output | [ * , * , *, * ]   | Output shape = minibatch x channels x floor((iH  + 2*padH - kH) / sH + 1) x floor((iW  + 2*padW - kW) / sW + 1)
-### AvgPool3d
-
-Applies a 3D average pooling over an input signal composed of several input
-
-```python
-# pool of square window of size=3, stride=2
-m = nn.AvgPool3d(3, stride=2)
-# pool of non-square window
-m = nn.AvgPool3d((3, 2, 2), stride=(2, 1, 2))
-input = autograd.Variable(torch.randn(20, 16, 50,44, 31))
-output = m(input)
-```
-
-planes.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-kernel_size |  | the size of the window to take a average over. Can be a single number k (for a square kernel of k x k x k) or a tuple (kt x kh x kw)
-stride | kernel_size | the stride of the window. Can be a single number s or a tuple (st x sh x sw).
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , * , *, *, * ]  | Input is minibatch x channels x iT x iH x iW
-output | [ * , * , *, *, * ]   | Output shape = minibatch x channels x floor((iT  + 2*padT - kT) / sT + 1) x floor((iH  + 2*padH - kH) / sH + 1) x floor((iW  + 2*padW - kW) / sW + 1)
-### FractionalMaxPool2d
-
-Applies a 2D fractional max pooling over an input signal composed of several input
-
-```python
-# pool of square window of size=3, and target output size 13x12
-m = nn.FractionalMaxPool2d(3, output_size=(13, 12))
-# pool of square window and target output size being half of input image size
-m = nn.FractionalMaxPool2d(3, output_ratio=(0.5, 0.5))
-input = autograd.Variable(torch.randn(20, 16, 50, 32))
-output = m(input)
-```
-
-planes.
-
-Fractiona MaxPooling is described in detail in the paper ["Fractional Max-Pooling" by Ben Graham](http://arxiv.org/abs/1412.6071)
-The max-pooling operation is applied in kHxkW regions by a stochastic
-step size determined by the target output size.
-The number of output features is equal to the number of input planes.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-kernel_size |  | the size of the window to take a max over. Can be a single number k (for a square kernel of k x k) or a tuple (kh x kw)
-output_size |  | the target output size of the image of the form oH x oW. Can be a tuple (oH, oW) or a single number oH for a square image oH x oH
-output_ratio |  | If one wants to have an output size as a ratio of the input size, this option can be given. This has to be a number or tuple in the range (0, 1)
-return_indices | False | if True, will return the indices along with the outputs. Useful to pass to nn.MaxUnpool2d .
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , * , *, * ]  | Input is minibatch x channels x iH x iW
-output | [ * , * , *, * ]   | Output shape = minibatch x channels x floor((iH  + 2*padH - kH) / sH + 1) x floor((iW  + 2*padW - kW) / sW + 1)
-### LPPool2d
-
-Applies a 2D power-average pooling over an input signal composed of several input
-
-```python
-# power-2 pool of square window of size=3, stride=2
-m = nn.LPPool2d(2, 3, stride=2)
-# pool of non-square window of power 1.2
-m = nn.LPPool2d(1.2, (3, 2), stride=(2, 1))
-input = autograd.Variable(torch.randn(20, 16, 50, 32))
-output = m(input)
-```
-
-planes.
-On each window, the function computed is: f(X) = pow(sum(pow(X, p)), 1/p)
-At p = infinity, one gets Max Pooling
-At p = 1, one gets Average Pooling
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-kernel_size |  | the size of the window. Can be a single number k (for a square kernel of k x k) or a tuple (kh x kw)
-stride | kernel_size | the stride of the window. Can be a single number s or a tuple (sh x sw).
-ceil_mode |  | when True, will use "ceil" instead of "floor" to compute the output shape
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ * , * , *, * ]  | Input is minibatch x channels x iH x iW
-output | [ * , * , *, * ]   | Output shape = minibatch x channels x floor((iH  + 2*padH - kH) / sH + 1) x floor((iW  + 2*padW - kW) / sW + 1)

docs/nn_recurrent.md

@@ -1,346 +0,0 @@
-## Recurrent layers
-### RNN
-
-Applies a multi-layer Elman RNN with tanh or ReLU non-linearity to an input sequence.
-
-```python
-h_t = tanh(w_ih * x_t + b_ih  +  w_hh * h_(t-1) + b_hh)
-```
-
-```python
-rnn = nn.RNN(10, 20, 2)
-input = Variable(torch.randn(5, 3, 10))
-h0 = Variable(torch.randn(2, 3, 20))
-output, hn = rnn(input, h0)
-```
-
-
-
-For each element in the input sequence, each layer computes the following
-function:
-where `h_t` is the hidden state at time t, and `x_t` is the hidden
-state of the previous layer at time t or `input_t` for the first layer.
-If nonlinearity='relu', then ReLU is used instead of tanh.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-input_size |  | The number of expected features in the input x
-hidden_size |  | The number of features in the hidden state h
-num_layers |  | the size of the convolving kernel.
-nonlinearity | 'tanh' | The non-linearity to use ['tanh'|'relu'].
-bias | True | If False, then the layer does not use bias weights b_ih and b_hh.
-batch_first |  | If True, then the input tensor is provided as (batch, seq, feature)
-dropout |  | If non-zero, introduces a dropout layer on the outputs of each RNN layer
-bidirectional | False | If True, becomes a bidirectional RNN.
-
-#### Inputs
-
-Parameter | Default | Description
---------- | ------- | -----------
-input |  | A (seq_len x batch x input_size) tensor containing the features of the input sequence.
-h_0 |  | A (num_layers x batch x hidden_size) tensor containing the initial hidden state for each element in the batch.
-
-#### Outputs
-
-Parameter |  Description
---------- |  -----------
-output | A (seq_len x batch x hidden_size) tensor containing the output features (h_k) from the last layer of the RNN, for each k
-h_n | A (num_layers x batch x hidden_size) tensor containing the hidden state for k=seq_len
-
-#### Members
-
-Parameter | Description
---------- | -----------
-weight_ih_l[k] | the learnable input-hidden weights of the k-th layer, of shape (input_size x hidden_size)
-weight_hh_l[k] | the learnable hidden-hidden weights of the k-th layer, of shape (hidden_size x hidden_size)
-bias_ih_l[k] | the learnable input-hidden bias of the k-th layer, of shape (hidden_size)
-bias_hh_l[k] | the learnable hidden-hidden bias of the k-th layer, of shape (hidden_size)
-### LSTM
-
-Applies a multi-layer long short-term memory (LSTM) RNN to an input sequence.
-
-```python
-i_t = sigmoid(W_ii x_t + b_ii + W_hi h_(t-1) + b_hi)
-f_t = sigmoid(W_if x_t + b_if + W_hf h_(t-1) + b_hf)
-g_t = tanh(W_ig x_t + b_ig + W_hc h_(t-1) + b_hg)
-o_t = sigmoid(W_io x_t + b_io + W_ho h_(t-1) + b_ho)
-c_t = f_t * c_(t-1) + i_t * c_t
-h_t = o_t * tanh(c_t)
-```
-
-```python
-rnn = nn.LSTM(10, 20, 2)
-input = Variable(torch.randn(5, 3, 10))
-h0 = Variable(torch.randn(2, 3, 20))
-c0 = Variable(torch.randn(2, 3, 20))
-output, hn = rnn(input, (h0, c0))
-```
-
-
-
-For each element in the input sequence, each layer computes the following
-function:
-where `h_t` is the hidden state at time t, `c_t` is the cell state at time t,
-`x_t` is the hidden state of the previous layer at time t or input_t for the first layer,
-and `i_t`, `f_t`, `g_t`, `o_t` are the input, forget, cell, and out gates, respectively.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-input_size |  | The number of expected features in the input x
-hidden_size |  | The number of features in the hidden state h
-num_layers |  | the size of the convolving kernel.
-bias | True | If False, then the layer does not use bias weights b_ih and b_hh.
-batch_first |  | If True, then the input tensor is provided as (batch, seq, feature)
-dropout |  | If non-zero, introduces a dropout layer on the outputs of each RNN layer
-bidirectional | False | If True, becomes a bidirectional RNN.
-
-#### Inputs
-
-Parameter | Default | Description
---------- | ------- | -----------
-input |  | A (seq_len x batch x input_size) tensor containing the features of the input sequence.
-h_0 |  | A (num_layers x batch x hidden_size) tensor containing the initial hidden state for each element in the batch.
-c_0 |  | A (num_layers x batch x hidden_size) tensor containing the initial cell state for each element in the batch.
-
-#### Outputs
-
-Parameter |  Description
---------- |  -----------
-output | A (seq_len x batch x hidden_size) tensor containing the output features (h_t) from the last layer of the RNN, for each t
-h_n | A (num_layers x batch x hidden_size) tensor containing the hidden state for t=seq_len
-c_n | A (num_layers x batch x hidden_size) tensor containing the cell state for t=seq_len
-
-#### Members
-
-Parameter | Description
---------- | -----------
-weight_ih_l[k] | the learnable input-hidden weights of the k-th layer (W_ir|W_ii|W_in), of shape (input_size x 3*hidden_size)
-weight_hh_l[k] | the learnable hidden-hidden weights of the k-th layer (W_hr|W_hi|W_hn), of shape (hidden_size x 3*hidden_size)
-bias_ih_l[k] | the learnable input-hidden bias of the k-th layer (b_ir|b_ii|b_in), of shape (3*hidden_size)
-bias_hh_l[k] | the learnable hidden-hidden bias of the k-th layer (W_hr|W_hi|W_hn), of shape (3*hidden_size)
-### GRU
-
-Applies a multi-layer gated recurrent unit (GRU) RNN to an input sequence.
-
-```python
-r_t = sigmoid(W_ir x_t + b_ir + W_hr h_(t-1) + b_hr)
-i_t = sigmoid(W_ii x_t + b_ii + W_hi h_(t-1) + b_hi)
-n_t = tanh(W_in x_t + resetgate * W_hn h_(t-1))
-h_t = (1 - i_t) * n_t + i_t * h_(t-1)
-```
-
-```python
-rnn = nn.GRU(10, 20, 2)
-input = Variable(torch.randn(5, 3, 10))
-h0 = Variable(torch.randn(2, 3, 20))
-output, hn = rnn(input, h0)
-```
-
-
-
-For each element in the input sequence, each layer computes the following
-function:
-where `h_t` is the hidden state at time t, `x_t` is the hidden
-state of the previous layer at time t or input_t for the first layer,
-and `r_t`, `i_t`, `n_t` are the reset, input, and new gates, respectively.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-input_size |  | The number of expected features in the input x
-hidden_size |  | The number of features in the hidden state h
-num_layers |  | the size of the convolving kernel.
-bias | True | If False, then the layer does not use bias weights b_ih and b_hh.
-batch_first |  | If True, then the input tensor is provided as (batch, seq, feature)
-dropout |  | If non-zero, introduces a dropout layer on the outputs of each RNN layer
-bidirectional | False | If True, becomes a bidirectional RNN.
-
-#### Inputs
-
-Parameter | Default | Description
---------- | ------- | -----------
-input |  | A (seq_len x batch x input_size) tensor containing the features of the input sequence.
-h_0 |  | A (num_layers x batch x hidden_size) tensor containing the initial hidden state for each element in the batch.
-
-#### Outputs
-
-Parameter |  Description
---------- |  -----------
-output | A (seq_len x batch x hidden_size) tensor containing the output features (h_t) from the last layer of the RNN, for each t
-h_n | A (num_layers x batch x hidden_size) tensor containing the hidden state for t=seq_len
-
-#### Members
-
-Parameter | Description
---------- | -----------
-weight_ih_l[k] | the learnable input-hidden weights of the k-th layer (W_ir|W_ii|W_in), of shape (input_size x 3*hidden_size)
-weight_hh_l[k] | the learnable hidden-hidden weights of the k-th layer (W_hr|W_hi|W_hn), of shape (hidden_size x 3*hidden_size)
-bias_ih_l[k] | the learnable input-hidden bias of the k-th layer (b_ir|b_ii|b_in), of shape (3*hidden_size)
-bias_hh_l[k] | the learnable hidden-hidden bias of the k-th layer (W_hr|W_hi|W_hn), of shape (3*hidden_size)
-### RNNCell
-
-An Elman RNN cell with tanh or ReLU non-linearity.
-
-```python
-h' = tanh(w_ih * x + b_ih  +  w_hh * h + b_hh)
-```
-
-```python
-rnn = nn.RNNCell(10, 20)
-input = Variable(torch.randn(6, 3, 10))
-hx = Variable(torch.randn(3, 20))
-output = []
-for i in range(6):
-    hx = rnn(input, hx)
-    output[i] = hx
-```
-
-If nonlinearity='relu', then ReLU is used in place of tanh.
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-input_size |  | The number of expected features in the input x
-hidden_size |  | The number of features in the hidden state h
-bias | True | If False, then the layer does not use bias weights b_ih and b_hh.
-nonlinearity | 'tanh' | The non-linearity to use ['tanh'|'relu'].
-
-#### Inputs
-
-Parameter | Default | Description
---------- | ------- | -----------
-input |  | A (batch x input_size) tensor containing input features
-hidden |  | A (batch x hidden_size) tensor containing the initial hidden state for each element in the batch.
-
-#### Outputs
-
-Parameter |  Description
---------- |  -----------
-h' | A (batch x hidden_size) tensor containing the next hidden state for each element in the batch
-
-#### Members
-
-Parameter | Description
---------- | -----------
-weight_ih | the learnable input-hidden weights, of shape (input_size x hidden_size)
-weight_hh | the learnable hidden-hidden weights, of shape (hidden_size x hidden_size)
-bias_ih | the learnable input-hidden bias, of shape (hidden_size)
-bias_hh | the learnable hidden-hidden bias, of shape (hidden_size)
-### LSTMCell
-
-A long short-term memory (LSTM) cell.
-
-```python
-i = sigmoid(W_ii x + b_ii + W_hi h + b_hi)
-f = sigmoid(W_if x + b_if + W_hf h + b_hf)
-g = tanh(W_ig x + b_ig + W_hc h + b_hg)
-o = sigmoid(W_io x + b_io + W_ho h + b_ho)
-c' = f * c + i * c
-h' = o * tanh(c_t)
-```
-
-```python
-rnn = nn.LSTMCell(10, 20)
-input = Variable(torch.randn(6, 3, 10))
-hx = Variable(torch.randn(3, 20))
-cx = Variable(torch.randn(3, 20))
-output = []
-for i in range(6):
-    hx, cx = rnn(input, (hx, cx))
-    output[i] = hx
-```
-
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-input_size |  | The number of expected features in the input x
-hidden_size |  | The number of features in the hidden state h
-bias | True | If False, then the layer does not use bias weights b_ih and b_hh.
-
-#### Inputs
-
-Parameter | Default | Description
---------- | ------- | -----------
-input |  | A (batch x input_size) tensor containing input features
-hidden |  | A (batch x hidden_size) tensor containing the initial hidden state for each element in the batch.
-
-#### Outputs
-
-Parameter |  Description
---------- |  -----------
-h' | A (batch x hidden_size) tensor containing the next hidden state for each element in the batch
-c' | A (batch x hidden_size) tensor containing the next cell state for each element in the batch
-
-#### Members
-
-Parameter | Description
---------- | -----------
-weight_ih | the learnable input-hidden weights, of shape (input_size x hidden_size)
-weight_hh | the learnable hidden-hidden weights, of shape (hidden_size x hidden_size)
-bias_ih | the learnable input-hidden bias, of shape (hidden_size)
-bias_hh | the learnable hidden-hidden bias, of shape (hidden_size)
-### GRUCell
-
-A gated recurrent unit (GRU) cell
-
-```python
-r = sigmoid(W_ir x + b_ir + W_hr h + b_hr)
-i = sigmoid(W_ii x + b_ii + W_hi h + b_hi)
-n = tanh(W_in x + resetgate * W_hn h)
-h' = (1 - i) * n + i * h
-```
-
-```python
-rnn = nn.RNNCell(10, 20)
-input = Variable(torch.randn(6, 3, 10))
-hx = Variable(torch.randn(3, 20))
-output = []
-for i in range(6):
-    hx = rnn(input, hx)
-    output[i] = hx
-```
-
-
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-input_size |  | The number of expected features in the input x
-hidden_size |  | The number of features in the hidden state h
-bias | True | If False, then the layer does not use bias weights b_ih and b_hh.
-
-#### Inputs
-
-Parameter | Default | Description
---------- | ------- | -----------
-input |  | A (batch x input_size) tensor containing input features
-hidden |  | A (batch x hidden_size) tensor containing the initial hidden state for each element in the batch.
-
-#### Outputs
-
-Parameter |  Description
---------- |  -----------
-h' | A (batch x hidden_size) tensor containing the next hidden state for each element in the batch
-
-#### Members
-
-Parameter | Description
---------- | -----------
-weight_ih | the learnable input-hidden weights, of shape (input_size x hidden_size)
-weight_hh | the learnable hidden-hidden weights, of shape (hidden_size x hidden_size)
-bias_ih | the learnable input-hidden bias, of shape (hidden_size)
-bias_hh | the learnable hidden-hidden bias, of shape (hidden_size)

docs/nn_sparse.md

@@ -1,37 +0,0 @@
-## Sparse layers
-### Embedding
-
-A simple lookup table that stores embeddings of a fixed dictionary and size
-
-```python
-# an Embedding module containing 10 tensors of size 3
-embedding = nn.Embedding(10, 3)
-# a batch of 2 samples of 4 indices each
-input = torch.LongTensor([[1,2,4,5],[4,3,2,9]])
-print(embedding(input))
-# example with padding_idx
-embedding = nn.Embedding(10, 3, padding_idx=0)
-input = torch.LongTensor([[0,2,0,5]])
-print(embedding(input))
-```
-
-This module is often used to store word embeddings and retrieve them using indices.
-The input to the module is a list of indices, and the output is the corresponding
-word embeddings.
-
-#### Constructor Arguments
-
-Parameter | Default | Description
---------- | ------- | -----------
-num_embeddings |  | size of the dictionary of embeddings
-embedding_dim |  | the size of each embedding vector
-padding_idx | None | If given, pads the output with zeros whenever it encounters the index.
-max_norm | None | If given, will renormalize the embeddings to always have a norm lesser than this
-norm_type |  | The p of the p-norm to compute for the max_norm option
-scale_grad_by_freq |  | if given, this will scale gradients by the frequency of the words in the dictionary.
-
-#### Expected Shape
-       | Shape | Description 
------- | ----- | ------------
- input | [ *, * ]  | Input is a 2D mini_batch LongTensor of m x n indices to extract from the Embedding dictionary
-output | [ * , *, * ]   | Output shape = m x n x embedding_dim

docs/optim.md

@@ -1,114 +0,0 @@
-# torch.optim
-
-The Optim package in Torch is targeted for one to optimize their neural networks
-using a wide variety of optimization methods such as SGD, Adam etc.
-
-Currently, the following optimization methods are supported, typically with
-options such as weight decay and other bells and whistles.
-
-- SGD          `(params, lr=required, momentum=0, dampening=0)`
-- AdaDelta     `(params, rho=0.9, eps=1e-6, weight_decay=0)`
-- Adagrad      `(params, lr=1e-2, lr_decay=0, weight_decay=0)`
-- Adam         `(params, lr=1e-2, betas=(0.9, 0.999), epsilon=1e-8, weight_decay=0)`
-- AdaMax       `(params, lr=1e-2, betas=(0.9, 0.999), eps=1e-38, weight_decay=0)`
-- Averaged SGD `(params, lr=1e-2, lambd=1e-4, alpha=0.75, t0=1e6, weight_decay=0)`
-- RProp        `(params, lr=1e-2, etas=(0.5, 1.2), step_sizes=(1e-6, 50))`
-- RMSProp      `(params, lr=1e-2, alpha=0.99, eps=1e-8, weight_decay=0)`
-
-
-The usage of the Optim package itself is as follows.
-
-1. Construct an optimizer
-2. Use `optimizer.step(...)` to optimize.
-   - Call `optimizer.zero_grad()` to zero out the gradient buffers when appropriate
-
-## 1. Constructing the optimizer
-
-One first constructs an `Optimizer` object by giving it a list of parameters
-to optimize, as well as the optimizer options,such as learning rate, weight decay, etc.
-
-Examples:
-
-`optimizer = optim.SGD(model.parameters(), lr = 0.01, momentum=0.9)`
-
-`optimizer = optim.Adam([var1, var2], lr = 0.0001)`
-
-### Per-parameter options
-
-In a more advanced usage, one can specify per-layer options by passing each parameter group along with it's custom options.
-
-**__Any parameter group that does not have an attribute defined will use the default attributes.__**
-
-This is very useful when one wants to specify per-layer learning rates for example.
-
-Example:
-
-`optim.SGD([{'params': model1.parameters()}, {'params': model2.parameters(), 'lr': 1e-3}, lr=1e-2, momentum=0.9)`
-
-`model1`'s parameters will use the default learning rate of `1e-2` and momentum of `0.9`
-`model2`'s parameters will use a learning rate of `1e-3`, and the default momentum of `0.9`
-
-Then, you can use the optimizer by calling `optimizer.zero_grad()` and `optimizer.step(...)`. Read the next sections.
-
-## 2. Taking an optimization step using `Optimizer.step(...)`
-
-The step function has the following two signatures:
-
-### a. `Optimizer.step(closure)`
-
-The `step` function takes a user-defined closure that computes f(x) and returns the loss.
-
-The closure needs to do the following:
-- Optimizer.zero_grad()
-- Compute the loss
-- Call loss.backward()
-- return the loss
-
-Example 1: training a neural network
-
-```python
-# Example 1: training a neural network with optimizer.step(closure)
-net = MNISTNet()
-criterion = ClassNLLLoss()
-optimizer = optim.SGD(net.parameters(), lr=0.001)
-
-for data in data_batches:
-    input, target = data
-	def closure():
-	    optimizer.zero_grad()
-	    output = net(input)
-		loss = criterion(output, target)
-		loss.backward()
-		return loss
-	optimizer.step(closure)
-```
-
-Notes: Why is this required? Why cant we simply have the optimizer take the parameters and grads?
-       Some optimization algorithms such as Conjugate Gradient and LBFGS need to evaluate their function
-	   multiple times. For such optimization methods, the function (i.e. the closure) has to be defined.
-      
-
-### b. `Optimizer.step()`
-
-This is a simplified usage that supports most, but not all optimization algorithms. For example, it does not support LBFGS or Conjugate Gradient.
-
-The usage for this is to simply call the function after the backward() is called on your model.
-
-Example 2: training a neural network
-
-```python
-# Example 2: training a neural network with optimizer.step()
-net = MNISTNet()
-criterion = ClassNLLLoss()
-optimizer = optim.SGD(net.parameters(), lr=0.001)
-
-for data in data_batches:
-    input, target = data
-	optimizer.zero_grad()
-	output = net(input)
-	loss = criterion(output, target)
-	loss.backward()
-	optimizer.step()
-```
-
-

docs/requirements.txt

@@ -0,0 +1,2 @@
+sphinx
+-e git://github.com/snide/sphinx_rtd_theme.git#egg=sphinx_rtd_theme

docs/source/_static/css/pytorch_theme.css

@@ -0,0 +1,118 @@
+body {
+    font-family: "Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;
+}
+
+/* Default header fonts are ugly */
+h1, h2, .rst-content .toctree-wrapper p.caption, h3, h4, h5, h6, legend, p.caption {
+    font-family: "Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;
+}
+
+/* Use white for docs background */
+.wy-side-nav-search {
+    background-color: #fff;
+}
+
+.wy-nav-content-wrap, .wy-menu li.current > a  {
+    background-color: #fff;
+}
+
+@media screen and (min-width: 1400px) {
+    .wy-nav-content-wrap {
+        background-color: rgba(0, 0, 0, 0.0470588);
+    }
+
+    .wy-nav-content {
+        background-color: #fff;
+    }
+}
+
+/* Fixes for mobile */
+.wy-nav-top {
+    background-color: #fff;
+    background-image: url('../img/pytorch-logo-dark.svg');
+    background-repeat: no-repeat;
+    background-position: center;
+    padding: 0;
+    margin: 0.4045em 0.809em;
+    color: #333;
+}
+
+.wy-nav-top > a {
+    display: none;
+}
+
+@media screen and (max-width: 768px) {
+    .wy-side-nav-search>a img.logo {
+        height: 60px;
+    }
+}
+
+/* This is needed to ensure that logo above search scales properly */
+.wy-side-nav-search a {
+    display: block;
+}
+
+/* This ensures that multiple constructors will remain in separate lines. */
+.rst-content dl:not(.docutils) dt {
+    display: table;
+}
+
+/* Use our red for literals (it's very similar to the original color) */
+.rst-content tt.literal, .rst-content tt.literal, .rst-content code.literal {
+    color: #F05732;
+}
+
+.rst-content tt.xref, a .rst-content tt, .rst-content tt.xref,
+.rst-content code.xref, a .rst-content tt, a .rst-content code {
+    color: #404040;
+}
+
+/* Change link colors (except for the menu) */
+
+a {
+    color: #F05732;
+}
+
+a:hover {
+    color: #F05732;
+}
+
+
+a:visited {
+    color: #D44D2C;
+}
+
+.wy-menu a {
+    color: #b3b3b3;
+}
+
+.wy-menu a:hover {
+    color: #b3b3b3;
+}
+
+/* Default footer text is quite big */
+footer {
+    font-size: 80%;
+}
+
+footer .rst-footer-buttons {
+    font-size: 125%; /* revert footer settings - 1/80% = 125% */
+}
+
+footer p {
+    font-size: 100%;
+}
+
+/* For hidden headers that appear in TOC tree */
+/* see http://stackoverflow.com/a/32363545/3343043 */
+.rst-content .hidden-section {
+    display: none;
+}
+
+nav .hidden-section {
+    display: inherit;
+}
+
+.wy-side-nav-search>div.version {
+    color: #000;
+}

docs/source/_static/img/pytorch-logo-dark.svg

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 21.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 199.7 40.2" style="enable-background:new 0 0 199.7 40.2;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#F05732;}
+	.st1{fill:#9E529F;}
+	.st2{fill:#333333;}
+</style>
+<path class="st0" d="M102.7,12.2c-1.3-1-1.8,3.9-4.4,3.9c-3,0-4-13-6.3-13c-0.7,0-0.8-0.4-7.9,21.3c-2.9,9,4.4,15.8,11.8,15.8
+	c4.6,0,12.3-3,12.3-12.6C108.2,20.5,104.7,13.7,102.7,12.2z M95.8,35.3c-3.7,0-6.7-3.1-6.7-7c0-3.9,3-7,6.7-7s6.7,3.1,6.7,7
+	C102.5,32.1,99.5,35.3,95.8,35.3z"/>
+<path class="st1" d="M99.8,0c-0.5,0-1.8,2.5-1.8,3.6c0,1.5,1,2,1.8,2c0.8,0,1.8-0.5,1.8-2C101.5,2.5,100.2,0,99.8,0z"/>
+<path class="st2" d="M0,39.5V14.9h11.5c5.3,0,8.3,3.6,8.3,7.9c0,4.3-3,7.9-8.3,7.9H5.2v8.8H0z M14.4,22.8c0-2.1-1.6-3.3-3.7-3.3H5.2
+	v6.6h5.5C12.8,26.1,14.4,24.8,14.4,22.8z"/>
+<path class="st2" d="M35.2,39.5V29.4l-9.4-14.5h6l6.1,9.8l6.1-9.8h5.9l-9.4,14.5v10.1H35.2z"/>
+<path class="st2" d="M63.3,39.5v-20h-7.2v-4.6h19.6v4.6h-7.2v20H63.3z"/>
+<path class="st2" d="M131.4,39.5l-4.8-8.7h-3.8v8.7h-5.2V14.9H129c5.1,0,8.3,3.4,8.3,7.9c0,4.3-2.8,6.7-5.4,7.3l5.6,9.4H131.4z
+	 M131.9,22.8c0-2-1.6-3.3-3.7-3.3h-5.5v6.6h5.5C130.3,26.1,131.9,24.9,131.9,22.8z"/>
+<path class="st2" d="M145.6,27.2c0-7.6,5.7-12.7,13.1-12.7c5.4,0,8.5,2.9,10.3,6l-4.5,2.2c-1-2-3.2-3.6-5.8-3.6
+	c-4.5,0-7.7,3.4-7.7,8.1c0,4.6,3.2,8.1,7.7,8.1c2.5,0,4.7-1.6,5.8-3.6l4.5,2.2c-1.7,3.1-4.9,6-10.3,6
+	C151.3,39.9,145.6,34.7,145.6,27.2z"/>
+<path class="st2" d="M194.5,39.5V29.1h-11.6v10.4h-5.2V14.9h5.2v9.7h11.6v-9.7h5.3v24.6H194.5z"/>
+</svg>

docs/source/_static/img/pytorch-logo-flame.svg

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   height="40.200001"
+   width="40.200001"
+   xml:space="preserve"
+   viewBox="0 0 40.200002 40.2"
+   y="0px"
+   x="0px"
+   id="Layer_1"
+   version="1.1"><metadata
+     id="metadata4717"><rdf:RDF><cc:Work
+         rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
+     id="defs4715" /><style
+     id="style4694"
+     type="text/css">
+	.st0{fill:#F05732;}
+	.st1{fill:#9E529F;}
+	.st2{fill:#333333;}
+</style><path
+     style="fill:#f05732"
+     id="path4696"
+     d="m 26.975479,12.199999 c -1.3,-1 -1.8,3.9 -4.4,3.9 -3,0 -4,-12.9999998 -6.3,-12.9999998 -0.7,0 -0.8,-0.4 -7.9000003,21.2999998 -2.9000001,9 4.4000003,15.8 11.8000003,15.8 4.6,0 12.3,-3 12.3,-12.6 0,-7.1 -3.5,-13.9 -5.5,-15.4 z m -6.9,23.1 c -3.7,0 -6.7,-3.1 -6.7,-7 0,-3.9 3,-7 6.7,-7 3.7,0 6.7,3.1 6.7,7 0,3.8 -3,7 -6.7,7 z"
+     class="st0" /><path
+     style="fill:#9e529f"
+     id="path4698"
+     d="m 24.075479,-7.6293945e-7 c -0.5,0 -1.8,2.49999996293945 -1.8,3.59999996293945 0,1.5 1,2 1.8,2 0.8,0 1.8,-0.5 1.8,-2 -0.1,-1.1 -1.4,-3.59999996293945 -1.8,-3.59999996293945 z"
+     class="st1" /></svg>

docs/source/_templates/layout.html

@@ -0,0 +1,15 @@
+{% extends "!layout.html" %}
+
+{% block footer %}
+{{ super() }}
+<script>
+  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
+
+  ga('create', 'UA-90545585-1', 'auto');
+  ga('send', 'pageview');
+
+</script>
+{% endblock %}

docs/source/autograd.rst

@@ -0,0 +1,71 @@
+.. role:: hidden
+    :class: hidden-section
+
+Automatic differentiation package - torch.autograd
+==================================================
+
+.. automodule:: torch.autograd
+.. currentmodule:: torch.autograd
+
+.. autofunction:: backward
+
+.. autofunction:: grad
+
+Variable
+--------
+
+API compatibility
+^^^^^^^^^^^^^^^^^
+
+Variable API is nearly the same as regular Tensor API (with the exception
+of a couple in-place methods, that would overwrite inputs required for
+gradient computation). In most cases Tensors can be safely replaced with
+Variables and the code will remain to work just fine. Because of this,
+we're not documenting all the operations on variables, and you should
+refer to :class:`torch.Tensor` docs for this purpose.
+
+In-place operations on Variables
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Supporting in-place operations in autograd is a hard matter, and we discourage
+their use in most cases. Autograd's aggressive buffer freeing and reuse makes
+it very efficient and there are very few occasions when in-place operations
+actually lower memory usage by any significant amount. Unless you're operating
+under heavy memory pressure, you might never need to use them.
+
+In-place correctness checks
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+All :class:`Variable` s keep track of in-place operations applied to them, and
+if the implementation detects that a variable was saved for backward in one of
+the functions, but it was modified in-place afterwards, an error will be raised
+once backward pass is started. This ensures that if you're using in-place
+functions and not seeing any errors, you can be sure that the computed
+gradients are correct.
+
+
+.. autoclass:: Variable
+    :members:
+
+:hidden:`Function`
+------------------
+
+.. autoclass:: Function
+    :members:
+
+Profiler
+--------
+
+Autograd includes a profiler that lets you inspect the cost of different
+operators inside your model - both on the CPU and GPU. There are two modes
+implemented at the moment - CPU-only using :class:`~torch.autograd.profiler.profile`.
+and nvprof based (registers both CPU and GPU activity) using
+:class:`~torch.autograd.profiler.emit_nvtx`.
+
+.. autoclass:: torch.autograd.profiler.profile
+    :members:
+
+.. autoclass:: torch.autograd.profiler.emit_nvtx
+    :members:
+
+.. autofunction:: torch.autograd.profiler.load_nvprof

docs/source/conf.py

@@ -0,0 +1,249 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# PyTorch documentation build configuration file, created by
+# sphinx-quickstart on Fri Dec 23 13:31:47 2016.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+import torch
+try:
+    import torchvision
+except ImportError:
+    import warnings
+    warnings.warn('unable to load "torchvision" package')
+import sphinx_rtd_theme
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.autosummary',
+    'sphinx.ext.doctest',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'sphinx.ext.coverage',
+    'sphinx.ext.mathjax',
+    'sphinx.ext.napoleon',
+    'sphinx.ext.viewcode',
+]
+
+napoleon_use_ivar = True
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'PyTorch'
+copyright = '2017, Torch Contributors'
+author = 'Torch Contributors'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+# TODO: change to [:2] at v1.0
+version = 'master (' + torch.__version__ + ' )'
+# The full version, including alpha/beta/rc tags.
+# TODO: verify this works as expected
+release = 'master'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+html_theme_options = {
+    'collapse_navigation': False,
+    'display_version': True,
+    'logo_only': True,
+}
+
+html_logo = '_static/img/pytorch-logo-dark.svg'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# html_style_path = 'css/pytorch_theme.css'
+html_context = {
+    'css_files': [
+        'https://fonts.googleapis.com/css?family=Lato',
+        '_static/css/pytorch_theme.css'
+    ],
+}
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'PyTorchdoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'pytorch.tex', 'PyTorch Documentation',
+     'Torch Contributors', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'PyTorch', 'PyTorch Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'PyTorch', 'PyTorch Documentation',
+     author, 'PyTorch', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/', None),
+    'numpy': ('http://docs.scipy.org/doc/numpy/', None),
+}
+
+# -- A patch that prevents Sphinx from cross-referencing ivar tags -------
+# See http://stackoverflow.com/a/41184353/3343043
+
+from docutils import nodes
+from sphinx.util.docfields import TypedField
+from sphinx import addnodes
+
+
+def patched_make_field(self, types, domain, items, **kw):
+    # `kw` catches `env=None` needed for newer sphinx while maintaining
+    #  backwards compatibility when passed along further down!
+
+    # type: (List, unicode, Tuple) -> nodes.field
+    def handle_item(fieldarg, content):
+        par = nodes.paragraph()
+        par += addnodes.literal_strong('', fieldarg)  # Patch: this line added
+        # par.extend(self.make_xrefs(self.rolename, domain, fieldarg,
+        #                           addnodes.literal_strong))
+        if fieldarg in types:
+            par += nodes.Text(' (')
+            # NOTE: using .pop() here to prevent a single type node to be
+            # inserted twice into the doctree, which leads to
+            # inconsistencies later when references are resolved
+            fieldtype = types.pop(fieldarg)
+            if len(fieldtype) == 1 and isinstance(fieldtype[0], nodes.Text):
+                typename = u''.join(n.astext() for n in fieldtype)
+                typename = typename.replace('int', 'python:int')
+                typename = typename.replace('long', 'python:long')
+                typename = typename.replace('float', 'python:float')
+                typename = typename.replace('type', 'python:type')
+                par.extend(self.make_xrefs(self.typerolename, domain, typename,
+                                           addnodes.literal_emphasis, **kw))
+            else:
+                par += fieldtype
+            par += nodes.Text(')')
+        par += nodes.Text(' -- ')
+        par += content
+        return par
+
+    fieldname = nodes.field_name('', self.label)
+    if len(items) == 1 and self.can_collapse:
+        fieldarg, content = items[0]
+        bodynode = handle_item(fieldarg, content)
+    else:
+        bodynode = self.list_type()
+        for fieldarg, content in items:
+            bodynode += nodes.list_item('', handle_item(fieldarg, content))
+    fieldbody = nodes.field_body('', bodynode)
+    return nodes.field('', fieldname, fieldbody)
+
+TypedField.make_field = patched_make_field

docs/source/cuda.rst

@@ -0,0 +1,49 @@
+torch.cuda
+===================================
+
+.. currentmodule:: torch.cuda
+
+.. automodule:: torch.cuda
+   :members:
+
+Random Number Generator
+-------------------------
+.. autofunction:: get_rng_state
+.. autofunction:: set_rng_state
+.. autofunction:: manual_seed
+.. autofunction:: manual_seed_all
+.. autofunction:: seed
+.. autofunction:: seed_all
+.. autofunction:: initial_seed
+
+
+Communication collectives
+-------------------------
+
+.. autofunction:: torch.cuda.comm.broadcast
+
+.. autofunction:: torch.cuda.comm.reduce_add
+
+.. autofunction:: torch.cuda.comm.scatter
+
+.. autofunction:: torch.cuda.comm.gather
+
+Streams and events
+------------------
+
+.. autoclass:: Stream
+   :members:
+
+.. autoclass:: Event
+   :members:
+
+Memory management
+-----------------
+.. autofunction:: empty_cache
+
+NVIDIA Tools Extension (NVTX)
+-----------------------------
+
+.. autofunction:: torch.cuda.nvtx.mark
+.. autofunction:: torch.cuda.nvtx.range_push
+.. autofunction:: torch.cuda.nvtx.range_pop

docs/source/data.rst

@@ -0,0 +1,14 @@
+torch.utils.data
+===================================
+
+.. automodule:: torch.utils.data
+.. autoclass:: Dataset
+.. autoclass:: TensorDataset
+.. autoclass:: ConcatDataset
+.. autoclass:: DataLoader
+.. autoclass:: torch.utils.data.sampler.Sampler
+.. autoclass:: torch.utils.data.sampler.SequentialSampler
+.. autoclass:: torch.utils.data.sampler.RandomSampler
+.. autoclass:: torch.utils.data.sampler.SubsetRandomSampler
+.. autoclass:: torch.utils.data.sampler.WeightedRandomSampler
+.. autoclass:: torch.utils.data.distributed.DistributedSampler

docs/source/distributed.rst

@@ -0,0 +1,201 @@
+.. role:: hidden
+    :class: hidden-section
+
+Distributed communication package - torch.distributed
+=====================================================
+
+.. automodule:: torch.distributed
+.. currentmodule:: torch.distributed
+
+Currently torch.distributed supports three backends, each with
+different capabilities. The table below shows which functions are available
+for use with CPU / CUDA tensors.
+MPI supports cuda only if the implementation used to build PyTorch supports it.
+
+
++------------+-----------+-----------+-----------+
+| Backend    | ``tcp``   | ``gloo``  | ``mpi``   |
++------------+-----+-----+-----+-----+-----+-----+
+| Device     | CPU | GPU | CPU | GPU | CPU | GPU |
++============+=====+=====+=====+=====+=====+=====+
+| send       | ✓   | ✘   | ✘   | ✘   | ✓   | ?   |
++------------+-----+-----+-----+-----+-----+-----+
+| recv       | ✓   | ✘   | ✘   | ✘   | ✓   | ?   |
++------------+-----+-----+-----+-----+-----+-----+
+| broadcast  | ✓   | ✘   | ✓   | ✓   | ✓   | ?   |
++------------+-----+-----+-----+-----+-----+-----+
+| all_reduce | ✓   | ✘   | ✓   | ✓   | ✓   | ?   |
++------------+-----+-----+-----+-----+-----+-----+
+| reduce     | ✓   | ✘   | ✘   | ✘   | ✓   | ?   |
++------------+-----+-----+-----+-----+-----+-----+
+| all_gather | ✓   | ✘   | ✘   | ✘   | ✓   | ?   |
++------------+-----+-----+-----+-----+-----+-----+
+| gather     | ✓   | ✘   | ✘   | ✘   | ✓   | ?   |
++------------+-----+-----+-----+-----+-----+-----+
+| scatter    | ✓   | ✘   | ✘   | ✘   | ✓   | ?   |
++------------+-----+-----+-----+-----+-----+-----+
+| barrier    | ✓   | ✘   | ✓   | ✓   | ✓   | ?   |
++------------+-----+-----+-----+-----+-----+-----+
+
+.. _distributed-basics:
+
+Basics
+------
+
+The `torch.distributed` package provides PyTorch support and communication primitives
+for multiprocess parallelism across several computation nodes running on one or more
+machines. The class :func:`torch.nn.parallel.DistributedDataParallel` builds on this
+functionality to provide synchronous distributed training as a wrapper around any
+PyTorch model. This differs from the kinds of parallelism provided by
+:doc:`multiprocessing` and :func:`torch.nn.DataParallel` in that it supports
+multiple network-connected machines and in that the user must explicitly launch a separate
+copy of the main training script for each process.
+
+In the single-machine synchronous case, `torch.distributed` or the
+:func:`torch.nn.parallel.DistributedDataParallel` wrapper may still have advantages over other
+approaches to data-parallelism, including :func:`torch.nn.DataParallel`:
+
+* Each process maintains its own optimizer and performs a complete optimization step with each
+  iteration. While this may appear redundant, since the gradients have already been gathered
+  together and averaged across processes and are thus the same for every process, this means
+  that no parameter broadcast step is needed, reducing time spent transferring tensors between
+  nodes.
+* Each process contains an independent Python interpreter, eliminating the extra interpreter
+  overhead and "GIL-thrashing" that comes from driving several execution threads, model
+  replicas, or GPUs from a single Python process. This is especially important for models that
+  make heavy use of the Python runtime, including models with recurrent layers or many small
+  components.
+
+Initialization
+--------------
+
+The package needs to be initialized using the :func:`torch.distributed.init_process_group`
+function before calling any other methods. This blocks until all processes have
+joined.
+
+.. autofunction:: init_process_group
+
+.. autofunction:: get_rank
+
+.. autofunction:: get_world_size
+
+--------------------------------------------------------------------------------
+
+Currently three initialization methods are supported:
+
+TCP initialization
+^^^^^^^^^^^^^^^^^^
+
+There are two ways to initialize using TCP, both requiring a network address
+reachable from all processes and a desired ``world_size``. The first way
+requires specifying an address that belongs to the rank 0 process. This first way of
+initialization requires that all processes have manually specified ranks.
+
+Alternatively, the address has to be a valid IP multicast address, in which case
+ranks can be assigned automatically. Multicast initialization also supports
+a ``group_name`` argument, which allows you to use the same address for multiple
+jobs, as long as they use different group names.
+
+::
+
+    import torch.distributed as dist
+
+    # Use address of one of the machines
+    dist.init_process_group(init_method='tcp://10.1.1.20:23456', rank=args.rank, world_size=4)
+
+    # or a multicast address - rank will be assigned automatically if unspecified
+    dist.init_process_group(init_method='tcp://[ff15:1e18:5d4c:4cf0:d02d:b659:53ba:b0a7]:23456',
+                            world_size=4)
+
+Shared file-system initialization
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Another initialization method makes use of a file system that is shared and
+visible from all machines in a group, along with a desired ``world_size``. The URL should start
+with ``file://`` and contain a path to a non-existent file (in an existing
+directory) on a shared file system. This initialization method also supports a
+``group_name`` argument, which allows you to use the same shared file path for
+multiple jobs, as long as they use different group names.
+
+.. warning::
+    This method assumes that the file system supports locking using ``fcntl`` - most
+    local systems and NFS support it.
+
+::
+
+    import torch.distributed as dist
+
+    # Rank will be assigned automatically if unspecified
+    dist.init_process_group(init_method='file:///mnt/nfs/sharedfile', world_size=4,
+                            group_name=args.group)
+
+Environment variable initialization
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This method will read the configuration from environment variables, allowing
+one to fully customize how the information is obtained. The variables to be set
+are:
+
+* ``MASTER_PORT`` - required; has to be a free port on machine with rank 0
+* ``MASTER_ADDR`` - required (except for rank 0); address of rank 0 node
+* ``WORLD_SIZE`` - required; can be set either here, or in a call to init function
+* ``RANK`` - required; can be set either here, or in a call to init function
+
+The machine with rank 0 will be used to set up all connections.
+
+This is the default method, meaning that ``init_method`` does not have to be specified (or
+can be ``env://``).
+
+Groups
+------
+
+By default collectives operate on the default group (also called the world) and
+require all processes to enter the distributed function call. However, some workloads can benefit
+from more fine-grained communication. This is where distributed groups come
+into play. :func:`~torch.distributed.new_group` function can be
+used to create new groups, with arbitrary subsets of all processes. It returns
+an opaque group handle that can be given as a ``group`` argument to all collectives
+(collectives are distributed functions to exchange information in certain well-known programming patterns).
+
+.. autofunction:: new_group
+
+Point-to-point communication
+----------------------------
+
+.. autofunction:: send
+
+.. autofunction:: recv
+
+:func:`~torch.distributed.isend` and :func:`~torch.distributed.irecv`
+return distributed request objects when used. In general, the type of this object is unspecified
+as they should never be created manually, but they are guaranteed to support two methods:
+
+* ``is_completed()`` - returns True if the operation has finished
+* ``wait()`` - will block the process until the operation is finished.
+  ``is_completed()`` is guaranteed to return True once it returns.
+  
+When using the MPI backend, :func:`~torch.distributed.isend` and :func:`~torch.distributed.irecv`
+support non-overtaking, which has some guarantees on supporting message order. For more detail, see
+http://mpi-forum.org/docs/mpi-2.2/mpi22-report/node54.htm#Node54
+
+.. autofunction:: isend
+
+.. autofunction:: irecv
+
+Collective functions
+--------------------
+
+.. autofunction:: broadcast
+
+.. autofunction:: all_reduce
+
+.. autofunction:: reduce
+
+.. autofunction:: all_gather
+
+.. autofunction:: gather
+
+.. autofunction:: scatter
+
+.. autofunction:: barrier
+

docs/source/distributions.rst

@@ -0,0 +1,32 @@
+.. role:: hidden
+    :class: hidden-section
+
+Probability distributions - torch.distributions
+==================================================
+
+.. automodule:: torch.distributions
+.. currentmodule:: torch.distributions
+
+:hidden:`Distribution`
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: Distribution
+    :members:
+
+:hidden:`Bernoulli`
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: Bernoulli
+    :members:
+
+:hidden:`Categorical`
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: Categorical
+    :members:
+
+:hidden:`Normal`
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: Normal
+    :members:

docs/source/ffi.rst

@@ -0,0 +1,6 @@
+torch.utils.ffi
+===============
+
+.. currentmodule:: torch.utils.ffi
+.. autofunction:: create_extension
+

docs/source/index.rst

@@ -0,0 +1,54 @@
+.. PyTorch documentation master file, created by
+   sphinx-quickstart on Fri Dec 23 13:31:47 2016.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+:github_url: https://github.com/pytorch/pytorch
+
+PyTorch documentation
+===================================
+
+PyTorch is an optimized tensor library for deep learning using GPUs and CPUs.
+
+.. toctree::
+   :glob:
+   :maxdepth: 1
+   :caption: Notes
+
+   notes/*
+
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Package Reference
+
+   torch
+   tensors
+   sparse
+   storage
+   nn
+   optim
+   torch.autograd <autograd>
+   torch.distributions <distributions>
+   torch.multiprocessing <multiprocessing>
+   torch.distributed <distributed>
+   torch.legacy <legacy>
+   cuda
+   ffi
+   data
+   model_zoo
+   onnx
+
+.. toctree::
+   :glob:
+   :maxdepth: 2
+   :caption: torchvision Reference
+
+   torchvision/index
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`

docs/source/legacy.rst

@@ -0,0 +1,4 @@
+Legacy package - torch.legacy
+===================================
+
+.. automodule:: torch.legacy

docs/source/model_zoo.rst

@@ -0,0 +1,5 @@
+torch.utils.model_zoo
+===================================
+
+.. automodule:: torch.utils.model_zoo
+.. autofunction:: load_url

docs/source/multiprocessing.rst

@@ -0,0 +1,88 @@
+Multiprocessing package - torch.multiprocessing
+===============================================
+
+.. automodule:: torch.multiprocessing
+.. currentmodule:: torch.multiprocessing
+
+.. warning::
+
+    If the main process exits abruptly (e.g. because of an incoming signal),
+    Python's ``multiprocessing`` sometimes fails to clean up its children.
+    It's a known caveat, so if you're seeing any resource leaks after
+    interrupting the interpreter, it probably means that this has just happened
+    to you.
+
+Strategy management
+-------------------
+
+.. autofunction:: get_all_sharing_strategies
+.. autofunction:: get_sharing_strategy
+.. autofunction:: set_sharing_strategy
+
+Sharing CUDA tensors
+--------------------
+
+Sharing CUDA tensors between processes is supported only in Python 3, using
+a ``spawn`` or ``forkserver`` start methods. :mod:`python:multiprocessing` in
+Python 2 can only create subprocesses using ``fork``, and it's not supported
+by the CUDA runtime.
+
+.. warning::
+
+    CUDA API requires that the allocation exported to other processes remains
+    valid as long as it's used by them. You should be careful and ensure that
+    CUDA tensors you shared don't go out of scope as long as it's necessary.
+    This shouldn't be a problem for sharing model parameters, but passing other
+    kinds of data should be done with care. Note that this restriction doesn't
+    apply to shared CPU memory.
+
+
+Sharing strategies
+------------------
+
+This section provides a brief overview into how different sharing strategies
+work. Note that it applies only to CPU tensor - CUDA tensors will always use
+the CUDA API, as that's the only way they can be shared.
+
+File descriptor - ``file_descriptor``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+
+.. note::
+
+    This is the default strategy (except for macOS and OS X where it's not
+    supported).
+
+This strategy will use file descriptors as shared memory handles. Whenever a
+storage is moved to shared memory, a file descriptor obtained from ``shm_open``
+is cached with the object, and when it's going to be sent to other processes,
+the file descriptor will be transferred (e.g. via UNIX sockets) to it. The
+receiver will also cache the file descriptor and ``mmap`` it, to obtain a shared
+view onto the storage data.
+
+Note that if there will be a lot of tensors shared, this strategy will keep a
+large number of file descriptors open most of the time. If your system has low
+limits for the number of open file descriptors, and you can't raise them, you
+should use the ``file_system`` strategy.
+
+File system - ``file_system``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This strategy will use file names given to ``shm_open`` to identify the shared
+memory regions. This has a benefit of not requiring the implementation to cache
+the file descriptors obtained from it, but at the same time is prone to shared
+memory leaks. The file can't be deleted right after its creation, because other
+processes need to access it to open their views. If the processes fatally
+crash, or are killed, and don't call the storage destructors, the files will
+remain in the system. This is very serious, because they keep using up the
+memory until the system is restarted, or they're freed manually.
+
+To counter the problem of shared memory file leaks, :mod:`torch.multiprocessing`
+will spawn a daemon named ``torch_shm_manager`` that will isolate itself from
+the current process group, and will keep track of all shared memory allocations.
+Once all processes connected to it exit, it will wait a moment to ensure there
+will be no new connections, and will iterate over all shared memory files
+allocated by the group. If it finds that any of them still exist, they will be
+deallocated. We've tested this method and it proved to be robust to various
+failures. Still, if your system has high enough limits, and ``file_descriptor``
+is a supported strategy, we do not recommend switching to this one.

docs/source/notes/autograd.rst

@@ -0,0 +1,151 @@
+Autograd mechanics
+==================
+
+This note will present an overview of how autograd works and records the
+operations. It's not strictly necessary to understand all this, but we recommend
+getting familiar with it, as it will help you write more efficient, cleaner
+programs, and can aid you in debugging.
+
+.. _excluding-subgraphs:
+
+Excluding subgraphs from backward
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Every Variable has two flags: :attr:`requires_grad` and :attr:`volatile`.
+They both allow for fine grained exclusion of subgraphs from gradient
+computation and can increase efficiency.
+
+.. _excluding-requires_grad:
+
+``requires_grad``
+~~~~~~~~~~~~~~~~~
+
+If there's a single input to an operation that requires gradient, its output
+will also require gradient. Conversely, only if all inputs don't require
+gradient, the output also won't require it. Backward computation is never
+performed in the subgraphs, where all Variables didn't require gradients.
+
+.. code::
+
+    >>> x = Variable(torch.randn(5, 5))
+    >>> y = Variable(torch.randn(5, 5))
+    >>> z = Variable(torch.randn(5, 5), requires_grad=True)
+    >>> a = x + y
+    >>> a.requires_grad
+    False
+    >>> b = a + z
+    >>> b.requires_grad
+    True
+
+This is especially useful when you want to freeze part of your model, or you
+know in advance that you're not going to use gradients w.r.t. some parameters.
+For example if you want to finetune a pretrained CNN, it's enough to switch the
+:attr:`requires_grad` flags in the frozen base, and no intermediate buffers will
+be saved, until the computation gets to the last layer, where the affine
+transform will use weights that require gradient, and the output of the network
+will also require them.
+
+.. code::
+
+    model = torchvision.models.resnet18(pretrained=True)
+    for param in model.parameters():
+        param.requires_grad = False
+    # Replace the last fully-connected layer
+    # Parameters of newly constructed modules have requires_grad=True by default
+    model.fc = nn.Linear(512, 100)
+
+    # Optimize only the classifier
+    optimizer = optim.SGD(model.fc.parameters(), lr=1e-2, momentum=0.9)
+
+``volatile``
+~~~~~~~~~~~~
+
+Volatile is recommended for purely inference mode, when you're sure you won't
+be even calling `.backward()`. It's more efficient than any other autograd
+setting - it will use the absolute minimal amount of memory to evaluate the
+model. ``volatile`` also determines that ``requires_grad is False``.
+
+Volatile differs from :ref:`excluding-requires_grad` in how the flag propagates.
+If there's even a single volatile input to an operation, its output is also
+going to be volatile. Volatility spreads across the graph much easier than
+non-requiring gradient - you only need a **single** volatile leaf to have a
+volatile output, while you need **all** leaves to not require gradient to
+have an output that doesn't require gradient. Using volatile flag you don't
+need to change any settings of your model parameters to use it for
+inference. It's enough to create a volatile input, and this will ensure that
+no intermediate states are saved.
+
+.. code::
+
+    >>> regular_input = Variable(torch.randn(1, 3, 227, 227))
+    >>> volatile_input = Variable(torch.randn(1, 3, 227, 227), volatile=True)
+    >>> model = torchvision.models.resnet18(pretrained=True)
+    >>> model(regular_input).requires_grad
+    True
+    >>> model(volatile_input).requires_grad
+    False
+    >>> model(volatile_input).volatile
+    True
+    >>> model(volatile_input).grad_fn is None
+    True
+
+How autograd encodes the history
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Autograd is reverse automatic differentiation system.  Conceptually,
+autograd records a graph recording all of the operations that created
+the data as you execute operations, giving you a directed acyclic graph
+whose leaves are the input variables and roots are the output variables.
+By tracing this graph from roots to leaves, you can automatically
+compute the gradients using the chain rule.
+
+Internally, autograd represents this graph as a graph of
+:class:`Function` objects (really expressions), which can be
+:meth:`~torch.autograd.Function.apply` ed to compute the result of
+evaluating the graph.  When computing the forwards pass, autograd
+simultaneously performs the requested computations and builds up a graph
+representing the function that computes the gradient (the ``.grad_fn``
+attribute of each :class:`Variable` is an entry point into this graph).
+When the forwards pass is completed, we evaluate this graph in the
+backwards pass to compute the gradients.
+
+An important thing to note is that the graph is recreated from scratch at every
+iteration, and this is exactly what allows for using arbitrary Python control
+flow statements, that can change the overall shape and size of the graph at
+every iteration. You don't have to encode all possible paths before you
+launch the training - what you run is what you differentiate.
+
+In-place operations on Variables
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Supporting in-place operations in autograd is a hard matter, and we discourage
+their use in most cases. Autograd's aggressive buffer freeing and reuse makes
+it very efficient and there are very few occasions when in-place operations
+actually lower memory usage by any significant amount. Unless you're operating
+under heavy memory pressure, you might never need to use them.
+
+There are two main reasons that limit the applicability of in-place operations:
+
+1. Overwriting values required to compute gradients. This is why variables don't
+   support ``log_``. Its gradient formula requires the original input, and while
+   it is possible to recreate it by computing the inverse operation, it is
+   numerically unstable, and requires additional work that often defeats the
+   purpose of using these functions.
+
+2. Every in-place operation actually requires the implementation to rewrite the
+   computational graph. Out-of-place versions simply allocate new objects and
+   keep references to the old graph, while in-place operations, require
+   changing the creator of all inputs to the :class:`Function` representing
+   this operation. This can be tricky, especially if there are many Variables
+   that reference the same storage (e.g. created by indexing or transposing),
+   and in-place functions will actually raise an error if the storage of
+   modified inputs is referenced by any other :class:`Variable`.
+
+In-place correctness checks
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Every variable keeps a version counter, that is incremented every time it's
+marked dirty in any operation. When a Function saves any tensors for backward,
+a version counter of their containing Variable is saved as well. Once you access
+``self.saved_tensors`` it is checked, and if it's greater than the saved value
+an error is raised.

docs/source/notes/broadcasting.rst

@@ -0,0 +1,113 @@
+.. _broadcasting-semantics:
+
+Broadcasting semantics
+======================
+
+Many PyTorch operations support :any:`NumPy Broadcasting Semantics <numpy.doc.broadcasting>`.
+
+In short, if a PyTorch operation supports broadcast, then its Tensor arguments can be
+automatically expanded to be of equal sizes (without making copies of the data).
+
+General semantics
+-----------------
+Two tensors are "broadcastable" if the following rules hold:
+
+- Each tensor has at least one dimension.
+- When iterating over the dimension sizes, starting at the trailing dimension,
+  the dimension sizes must either be equal, one of them is 1, or one of them
+  does not exist.
+
+For Example::
+
+    >>> x=torch.FloatTensor(5,7,3)
+    >>> y=torch.FloatTensor(5,7,3)
+    # same shapes are always broadcastable (i.e. the above rules always hold)
+
+    >>> x=torch.FloatTensor()
+    >>> y=torch.FloatTensor(2,2)
+    # x and y are not broadcastable, because x does not have at least 1 dimension
+
+    # can line up trailing dimensions
+    >>> x=torch.FloatTensor(5,3,4,1)
+    >>> y=torch.FloatTensor(  3,1,1)
+    # x and y are broadcastable.
+    # 1st trailing dimension: both have size 1
+    # 2nd trailing dimension: y has size 1
+    # 3rd trailing dimension: x size == y size
+    # 4th trailing dimension: y dimension doesn't exist
+
+    # but:
+    >>> x=torch.FloatTensor(5,2,4,1)
+    >>> y=torch.FloatTensor(  3,1,1)
+    # x and y are not broadcastable, because in the 3rd trailing dimension 2 != 3
+
+If two tensors :attr:`x`, :attr:`y` are "broadcastable", the resulting tensor size
+is calculated as follows:
+
+- If the number of dimensions of :attr:`x` and :attr:`y` are not equal, prepend 1
+  to the dimensions of the tensor with fewer dimensions to make them equal length.
+- Then, for each dimension size, the resulting dimension size is the max of the sizes of
+  :attr:`x` and :attr:`y` along that dimension.
+
+For Example::
+
+    # can line up trailing dimensions to make reading easier
+    >>> x=torch.FloatTensor(5,1,4,1)
+    >>> y=torch.FloatTensor(  3,1,1)
+    >>> (x+y).size()
+    torch.Size([5, 3, 4, 1])
+
+    # but not necessary:
+    >>> x=torch.FloatTensor(1)
+    >>> y=torch.FloatTensor(3,1,7)
+    >>> (x+y).size()
+    torch.Size([3, 1, 7])
+
+    >>> x=torch.FloatTensor(5,2,4,1)
+    >>> y=torch.FloatTensor(3,1,1)
+    >>> (x+y).size()
+    RuntimeError: The size of tensor a (2) must match the size of tensor b (3) at non-singleton dimension 1
+
+In-place semantics
+------------------
+One complication is that in-place operations do not allow the in-place tensor to change shape
+as a result of the broadcast.
+
+For Example::
+
+    >>> x=torch.FloatTensor(5,3,4,1)
+    >>> y=torch.FloatTensor(3,1,1)
+    >>> (x.add_(y)).size()
+    torch.Size([5, 3, 4, 1])
+
+    # but:
+    >>> x=torch.FloatTensor(1,3,1)
+    >>> y=torch.FloatTensor(3,1,7)
+    >>> (x.add_(y)).size()
+    RuntimeError: The expanded size of the tensor (1) must match the existing size (7) at non-singleton dimension 2.
+
+Backwards compatibility
+-----------------------
+Prior versions of PyTorch allowed certain pointwise functions to execute on tensors with different shapes,
+as long as the number of elements in each tensor was equal.  The pointwise operation would then be carried
+out by viewing each tensor as 1-dimensional.  PyTorch now supports broadcasting and the "1-dimensional"
+pointwise behavior is considered deprecated and will generate a Python warning in cases where tensors are
+not broadcastable, but have the same number of elements.
+
+Note that the introduction of broadcasting can cause backwards incompatible changes in the case where
+two tensors do not have the same shape, but are broadcastable and have the same number of elements.
+For Example::
+
+    >>> torch.add(torch.ones(4,1), torch.randn(4))
+
+would previously produce a Tensor with size: torch.Size([4,1]), but now produces a Tensor with size: torch.Size([4,4]).
+In order to help identify cases in your code where backwards incompatibilities introduced by broadcasting may exist,
+you may set `torch.utils.backcompat.broadcast_warning.enabled` to `True`, which will generate a python warning
+in such cases.
+
+For Example::
+
+    >>> torch.utils.backcompat.broadcast_warning.enabled=True
+    >>> torch.add(torch.ones(4,1), torch.ones(4))
+    __main__:1: UserWarning: self and other do not have the same shape, but are broadcastable, and have the same number of elements.
+    Changing behavior in a backwards incompatible manner to broadcasting rather than viewing as 1-dimensional.

docs/source/notes/cuda.rst

@@ -0,0 +1,172 @@
+.. _cuda-semantics:
+
+CUDA semantics
+==============
+
+:mod:`torch.cuda` is used to set up and run CUDA operations. It keeps track of
+the currently selected GPU, and all CUDA tensors you allocate will by default be
+created on that device. The selected device can be changed with a
+:any:`torch.cuda.device` context manager.
+
+However, once a tensor is allocated, you can do operations on it irrespective
+of the selected device, and the results will be always placed in on the same
+device as the tensor.
+
+Cross-GPU operations are not allowed by default, with the only exception of
+:meth:`~torch.Tensor.copy_`. Unless you enable peer-to-peer memory access, any
+attempts to launch ops on tensors spread across different devices will raise an
+error.
+
+Below you can find a small example showcasing this::
+
+    x = torch.cuda.FloatTensor(1)
+    # x.get_device() == 0
+    y = torch.FloatTensor(1).cuda()
+    # y.get_device() == 0
+
+    with torch.cuda.device(1):
+        # allocates a tensor on GPU 1
+        a = torch.cuda.FloatTensor(1)
+
+        # transfers a tensor from CPU to GPU 1
+        b = torch.FloatTensor(1).cuda()
+        # a.get_device() == b.get_device() == 1
+
+        c = a + b
+        # c.get_device() == 1
+
+        z = x + y
+        # z.get_device() == 0
+
+        # even within a context, you can give a GPU id to the .cuda call
+        d = torch.randn(2).cuda(2)
+        # d.get_device() == 2
+
+Memory management
+-----------------
+
+PyTorch use a caching memory allocator to speed up memory allocations. This
+allows fast memory deallocation without device synchronizations. However, the
+unused memory managed by the allocator will still show as if used in
+`nvidia-smi`. Calling :meth:`~torch.cuda.empty_cache` can release all unused
+cached memory from PyTorch so that those can be used by other GPU applications.
+
+
+Best practices
+--------------
+
+Device-agnostic code
+^^^^^^^^^^^^^^^^^^^^
+
+Due to the structure of PyTorch, you may need to explicitly write
+device-agnostic (CPU or GPU) code; an example may be creating a new tensor as
+the initial hidden state of a recurrent neural network.
+
+The first step is to determine whether the GPU should be used or not. A common
+pattern is to use Python's ``argparse`` module to read in user arguments, and
+have a flag that can be used to disable CUDA, in combination with
+:meth:`~torch.cuda.is_available`. In the following, ``args.cuda`` results in a
+flag that can be used to cast tensors and modules to CUDA if desired::
+
+    import argparse
+    import torch
+
+    parser = argparse.ArgumentParser(description='PyTorch Example')
+    parser.add_argument('--disable-cuda', action='store_true',
+                        help='Disable CUDA')
+    args = parser.parse_args()
+    args.cuda = not args.disable_cuda and torch.cuda.is_available()
+
+If modules or tensors need to be sent to the GPU, ``args.cuda`` can be used as
+follows::
+
+    x = torch.Tensor(8, 42)
+    net = Network()
+    if args.cuda:
+      x = x.cuda()
+      net.cuda()
+
+When creating tensors, an alternative to the if statement is to have a default
+datatype defined, and cast all tensors using that. An example when using a
+dataloader would be as follows::
+
+    dtype = torch.cuda.FloatTensor
+    for i, x in enumerate(train_loader):
+        x = Variable(x.type(dtype))
+
+When working with multiple GPUs on a system, you can use the
+``CUDA_VISIBLE_DEVICES`` environment flag to manage which GPUs are available to
+PyTorch. As mentioned above, to manually control which GPU a tensor is created
+on, the best practice is to use a :any:`torch.cuda.device` context manager::
+
+    print("Outside device is 0")  # On device 0 (default in most scenarios)
+    with torch.cuda.device(1):
+        print("Inside device is 1")  # On device 1
+    print("Outside device is still 0")  # On device 0
+
+If you have a tensor and would like to create a new tensor of the same type on
+the same device, then you can use the :meth:`~torch.Tensor.new` method, which
+acts the same as a normal tensor constructor. Whilst the previously mentioned
+methods depend on the current GPU context, :meth:`~torch.Tensor.new` preserves
+the device of the original tensor.
+
+This is the recommended practice when creating modules in which new
+tensors/variables need to be created internally during the forward pass::
+
+    x_cpu = torch.FloatTensor(1)
+    x_gpu = torch.cuda.FloatTensor(1)
+    x_cpu_long = torch.LongTensor(1)
+
+    y_cpu = x_cpu.new(8, 10, 10).fill_(0.3)
+    y_gpu = x_gpu.new(x_gpu.size()).fill_(-5)
+    y_cpu_long = x_cpu_long.new([[1, 2, 3]])
+
+If you want to create a tensor of the same type and size of another tensor, and
+fill it with either ones or zeros, :meth:`~torch.ones_like` or
+:meth:`~torch.zeros_like` are provided as convenient helper functions (which
+also preserve device)::
+
+    x_cpu = torch.FloatTensor(1)
+    x_gpu = torch.cuda.FloatTensor(1)
+
+    y_cpu = torch.ones_like(x_cpu)
+    y_gpu = torch.zeros_like(x_gpu)
+
+
+Use pinned memory buffers
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. warning:
+
+    This is an advanced tip. You overuse of pinned memory can cause serious
+    problems if you'll be running low on RAM, and you should be aware that
+    pinning is often an expensive operation.
+
+Host to GPU copies are much faster when they originate from pinned (page-locked)
+memory. CPU tensors and storages expose a :meth:`~torch.Tensor.pin_memory`
+method, that returns a copy of the object, with data put in a pinned region.
+
+Also, once you pin a tensor or storage, you can use asynchronous GPU copies.
+Just pass an additional ``async=True`` argument to a :meth:`~torch.Tensor.cuda`
+call. This can be used to overlap data transfers with computation.
+
+You can make the :class:`~torch.utils.data.DataLoader` return batches placed in
+pinned memory by passing ``pin_memory=True`` to its constructor.
+
+.. _cuda-nn-dataparallel-instead:
+
+Use nn.DataParallel instead of multiprocessing
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Most use cases involving batched inputs and multiple GPUs should default to
+using :class:`~torch.nn.DataParallel` to utilize more than one GPU. Even with
+the GIL, a single Python process can saturate multiple GPUs.
+
+As of version 0.1.9, large numbers of GPUs (8+) might not be fully utilized.
+However, this is a known issue that is under active development. As always,
+test your use case.
+
+There are significant caveats to using CUDA models with
+:mod:`~torch.multiprocessing`; unless care is taken to meet the data handling
+requirements exactly, it is likely that your program will have incorrect or
+undefined behavior.

docs/source/notes/extending.rst

@@ -0,0 +1,182 @@
+Extending PyTorch
+=================
+
+In this note we'll cover ways of extending :mod:`torch.nn`,
+:mod:`torch.autograd`, and writing custom C extensions utilizing our C
+libraries.
+
+Extending :mod:`torch.autograd`
+-------------------------------
+
+.. currentmodule:: torch.autograd
+
+Adding operations to :mod:`~torch.autograd` requires implementing a new
+:class:`Function` subclass for each operation. Recall that :class:`Function` s
+are what :mod:`~torch.autograd` uses to compute the results and gradients, and
+encode the operation history. Every new function requires you to implement 2
+methods:
+
+- :meth:`~Function.forward` - the code that performs the operation. It can take
+  as many arguments as you want, with some of them being optional, if you
+  specify the default values. All kinds of Python objects are accepted here.
+  :class:`Variable` arguments will be converted to :class:`Tensor` s before the
+  call, and their use will be registered in the graph. Note that this logic won't
+  traverse lists/dicts/any other data structures and will only consider Variables
+  that are direct arguments to the call. You can return either a single
+  :class:`Tensor` output, or a :class:`tuple` of :class:`Tensor` s if there are
+  multiple outputs. Also, please refer to the docs of :class:`Function` to find
+  descriptions of useful methods that can be called only from :meth:`~Function.forward`.
+- :meth:`~Function.backward` - gradient formula. It will be given
+  as many :class:`Variable` arguments as there were outputs, with each of them
+  representing gradient w.r.t. that output. It should return as many
+  :class:`Variable` s as there were inputs, with each of them containing the
+  gradient w.r.t. its corresponding input. If your inputs didn't require
+  gradient (see :attr:`~Variable.needs_input_grad`), or were non-:class:`Variable`
+  objects, you can return :class:`python:None`. Also, if you have optional
+  arguments to :meth:`~Variable.forward` you can return more gradients than there
+  were inputs, as long as they're all :any:`python:None`.
+
+Below you can find code for a ``Linear`` function from :mod:`torch.nn`, with
+additional comments::
+
+    # Inherit from Function
+    class LinearFunction(Function):
+
+        # Note that both forward and backward are @staticmethods
+        @staticmethod
+        # bias is an optional argument
+        def forward(ctx, input, weight, bias=None):
+            ctx.save_for_backward(input, weight, bias)
+            output = input.mm(weight.t())
+            if bias is not None:
+                output += bias.unsqueeze(0).expand_as(output)
+            return output
+
+        # This function has only a single output, so it gets only one gradient
+        @staticmethod
+        def backward(ctx, grad_output):
+            # This is a pattern that is very convenient - at the top of backward
+            # unpack saved_tensors and initialize all gradients w.r.t. inputs to
+            # None. Thanks to the fact that additional trailing Nones are
+            # ignored, the return statement is simple even when the function has
+            # optional inputs.
+            input, weight, bias = ctx.saved_variables
+            grad_input = grad_weight = grad_bias = None
+
+            # These needs_input_grad checks are optional and there only to
+            # improve efficiency. If you want to make your code simpler, you can
+            # skip them. Returning gradients for inputs that don't require it is
+            # not an error.
+            if ctx.needs_input_grad[0]:
+                grad_input = grad_output.mm(weight)
+            if ctx.needs_input_grad[1]:
+                grad_weight = grad_output.t().mm(input)
+            if bias is not None and ctx.needs_input_grad[2]:
+                grad_bias = grad_output.sum(0).squeeze(0)
+
+            return grad_input, grad_weight, grad_bias
+
+Now, to make it easier to use these custom ops, we recommend aliasing their
+``apply`` method::
+
+    linear = LinearFunction.apply
+
+Here, we give an additional example of a function that is parametrized by
+non-Variable arguments::
+
+    class MulConstant(Function):
+        @staticmethod
+        def forward(ctx, tensor, constant):
+            # ctx is a context object that can be used to stash information
+            # for backward computation
+            ctx.constant = constant
+            return tensor * constant
+
+        @staticmethod
+        def backward(ctx, grad_output):
+            # We return as many input gradients as there were arguments.
+            # Gradients of non-Tensor arguments to forward must be None.
+            return grad_output * ctx.constant, None
+
+You probably want to check if the backward method you implemented actually
+computes the derivatives of your function. It is possible by comparing with
+numerical approximations using small finite differences::
+
+    from torch.autograd import gradcheck
+
+    # gradchek takes a tuple of tensor as input, check if your gradient
+    # evaluated with these tensors are close enough to numerical
+    # approximations and returns True if they all verify this condition.
+    input = (Variable(torch.randn(20,20).double(), requires_grad=True), Variable(torch.randn(30,20).double(), requires_grad=True),)
+    test = gradcheck(Linear.apply, input, eps=1e-6, atol=1e-4)
+    print(test)
+
+Extending :mod:`torch.nn`
+-------------------------
+
+.. currentmodule:: torch.nn
+
+:mod:`~torch.nn` exports two kinds of interfaces - modules and their functional
+versions. You can extend it in both ways, but we recommend using modules for
+all kinds of layers, that hold any parameters or buffers, and recommend using
+a functional form parameter-less operations like activation functions, pooling,
+etc.
+
+Adding a functional version of an operation is already fully covered in the
+section above.
+
+Adding a :class:`Module`
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Since :mod:`~torch.nn` heavily utilizes :mod:`~torch.autograd`, adding a new
+:class:`Module` requires implementing a :class:`~torch.autograd.Function`
+that performs the operation and can compute the gradient. From now on let's
+assume that we want to implement a ``Linear`` module and we have the function
+implementated as in the listing above. There's very little code required to
+add this. Now, there are two functions that need to be implemented:
+
+- ``__init__`` (*optional*) - takes in arguments such as kernel sizes, numbers
+  of features, etc. and initializes parameters and buffers.
+- :meth:`~Module.forward` - instantiates a :class:`~torch.autograd.Function` and
+  uses it to perform the operation. It's very similar to a functional wrapper
+  shown above.
+
+This is how a ``Linear`` module can be implemented::
+
+    class Linear(nn.Module):
+        def __init__(self, input_features, output_features, bias=True):
+            super(Linear, self).__init__()
+            self.input_features = input_features
+            self.output_features = output_features
+
+            # nn.Parameter is a special kind of Variable, that will get
+            # automatically registered as Module's parameter once it's assigned
+            # as an attribute. Parameters and buffers need to be registered, or
+            # they won't appear in .parameters() (doesn't apply to buffers), and
+            # won't be converted when e.g. .cuda() is called. You can use
+            # .register_buffer() to register buffers.
+            # nn.Parameters can never be volatile and, different than Variables,
+            # they require gradients by default.
+            self.weight = nn.Parameter(torch.Tensor(output_features, input_features))
+            if bias:
+                self.bias = nn.Parameter(torch.Tensor(output_features))
+            else:
+                # You should always register all possible parameters, but the
+                # optional ones can be None if you want.
+                self.register_parameter('bias', None)
+
+            # Not a very smart way to initialize weights
+            self.weight.data.uniform_(-0.1, 0.1)
+            if bias is not None:
+                self.bias.data.uniform_(-0.1, 0.1)
+
+        def forward(self, input):
+            # See the autograd section for explanation of what happens here.
+            return LinearFunction.apply(input, self.weight, self.bias)
+
+
+Writing custom C extensions
+---------------------------
+
+Coming soon. For now you can find an example at
+`GitHub <https://github.com/pytorch/extension-ffi>`_.

docs/source/notes/multiprocessing.rst

@@ -0,0 +1,124 @@
+Multiprocessing best practices
+==============================
+
+:mod:`torch.multiprocessing` is a drop in replacement for Python's
+:mod:`python:multiprocessing` module. It supports the exact same operations,
+but extends it, so that all tensors sent through a
+:class:`python:multiprocessing.Queue`, will have their data moved into shared
+memory and will only send a handle to another process.
+
+.. note::
+
+    When a :class:`~torch.autograd.Variable` is sent to another process, both
+    the :attr:`Variable.data` and :attr:`Variable.grad.data` are going to be
+    shared.
+
+This allows to implement various training methods, like Hogwild, A3C, or any
+others that require asynchronous operation.
+
+Sharing CUDA tensors
+--------------------
+
+Sharing CUDA tensors between processes is supported only in Python 3, using
+a ``spawn`` or ``forkserver`` start methods. :mod:`python:multiprocessing` in
+Python 2 can only create subprocesses using ``fork``, and it's not supported
+by the CUDA runtime.
+
+.. warning::
+
+    CUDA API requires that the allocation exported to other processes remains
+    valid as long as it's used by them. You should be careful and ensure that
+    CUDA tensors you shared don't go out of scope as long as it's necessary.
+    This shouldn't be a problem for sharing model parameters, but passing other
+    kinds of data should be done with care. Note that this restriction doesn't
+    apply to shared CPU memory.
+
+See also: :ref:`cuda-nn-dataparallel-instead`
+
+
+Best practices and tips
+-----------------------
+
+Avoiding and fighting deadlocks
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+There are a lot of things that can go wrong when a new process is spawned, with
+the most common cause of deadlocks being background threads. If there's any
+thread that holds a lock or imports a module, and ``fork`` is called, it's very
+likely that the subprocess will be in a corrupted state and will deadlock or
+fail in a different way. Note that even if you don't, Python built in
+libraries do - no need to look further than :mod:`python:multiprocessing`.
+:class:`python:multiprocessing.Queue` is actually a very complex class, that
+spawns multiple threads used to serialize, send and receive objects, and they
+can cause aforementioned problems too. If you find yourself in such situation
+try using a :class:`~python:multiprocessing.queues.SimpleQueue`, that doesn't
+use any additional threads.
+
+We're trying our best to make it easy for you and ensure these deadlocks don't
+happen but some things are out of our control. If you have any issues you can't
+cope with for a while, try reaching out on forums, and we'll see if it's an
+issue we can fix.
+
+Reuse buffers passed through a Queue
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Remember that each time you put a :class:`~torch.Tensor` into a
+:class:`python:multiprocessing.Queue`, it has to be moved into shared memory.
+If it's already shared, it is a no-op, otherwise it will incur an additional
+memory copy that can slow down the whole process. Even if you have a pool of
+processes sending data to a single one, make it send the buffers back - this
+is nearly free and will let you avoid a copy when sending next batch.
+
+Asynchronous multiprocess training (e.g. Hogwild)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Using :mod:`torch.multiprocessing`, it is possible to train a model
+asynchronously, with parameters either shared all the time, or being
+periodically synchronized. In the first case, we recommend sending over the whole
+model object, while in the latter, we advise to only send the
+:meth:`~torch.nn.Module.state_dict`.
+
+We recommend using :class:`python:multiprocessing.Queue` for passing all kinds
+of PyTorch objects between processes. It is possible to e.g. inherit the tensors
+and storages already in shared memory, when using the ``fork`` start method,
+however it is very bug prone and should be used with care, and only by advanced
+users. Queues, even though they're sometimes a less elegant solution, will work
+properly in all cases.
+
+.. warning::
+
+    You should be careful about having global statements, that are not guarded
+    with an ``if __name__ == '__main__'``. If a different start method than
+    ``fork`` is used, they will be executed in all subprocesses.
+
+Hogwild
+~~~~~~~
+
+A concrete Hogwild implementation can be found in the `examples repository`__,
+but to showcase the overall structure of the code, there's also a minimal
+example below as well::
+
+    import torch.multiprocessing as mp
+    from model import MyModel
+
+    def train(model):
+        # Construct data_loader, optimizer, etc.
+        for data, labels in data_loader:
+            optimizer.zero_grad()
+            loss_fn(model(data), labels).backward()
+            optimizer.step()  # This will update the shared parameters
+
+    if __name__ == '__main__':
+        num_processes = 4
+        model = MyModel()
+        # NOTE: this is required for the ``fork`` method to work
+        model.share_memory()
+        processes = []
+        for rank in range(num_processes):
+            p = mp.Process(target=train, args=(model,))
+            p.start()
+            processes.append(p)
+        for p in processes:
+          p.join()
+
+.. __: https://github.com/pytorch/examples/tree/master/mnist_hogwild

docs/source/notes/serialization.rst

@@ -0,0 +1,34 @@
+
+Serialization semantics
+=======================
+
+Best practices
+--------------
+
+.. _recommend-saving-models:
+
+Recommended approach for saving a model
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+There are two main approaches for serializing and restoring a model.
+
+The first (recommended) saves and loads only the model parameters::
+
+    torch.save(the_model.state_dict(), PATH)
+
+Then later::
+
+    the_model = TheModelClass(*args, **kwargs)
+    the_model.load_state_dict(torch.load(PATH))
+
+The second saves and loads the entire model::
+
+    torch.save(the_model, PATH)
+
+Then later::
+
+    the_model = torch.load(PATH)
+
+However in this case, the serialized data is bound to the specific classes
+and the exact directory structure used, so it can break in various ways when
+used in other projects, or after some serious refactors.

docs/source/onnx.rst

@@ -0,0 +1,182 @@
+torch.onnx
+============
+.. automodule:: torch.onnx
+
+Example: End-to-end AlexNet from PyTorch to Caffe2
+--------------------------------------------------
+
+Here is a simple script which exports a pretrained AlexNet as defined in
+torchvision into ONNX.  It runs a single round of inference and then
+saves the resulting traced model to ``alexnet.proto``::
+
+    from torch.autograd import Variable
+    import torch.onnx
+    import torchvision
+
+    dummy_input = Variable(torch.randn(10, 3, 224, 224)).cuda()
+    model = torchvision.models.alexnet(pretrained=True).cuda()
+    torch.onnx.export(model, dummy_input, "alexnet.proto", verbose=True)
+
+The resulting ``alexnet.proto`` is a binary protobuf file which contains both
+the network structure and parameters of the model you exported
+(in this case, AlexNet).  The keyword argument ``verbose=True`` causes the
+exporter to print out a human-readable representation of the network::
+
+    # All parameters are encoded explicitly as inputs.  By convention,
+    # learned parameters (ala nn.Module.state_dict) are first, and the
+    # actual inputs are last.
+    graph(%1 : Float(64, 3, 11, 11)
+          %2 : Float(64)
+          # The definition sites of all variables are annotated with type
+          # information, specifying the type and size of tensors.
+          # For example, %3 is a 192 x 64 x 5 x 5 tensor of floats.
+          %3 : Float(192, 64, 5, 5)
+          %4 : Float(192)
+          # ---- omitted for brevity ----
+          %15 : Float(1000, 4096)
+          %16 : Float(1000)
+          %17 : Float(10, 3, 224, 224)) { # the actual input!
+      # Every statement consists of some output tensors (and their types),
+      # the operator to be run (with its attributes, e.g., kernels, strides,
+      # etc.), its input tensors (%17, %1)
+      %19 : UNKNOWN_TYPE = Conv[kernels=[11, 11], strides=[4, 4], pads=[2, 2, 2, 2], dilations=[1, 1], group=1](%17, %1), uses = [[%20.i0]];
+      # UNKNOWN_TYPE: sometimes type information is not known.  We hope to eliminate
+      # all such cases in a later release.
+      %20 : Float(10, 64, 55, 55) = Add[broadcast=1, axis=1](%19, %2), uses = [%21.i0];
+      %21 : Float(10, 64, 55, 55) = Relu(%20), uses = [%22.i0];
+      %22 : Float(10, 64, 27, 27) = MaxPool[kernels=[3, 3], pads=[0, 0, 0, 0], dilations=[1, 1], strides=[2, 2]](%21), uses = [%23.i0];
+      # ...
+      # Finally, a network returns some tensors
+      return (%58);
+    }
+
+You can also verify the protobuf using the `onnx <https://github.com/onnx/onnx/>`_ library.
+You can install ``onnx`` with conda::
+
+    conda install -c conda-forge onnx
+
+Then, you can run::
+
+    import onnx
+
+    # Load the ONNX model
+    model = onnx.load("alexnet.proto")
+
+    # Check that the IR is well formed
+    onnx.checker.check_model(model)
+
+    # Print a human readable representation of the graph
+    onnx.helper.printable_graph(model.graph)
+
+To run the exported script with `caffe2 <https://caffe2.ai/>`_, you will need three things:
+
+1. You'll need an install of Caffe2.  If you don't have one already, Please
+   `follow the install instructions <https://caffe2.ai/docs/getting-started.html>`_.
+
+2. You'll need `onnx-caffe2 <https://github.com/onnx/onnx-caffe2>`_, a
+   pure-Python library which provides a Caffe2 backend for ONNX.  You can install ``onnx-caffe2``
+   with pip::
+
+      pip install onnx-caffe2
+
+Once these are installed, you can use the backend for Caffe2::
+
+    # ...continuing from above
+    import onnx_caffe2.backend as backend
+    import numpy as np
+
+    rep = backend.prepare(model, device="CUDA:0") # or "CPU"
+    # For the Caffe2 backend:
+    #     rep.predict_net is the Caffe2 protobuf for the network
+    #     rep.workspace is the Caffe2 workspace for the network
+    #       (see the class onnx_caffe2.backend.Workspace)
+    outputs = rep.run(np.random.randn(10, 3, 224, 224).astype(np.float32))
+    # To run networks with more than one input, pass a tuple
+    # rather than a single numpy ndarray.
+    print(outputs[0])
+
+In the future, there will be backends for other frameworks as well.
+
+Limitations
+-----------
+
+* The ONNX exporter is a *trace-based* exporter, which means that it
+  operates by executing your model once, and exporting the operators which
+  were actually run during this run.  This means that if your model is
+  dynamic, e.g., changes behavior depending on input data, the export
+  won't be accurate.  Similarly, a trace is likely to be valid only
+  for a specific input size (which is one reason why we require explicit inputs
+  on tracing.)  We recommend examining the model trace and making sure
+  the traced operators look reasonable.
+
+* PyTorch and Caffe2 often have implementations of operators with some
+  numeric differences.  Depending on model structure, these differences
+  may be negligible, but they can also cause major divergences in behavior
+  (especially on untrained models.)  In a future release, we plan to
+  allow Caffe2 to call directly to Torch implementations of operators, to
+  help you smooth over these differences when precision is important,
+  and to also document these differences.
+
+Supported operators
+-------------------
+
+The following operators are supported:
+
+* add (nonzero alpha not supported)
+* sub (nonzero alpha not supported)
+* mul
+* div
+* cat
+* mm
+* addmm
+* neg
+* tanh
+* sigmoid
+* mean
+* t
+* expand (only when used before a broadcasting ONNX operator; e.g., add)
+* transpose
+* view
+* split
+* squeeze
+* prelu (single weight shared among input channels not supported)
+* threshold (non-zero threshold/non-zero value not supported)
+* leaky_relu
+* glu
+* softmax
+* avg_pool2d (ceil_mode not supported)
+* log_softmax
+* unfold (experimental support with ATen-Caffe2 integration)
+* elu
+* Conv
+* BatchNorm
+* MaxPool1d (ceil_mode not supported)
+* MaxPool2d (ceil_mode not supported)
+* MaxPool3d (ceil_mode not supported)
+* Embedding (no optional arguments supported)
+* RNN
+* ConstantPadNd
+* Dropout
+* FeatureDropout (training mode not supported)
+* Index (constant integer and tuple indices supported)
+* Negate
+
+The operator set above is sufficient to export the following models:
+
+* AlexNet
+* DCGAN
+* DenseNet
+* Inception (warning: this model is highly sensitive to changes in operator
+  implementation)
+* ResNet
+* SuperResolution
+* VGG
+* `word_language_model <https://github.com/pytorch/examples/tree/master/word_language_model>`_
+
+The interface for specifying operator definitions is highly experimental
+and undocumented; adventurous users should note that the APIs will probably
+change in a future interface.
+
+Functions
+--------------------------
+.. autofunction:: export

docs/source/optim.rst

@@ -0,0 +1,145 @@
+torch.optim
+===================================
+
+.. automodule:: torch.optim
+
+How to use an optimizer
+-----------------------
+
+To use :mod:`torch.optim` you have to construct an optimizer object, that will hold
+the current state and will update the parameters based on the computed gradients.
+
+Constructing it
+^^^^^^^^^^^^^^^
+
+To construct an :class:`Optimizer` you have to give it an iterable containing the
+parameters (all should be :class:`~torch.autograd.Variable` s) to optimize. Then,
+you can specify optimizer-specific options such as the learning rate, weight decay, etc.
+
+.. note::
+
+    If you need to move a model to GPU via `.cuda()`, please do so before
+    constructing optimizers for it. Parameters of a model after `.cuda()` will
+    be different objects with those before the call.
+
+    In general, you should make sure that optimized parameters live in
+    consistent locations when optimizers are constructed and used.
+
+Example::
+
+    optimizer = optim.SGD(model.parameters(), lr = 0.01, momentum=0.9)
+    optimizer = optim.Adam([var1, var2], lr = 0.0001)
+
+Per-parameter options
+^^^^^^^^^^^^^^^^^^^^^
+
+:class:`Optimizer` s also support specifying per-parameter options. To do this, instead
+of passing an iterable of :class:`~torch.autograd.Variable` s, pass in an iterable of
+:class:`dict` s. Each of them will define a separate parameter group, and should contain
+a ``params`` key, containing a list of parameters belonging to it. Other keys
+should match the keyword arguments accepted by the optimizers, and will be used
+as optimization options for this group.
+
+.. note::
+
+    You can still pass options as keyword arguments. They will be used as
+    defaults, in the groups that didn't override them. This is useful when you
+    only want to vary a single option, while keeping all others consistent
+    between parameter groups.
+
+
+For example, this is very useful when one wants to specify per-layer learning rates::
+
+    optim.SGD([
+                    {'params': model.base.parameters()},
+                    {'params': model.classifier.parameters(), 'lr': 1e-3}
+                ], lr=1e-2, momentum=0.9)
+
+This means that ``model.base``'s parameters will use the default learning rate of ``1e-2``,
+``model.classifier``'s parameters will use a learning rate of ``1e-3``, and a momentum of
+``0.9`` will be used for all parameters
+
+Taking an optimization step
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+All optimizers implement a :func:`~Optimizer.step` method, that updates the
+parameters. It can be used in two ways:
+
+``optimizer.step()``
+~~~~~~~~~~~~~~~~~~~~
+
+This is a simplified version supported by most optimizers. The function can be
+called once the gradients are computed using e.g.
+:func:`~torch.autograd.Variable.backward`.
+
+Example::
+
+    for input, target in dataset:
+        optimizer.zero_grad()
+        output = model(input)
+        loss = loss_fn(output, target)
+        loss.backward()
+        optimizer.step()
+
+``optimizer.step(closure)``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some optimization algorithms such as Conjugate Gradient and LBFGS need to
+reevaluate the function multiple times, so you have to pass in a closure that
+allows them to recompute your model. The closure should clear the gradients,
+compute the loss, and return it.
+
+Example::
+
+    for input, target in dataset:
+        def closure():
+            optimizer.zero_grad()
+            output = model(input)
+            loss = loss_fn(output, target)
+            loss.backward()
+            return loss
+        optimizer.step(closure)
+
+Algorithms
+----------
+
+.. autoclass:: Optimizer
+    :members:
+.. autoclass:: Adadelta
+    :members:
+.. autoclass:: Adagrad
+    :members:
+.. autoclass:: Adam
+    :members:
+.. autoclass:: SparseAdam
+    :members:
+.. autoclass:: Adamax
+    :members:
+.. autoclass:: ASGD
+    :members:
+.. autoclass:: LBFGS
+    :members:
+.. autoclass:: RMSprop
+    :members:
+.. autoclass:: Rprop
+    :members:
+.. autoclass:: SGD
+    :members:
+
+How to adjust Learning Rate
+---------------------------
+
+:mod:`torch.optim.lr_scheduler` provides several methods to adjust the learning
+rate based on the number of epoches. :class:`torch.optim.lr_scheduler.ReduceLROnPlateau`
+allows dynamic learning rate reducing based on some validation measurements.
+
+.. autoclass:: torch.optim.lr_scheduler.LambdaLR
+    :members:
+.. autoclass:: torch.optim.lr_scheduler.StepLR
+    :members:
+.. autoclass:: torch.optim.lr_scheduler.MultiStepLR
+    :members:
+.. autoclass:: torch.optim.lr_scheduler.ExponentialLR
+    :members:
+.. autoclass:: torch.optim.lr_scheduler.ReduceLROnPlateau
+    :members:

docs/source/sparse.rst

@@ -0,0 +1,128 @@
+.. currentmodule:: torch.sparse
+
+torch.sparse
+============
+
+.. warning::
+
+    This API is currently experimental and may change in the near future.
+
+Torch supports sparse tensors in COO(rdinate) format, which can
+efficiently store and process tensors for which the majority of elements
+are zeros.
+
+A sparse tensor is represented as a pair of dense tensors: a tensor
+of values and a 2D tensor of indices.  A sparse tensor can be constructed
+by providing these two tensors, as well as the size of the sparse tensor
+(which cannot be inferred from these tensors!)  Suppose we want to define
+a sparse tensor with the entry 3 at location (0, 2), entry 4 at
+location (1, 0), and entry 5 at location (1, 2).  We would then write:
+
+    >>> i = torch.LongTensor([[0, 1, 1],
+                              [2, 0, 2]])
+    >>> v = torch.FloatTensor([3, 4, 5])
+    >>> torch.sparse.FloatTensor(i, v, torch.Size([2,3])).to_dense()
+     0  0  3
+     4  0  5
+    [torch.FloatTensor of size 2x3]
+
+Note that the input to LongTensor is NOT a list of index tuples.  If you want
+to write your indices this way, you should transpose before passing them to
+the sparse constructor:
+
+    >>> i = torch.LongTensor([[0, 2], [1, 0], [1, 2]])
+    >>> v = torch.FloatTensor([3,      4,      5    ])
+    >>> torch.sparse.FloatTensor(i.t(), v, torch.Size([2,3])).to_dense()
+     0  0  3
+     4  0  5
+    [torch.FloatTensor of size 2x3]
+
+You can also construct hybrid sparse tensors, where only the first n
+dimensions are sparse, and the rest of the dimensions are dense.
+
+    >>> i = torch.LongTensor([[2, 4]])
+    >>> v = torch.FloatTensor([[1, 3], [5, 7]])
+    >>> torch.sparse.FloatTensor(i, v).to_dense()
+     0  0
+     0  0
+     1  3
+     0  0
+     5  7
+    [torch.FloatTensor of size 5x2]
+
+An empty sparse tensor can be constructed by specifying its size:
+
+    >>> torch.sparse.FloatTensor(2, 3)
+    SparseFloatTensor of size 2x3 with indices:
+    [torch.LongTensor with no dimension]
+    and values:
+    [torch.FloatTensor with no dimension]
+
+.. note::
+
+    Our sparse tensor format permits *uncoalesced* sparse tensors, where
+    there may be duplicate coordinates in the indices; in this case,
+    the interpretation is that the value at that index is the sum of all
+    duplicate value entries. Uncoalesced tensors permit us to implement
+    certain operators more efficiently.
+
+    For the most part, you shouldn't have to care whether or not a
+    sparse tensor is coalesced or not, as most operations will work
+    identically given a coalesced or uncoalesced sparse tensor.
+    However, there are two cases in which you may need to care.
+
+    First, if you repeatedly perform an operation that can produce
+    duplicate entries (e.g., :func:`torch.sparse.FloatTensor.add`), you
+    should occasionally coalesce your sparse tensors to prevent
+    them from growing too large.
+
+    Second, some operators will produce different values depending on
+    whether or not they are coalesced or not (e.g.,
+    :func:`torch.sparse.FloatTensor._values` and
+    :func:`torch.sparse.FloatTensor._indices`, as well as
+    :func:`torch.Tensor._sparse_mask`).  These operators are
+    prefixed by an underscore to indicate that they reveal internal
+    implementation details and should be used with care, since code
+    that works with coalesced sparse tensors may not work with
+    uncoalesced sparse tensors; generally speaking, it is safest
+    to explicitly coalesce before working with these operators.
+
+    For example, suppose that we wanted to implement an operator
+    by operating directly on :func:`torch.sparse.FloatTensor._values`.
+    Multiplication by a scalar can be implemented in the obvious way,
+    as multiplication distributes over addition; however, square root
+    cannot be implemented directly, since ``sqrt(a + b) != sqrt(a) +
+    sqrt(b)`` (which is what would be computed if you were given an
+    uncoalesced tensor.)
+
+.. class:: FloatTensor()
+
+    .. method:: add
+    .. method:: add_
+    .. method:: clone
+    .. method:: dim
+    .. method:: div
+    .. method:: div_
+    .. method:: get_device
+    .. method:: hspmm
+    .. method:: mm
+    .. method:: mul
+    .. method:: mul_
+    .. method:: resizeAs_
+    .. method:: size
+    .. method:: spadd
+    .. method:: spmm
+    .. method:: sspaddmm
+    .. method:: sspmm
+    .. method:: sub
+    .. method:: sub_
+    .. method:: t_
+    .. method:: toDense
+    .. method:: transpose
+    .. method:: transpose_
+    .. method:: zero_
+    .. method:: coalesce
+    .. method:: is_coalesced
+    .. method:: _indices
+    .. method:: _values
+    .. method:: _nnz

docs/source/storage.rst

@@ -0,0 +1,12 @@
+torch.Storage
+===================================
+
+A :class:`torch.Storage` is a contiguous, one-dimensional array of a single
+data type.
+
+Every :class:`torch.Tensor` has a corresponding storage of the same data type.
+
+.. autoclass:: torch.FloatStorage
+   :members:
+   :undoc-members:
+   :inherited-members:

docs/source/tensors.rst

@@ -0,0 +1,322 @@
+.. currentmodule:: torch
+
+torch.Tensor
+===================================
+
+A :class:`torch.Tensor` is a multi-dimensional matrix containing elements of
+a single data type.
+
+Torch defines seven CPU tensor types and eight GPU tensor types:
+
+======================== ===========================   ================================
+Data type                CPU tensor                    GPU tensor
+======================== ===========================   ================================
+32-bit floating point    :class:`torch.FloatTensor`    :class:`torch.cuda.FloatTensor`
+64-bit floating point    :class:`torch.DoubleTensor`   :class:`torch.cuda.DoubleTensor`
+16-bit floating point    :class:`torch.HalfTensor`     :class:`torch.cuda.HalfTensor`
+8-bit integer (unsigned) :class:`torch.ByteTensor`     :class:`torch.cuda.ByteTensor`
+8-bit integer (signed)   :class:`torch.CharTensor`     :class:`torch.cuda.CharTensor`
+16-bit integer (signed)  :class:`torch.ShortTensor`    :class:`torch.cuda.ShortTensor`
+32-bit integer (signed)  :class:`torch.IntTensor`      :class:`torch.cuda.IntTensor`
+64-bit integer (signed)  :class:`torch.LongTensor`     :class:`torch.cuda.LongTensor`
+======================== ===========================   ================================
+
+The :class:`torch.Tensor` constructor is an alias for the default tensor type
+(:class:`torch.FloatTensor`).
+
+A tensor can be constructed from a Python :class:`list` or sequence:
+
+::
+
+    >>> torch.FloatTensor([[1, 2, 3], [4, 5, 6]])
+    1  2  3
+    4  5  6
+    [torch.FloatTensor of size 2x3]
+
+An empty tensor can be constructed by specifying its size:
+
+::
+
+    >>> torch.IntTensor(2, 4).zero_()
+    0  0  0  0
+    0  0  0  0
+    [torch.IntTensor of size 2x4]
+
+The contents of a tensor can be accessed and modified using Python's indexing
+and slicing notation:
+
+::
+
+    >>> x = torch.FloatTensor([[1, 2, 3], [4, 5, 6]])
+    >>> print(x[1][2])
+    6.0
+    >>> x[0][1] = 8
+    >>> print(x)
+     1  8  3
+     4  5  6
+    [torch.FloatTensor of size 2x3]
+
+Each tensor has an associated :class:`torch.Storage`, which holds its data.
+The tensor class provides multi-dimensional, `strided <https://en.wikipedia.org/wiki/Stride_of_an_array>`_
+view of a storage and defines numeric operations on it.
+
+.. note::
+   Methods which mutate a tensor are marked with an underscore suffix.
+   For example, :func:`torch.FloatTensor.abs_` computes the absolute value
+   in-place and returns the modified tensor, while :func:`torch.FloatTensor.abs`
+   computes the result in a new tensor.
+
+.. class:: Tensor()
+           Tensor(*sizes)
+           Tensor(size)
+           Tensor(sequence)
+           Tensor(ndarray)
+           Tensor(tensor)
+           Tensor(storage)
+
+   Creates a new tensor from an optional size or data.
+
+   If no arguments are given, an empty zero-dimensional tensor is returned.
+   If a :class:`numpy.ndarray`, :class:`torch.Tensor`, or :class:`torch.Storage`
+   is given, a new tensor that shares the same data is returned. If a Python
+   sequence is given, a new tensor is created from a copy of the sequence.
+
+   .. automethod:: abs
+   .. automethod:: abs_
+   .. automethod:: acos
+   .. automethod:: acos_
+   .. automethod:: add
+   .. automethod:: add_
+   .. automethod:: addbmm
+   .. automethod:: addbmm_
+   .. automethod:: addcdiv
+   .. automethod:: addcdiv_
+   .. automethod:: addcmul
+   .. automethod:: addcmul_
+   .. automethod:: addmm
+   .. automethod:: addmm_
+   .. automethod:: addmv
+   .. automethod:: addmv_
+   .. automethod:: addr
+   .. automethod:: addr_
+   .. automethod:: apply_
+   .. automethod:: asin
+   .. automethod:: asin_
+   .. automethod:: atan
+   .. automethod:: atan2
+   .. automethod:: atan2_
+   .. automethod:: atan_
+   .. automethod:: baddbmm
+   .. automethod:: baddbmm_
+   .. automethod:: bernoulli
+   .. automethod:: bernoulli_
+   .. automethod:: bmm
+   .. automethod:: byte
+   .. automethod:: cauchy_
+   .. automethod:: ceil
+   .. automethod:: ceil_
+   .. automethod:: char
+   .. automethod:: chunk
+   .. automethod:: clamp
+   .. automethod:: clamp_
+   .. automethod:: clone
+   .. automethod:: contiguous
+   .. automethod:: copy_
+   .. automethod:: cos
+   .. automethod:: cos_
+   .. automethod:: cosh
+   .. automethod:: cosh_
+   .. automethod:: cpu
+   .. automethod:: cross
+   .. automethod:: cuda
+   .. automethod:: cumprod
+   .. automethod:: cumsum
+   .. automethod:: data_ptr
+   .. automethod:: diag
+   .. automethod:: dim
+   .. automethod:: dist
+   .. automethod:: div
+   .. automethod:: div_
+   .. automethod:: dot
+   .. automethod:: double
+   .. automethod:: eig
+   .. automethod:: element_size
+   .. automethod:: eq
+   .. automethod:: eq_
+   .. automethod:: equal
+   .. automethod:: erf
+   .. automethod:: erf_
+   .. automethod:: erfinv
+   .. automethod:: erfinv_
+   .. automethod:: exp
+   .. automethod:: exp_
+   .. automethod:: expand
+   .. automethod:: expand_as
+   .. automethod:: exponential_
+   .. automethod:: fill_
+   .. automethod:: float
+   .. automethod:: floor
+   .. automethod:: floor_
+   .. automethod:: fmod
+   .. automethod:: fmod_
+   .. automethod:: frac
+   .. automethod:: frac_
+   .. automethod:: gather
+   .. automethod:: ge
+   .. automethod:: ge_
+   .. automethod:: gels
+   .. automethod:: geometric_
+   .. automethod:: geqrf
+   .. automethod:: ger
+   .. automethod:: gesv
+   .. automethod:: gt
+   .. automethod:: gt_
+   .. automethod:: half
+   .. automethod:: histc
+   .. automethod:: index
+   .. automethod:: index_add_
+   .. automethod:: index_copy_
+   .. automethod:: index_fill_
+   .. automethod:: index_select
+   .. automethod:: int
+   .. automethod:: inverse
+   .. automethod:: is_contiguous
+   .. autoattribute:: is_cuda
+      :annotation:
+   .. automethod:: is_pinned
+   .. automethod:: is_set_to
+   .. automethod:: is_signed
+   .. automethod:: kthvalue
+   .. automethod:: le
+   .. automethod:: le_
+   .. automethod:: lerp
+   .. automethod:: lerp_
+   .. automethod:: log
+   .. automethod:: log1p
+   .. automethod:: log1p_
+   .. automethod:: log_
+   .. automethod:: log_normal_
+   .. automethod:: long
+   .. automethod:: lt
+   .. automethod:: lt_
+   .. automethod:: map_
+   .. automethod:: masked_scatter_
+   .. automethod:: masked_fill_
+   .. automethod:: masked_select
+   .. automethod:: matmul
+   .. automethod:: max
+   .. automethod:: mean
+   .. automethod:: median
+   .. automethod:: min
+   .. automethod:: mm
+   .. automethod:: mode
+   .. automethod:: mul
+   .. automethod:: mul_
+   .. automethod:: multinomial
+   .. automethod:: mv
+   .. automethod:: narrow
+   .. automethod:: ndimension
+   .. automethod:: ne
+   .. automethod:: ne_
+   .. automethod:: neg
+   .. automethod:: neg_
+   .. automethod:: nelement
+   .. automethod:: new
+   .. automethod:: nonzero
+   .. automethod:: norm
+   .. automethod:: normal_
+   .. automethod:: numel
+   .. automethod:: numpy
+   .. automethod:: orgqr
+   .. automethod:: ormqr
+   .. automethod:: permute
+   .. automethod:: pin_memory
+   .. automethod:: potrf
+   .. automethod:: potri
+   .. automethod:: potrs
+   .. automethod:: pow
+   .. automethod:: pow_
+   .. automethod:: prod
+   .. automethod:: pstrf
+   .. automethod:: put_
+   .. automethod:: qr
+   .. automethod:: random_
+   .. automethod:: reciprocal
+   .. automethod:: reciprocal_
+   .. automethod:: remainder
+   .. automethod:: remainder_
+   .. automethod:: renorm
+   .. automethod:: renorm_
+   .. automethod:: repeat
+   .. automethod:: resize_
+   .. automethod:: resize_as_
+   .. automethod:: round
+   .. automethod:: round_
+   .. automethod:: rsqrt
+   .. automethod:: rsqrt_
+   .. automethod:: scatter_
+   .. automethod:: select
+   .. automethod:: set_
+   .. automethod:: share_memory_
+   .. automethod:: short
+   .. automethod:: sigmoid
+   .. automethod:: sigmoid_
+   .. automethod:: sign
+   .. automethod:: sign_
+   .. automethod:: sin
+   .. automethod:: sin_
+   .. automethod:: sinh
+   .. automethod:: sinh_
+   .. automethod:: size
+   .. automethod:: sort
+   .. automethod:: split
+   .. automethod:: sqrt
+   .. automethod:: sqrt_
+   .. automethod:: squeeze
+   .. automethod:: squeeze_
+   .. automethod:: std
+   .. automethod:: storage
+   .. automethod:: storage_offset
+   .. automethod:: storage_type
+   .. automethod:: stride
+   .. automethod:: sub
+   .. automethod:: sub_
+   .. automethod:: sum
+   .. automethod:: svd
+   .. automethod:: symeig
+   .. automethod:: t
+   .. automethod:: t_
+   .. automethod:: take
+   .. automethod:: tan
+   .. automethod:: tan_
+   .. automethod:: tanh
+   .. automethod:: tanh_
+   .. automethod:: tolist
+   .. automethod:: topk
+   .. automethod:: trace
+   .. automethod:: transpose
+   .. automethod:: transpose_
+   .. automethod:: tril
+   .. automethod:: tril_
+   .. automethod:: triu
+   .. automethod:: triu_
+   .. automethod:: trtrs
+   .. automethod:: trunc
+   .. automethod:: trunc_
+   .. automethod:: type
+   .. automethod:: type_as
+   .. automethod:: unfold
+   .. automethod:: uniform_
+   .. automethod:: unsqueeze
+   .. automethod:: unsqueeze_
+   .. automethod:: var
+   .. automethod:: view
+   .. automethod:: view_as
+   .. automethod:: zero_
+
+.. class:: ByteTensor()
+
+   The following methods are unique to :class:`torch.ByteTensor`.
+
+   .. automethod:: all
+   .. automethod:: any

docs/source/torch.rst

@@ -0,0 +1,203 @@
+torch
+===================================
+.. automodule:: torch
+
+Tensors
+----------------------------------
+.. autofunction:: is_tensor
+.. autofunction:: is_storage
+.. autofunction:: set_default_tensor_type
+.. autofunction:: numel
+.. autofunction:: set_printoptions
+
+
+Creation Ops
+~~~~~~~~~~~~~~~~~~~~~~
+.. autofunction:: eye
+.. autofunction:: from_numpy
+.. autofunction:: linspace
+.. autofunction:: logspace
+.. autofunction:: ones
+.. autofunction:: ones_like
+.. autofunction:: arange
+.. autofunction:: range
+.. autofunction:: zeros
+.. autofunction:: zeros_like
+
+Indexing, Slicing, Joining, Mutating Ops
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. autofunction:: cat
+.. autofunction:: chunk
+.. autofunction:: gather
+.. autofunction:: index_select
+.. autofunction:: masked_select
+.. autofunction:: nonzero
+.. autofunction:: split
+.. autofunction:: squeeze
+.. autofunction:: stack
+.. autofunction:: t
+.. autofunction:: take
+.. autofunction:: transpose
+.. autofunction:: unbind
+.. autofunction:: unsqueeze
+
+
+Random sampling
+----------------------------------
+.. autofunction:: manual_seed
+.. autofunction:: initial_seed
+.. autofunction:: get_rng_state
+.. autofunction:: set_rng_state
+.. autodata:: default_generator
+.. autofunction:: bernoulli
+.. autofunction:: multinomial
+.. autofunction:: normal
+.. autofunction:: rand
+.. autofunction:: randn
+.. autofunction:: randperm
+
+In-place random sampling
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are a few more in-place random sampling functions defined on Tensors as well. Click through to refer to their documentation:
+
+- :func:`torch.Tensor.bernoulli_` - in-place version of :func:`torch.bernoulli`
+- :func:`torch.Tensor.cauchy_` - numbers drawn from the Cauchy distribution
+- :func:`torch.Tensor.exponential_` - numbers drawn from the exponential distribution
+- :func:`torch.Tensor.geometric_` - elements drawn from the geometric distribution
+- :func:`torch.Tensor.log_normal_` - samples from the log-normal distribution
+- :func:`torch.Tensor.normal_` - in-place version of :func:`torch.normal`
+- :func:`torch.Tensor.random_` - numbers sampled from the discrete uniform distribution
+- :func:`torch.Tensor.uniform_` - numbers sampled from the uniform distribution
+
+
+Serialization
+----------------------------------
+.. autofunction:: save
+.. autofunction:: load
+
+
+Parallelism
+----------------------------------
+.. autofunction:: get_num_threads
+.. autofunction:: set_num_threads
+
+
+Math operations
+----------------------------------
+
+Pointwise Ops
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. autofunction:: abs
+.. autofunction:: acos
+.. autofunction:: add
+.. autofunction:: addcdiv
+.. autofunction:: addcmul
+.. autofunction:: asin
+.. autofunction:: atan
+.. autofunction:: atan2
+.. autofunction:: ceil
+.. autofunction:: clamp
+.. autofunction:: cos
+.. autofunction:: cosh
+.. autofunction:: div
+.. autofunction:: erf
+.. autofunction:: erfinv
+.. autofunction:: exp
+.. autofunction:: floor
+.. autofunction:: fmod
+.. autofunction:: frac
+.. autofunction:: lerp
+.. autofunction:: log
+.. autofunction:: log1p
+.. autofunction:: mul
+.. autofunction:: neg
+.. autofunction:: pow
+.. autofunction:: reciprocal
+.. autofunction:: remainder
+.. autofunction:: round
+.. autofunction:: rsqrt
+.. autofunction:: sigmoid
+.. autofunction:: sign
+.. autofunction:: sin
+.. autofunction:: sinh
+.. autofunction:: sqrt
+.. autofunction:: tan
+.. autofunction:: tanh
+.. autofunction:: trunc
+
+
+Reduction Ops
+~~~~~~~~~~~~~~~~~~~~~~
+.. autofunction:: cumprod
+.. autofunction:: cumsum
+.. autofunction:: dist
+.. autofunction:: mean
+.. autofunction:: median
+.. autofunction:: mode
+.. autofunction:: norm
+.. autofunction:: prod
+.. autofunction:: std
+.. autofunction:: sum
+.. autofunction:: var
+
+
+Comparison Ops
+~~~~~~~~~~~~~~~~~~~~~~
+.. autofunction:: eq
+.. autofunction:: equal
+.. autofunction:: ge
+.. autofunction:: gt
+.. autofunction:: kthvalue
+.. autofunction:: le
+.. autofunction:: lt
+.. autofunction:: max
+.. autofunction:: min
+.. autofunction:: ne
+.. autofunction:: sort
+.. autofunction:: topk
+
+
+Other Operations
+~~~~~~~~~~~~~~~~~~~~~~
+.. autofunction:: cross
+.. autofunction:: diag
+.. autofunction:: histc
+.. autofunction:: renorm
+.. autofunction:: trace
+.. autofunction:: tril
+.. autofunction:: triu
+
+
+BLAS and LAPACK Operations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autofunction:: addbmm
+.. autofunction:: addmm
+.. autofunction:: addmv
+.. autofunction:: addr
+.. autofunction:: baddbmm
+.. autofunction:: bmm
+.. autofunction:: btrifact
+.. autofunction:: btrisolve
+.. autofunction:: dot
+.. autofunction:: eig
+.. autofunction:: gels
+.. autofunction:: geqrf
+.. autofunction:: ger
+.. autofunction:: gesv
+.. autofunction:: inverse
+.. autofunction:: matmul
+.. autofunction:: mm
+.. autofunction:: mv
+.. autofunction:: orgqr
+.. autofunction:: ormqr
+.. autofunction:: potrf
+.. autofunction:: potri
+.. autofunction:: potrs
+.. autofunction:: pstrf
+.. autofunction:: qr
+.. autofunction:: svd
+.. autofunction:: symeig
+.. autofunction:: trtrs

docs/tensor.md

@@ -1,407 +0,0 @@
-# Tensors
-
-A `Tensor` is a potentially multi-dimensional matrix.
-The number of dimensions is unlimited.
-
-The `Tensor` set of classes are probably the most important class in
-`torch`. Almost every package depends on these classes. They are *__the__*
-class for handling numeric data. As with pretty much anything in
-[torch], tensors are serializable with `torch.save` and `torch.load`
-
-There are 7 Tensor classes in torch:
-
-- `torch.FloatTensor`   :   Signed 32-bit floating point tensor
-- `torch.DoubleTensor`  :   Signed 64-bit floating point tensor
-- `torch.ByteTensor`    :   Signed  8-bit integer tensor
-- `torch.CharTensor`    : Unsigned  8-bit integer tensor
-- `torch.ShortTensor`   :   Signed 16-bit integer tensor
-- `torch.IntTensor`     :   Signed 32-bit integer tensor
-- `torch.LongTensor`    :   Signed 64-bit integer tensor
-
-The data in these tensors lives on the system memory connected to your CPU.
-
-Most numeric operations are implemented _only_ for `FloatTensor` and `DoubleTensor`.
-Other Tensor types are useful if you want to save memory space or specifically
-do integer operations.
-
-The number of dimensions of a `Tensor` can be queried by
-`ndimension()` or `dim()`. Size of the `i-th` dimension is
-returned by `size(i)`. A tuple containing the size of all the dimensions
-can be returned by `size()`.
-
-```python
-import torch
-
-# allocate a matrix of shape 3x4
-a = torch.FloatTensor(3, 4)
-print(a)
-
-# convert this into a LongTensor
-b = a.long()
-print(b)
-
-# print the size of the tensor
-print(a.size())
-
-# print the number of dimensions
-print(a.dim())
-```
-
-These tensors can be converted to numpy arrays very efficiently
-with zero memory copies.
-For this, the two provided functions are `.numpy()` and `torch.from_numpy()`
-
-```python
-import numpy as np
-
-# convert to numpy
-c = a.numpy()
-print(type(c))
-```
-
-When using GPUs, each of the classes above has an equivalent
-class such as: `torch.cuda.FloatTensor`, `torch.cuda.LongTensor`, etc.
-When one allocates a CUDA tensor, the data in these tensors lives in the
-GPU memory.
-
-One can seamlessly transfer a tensor from the CPU to the GPU, as well as
-between different GPUs on your machine.
-
-Apart from the above 7 tensor types, there is one additional tensor type on the GPU
-
-- `torch.cuda.HalfTensor` : Signed 16-bit floating point tensor
-
-```python
-import torch.cuda
-
-# allocate a matrix of shape 3x4
-a = torch.cuda.FloatTensor(3, 4)
-print(a)
-
-# transfer this to the CPU
-b = a.cpu()
-print(b)
-
-# transfer this back to the GPU-1
-a = b.cuda()
-print(a)
-
-# transfer this to GPU-2
-b = a.cuda(1)
-```
-
-## Internal data representation
-
-The actual data of a `Tensor` is contained into a
-`Storage`. It can be accessed using
-`storage()`. While the memory of a
-`Tensor` has to be contained in this unique `Storage`, it might
-not be contiguous: the first position used in the `Storage` is given
-by `storage_offset()` (starting at `0`).
-And the _jump_ needed to go from one element to another
-element in the `i-th` dimension is given by
-`stride(i-1)`. See the code example for an illustration.
-
-```python
-# given a 3d tensor
-x = torch.FloatTensor(7,7,7)
-
-# accessing the element `(3,4,5)` can be done by
-x[3 - 1][4 - 1][5 - 1]
-# or equivalently (but slowly!)
-x.storage()[x.storageOffset()
-            + (3 - 1) * x.stride(0)
-			+ (4 - 1) * x.stride(1)
-			+ (5 - 1) * x.stride(2)]
-```
-
-One could say that a `Tensor` is a particular way of _viewing_ a
-`Storage`: a `Storage` only represents a chunk of memory, while the
-`Tensor` interprets this chunk of memory as having dimensions:
-
-```python
-# a tensor interprets a chunk of memory as having dimensions
->>> x = torch.Tensor(4,5)
->>> s = x.storage()
->>> for i in range(s.size()): # fill up the Storage
->>>   s[i] = i
-
-# s is interpreted by x as a 2D matrix
->>> print(x)
-
-  1   2   3   4   5
-  6   7   8   9  10
- 11  12  13  14  15
- 16  17  18  19  20
-[torch.FloatTensor of dimension 4x5]
-```
-
-Note also that in Torch7 ___elements in the same row___ [elements along the __last__ dimension]
-are contiguous in memory for a matrix [tensor]:
-
-This is exactly like in `C` and `numpy` (and not `Fortran`).
-
-## Default Tensor type
-
-For convenience, _an alias_ `torch.Tensor` is provided, which allows the user to write
-type-independent scripts, which can then ran after choosing the desired Tensor type with
-a call like
-
-`torch.set_default_tensor_type('torch.DoubleTensor')`
-
-
-By default, the alias points to `torch.FloatTensor`.
-
-## Efficient memory management
-
-_All_ tensor operations post-fixed with an underscore (for example `.fill_`)
-do _not_ make any memory copy. All these methods transform the existing tensor.
-Tensor methods such as `narrow` and `select` return a new tensor referencing _the same storage_.
-This magical behavior is internally obtained by good usage of the `stride()` and
-`storage_offset()`. See the code example illustrating this.
-
-```python
->>> x = torch.Tensor(5).zero_()
->>> print(x)
-0
-0
-0
-0
-0
-[torch.FloatTensor of dimension 5]
->>> x.narrow(0, 1, 2).fill_(1)
->>> # narrow() returns a Tensor referencing the same Storage as x
->>> print(x)
- 0
- 1
- 1
- 1
- 0
-[torch.FloatTensor of dimension 5]
-```
-
-If you really need to copy a `Tensor`, you can use the `copy_()` method:
-
-```python
-# making a copy of a tensor
-y = x.new(x.size()).copy_(x)
-y = x.clone()
-```
-Or the convenience method `clone()`
-
-We now describe all the methods for `Tensor`. If you want to specify the Tensor type,
-just replace `Tensor` by the name of the Tensor variant (like `CharTensor`).
-
-## Constructors ##
-
-Tensor constructors, create new Tensor object, optionally, allocating
-new memory. By default the elements of a newly allocated memory are
-not initialized, therefore, might contain arbitrary numbers. Here are
-several ways to construct a new `Tensor`.
-
-### torch.Tensor() ###
-
-Returns an empty tensor.
-
-### torch.Tensor(tensor) ###
-
-Returns a new tensor which reference the same `Storage` than the given `tensor`.
-The `size`, `stride`, and `storage_offset` are the same than the given tensor.
-
-The new `Tensor` is now going to "view" the same `storage`
-as the given `tensor`. As a result, any modification in the elements
-of the `Tensor` will have a impact on the elements of the given
-`tensor`, and vice-versa. No memory copy!
-
-```python
->>> x = torch.Tensor(2,5).fill_(3.14)
->>> x
- 3.1400  3.1400  3.1400  3.1400  3.1400
- 3.1400  3.1400  3.1400  3.1400  3.1400
-[torch.FloatTensor of dimension 2x5]
-
->>> y = torch.Tensor(x)
->>> y
- 3.1400  3.1400  3.1400  3.1400  3.1400
- 3.1400  3.1400  3.1400  3.1400  3.1400
-[torch.FloatTensor of dimension 2x5]
-
->>> y.zero_()
->>> x # elements of x are the same as y!
-0 0 0 0 0
-0 0 0 0 0
-[torch.FloatTensor of dimension 2x5]
-```
-
-### torch.Tensor(sz1 [,sz2 [,sz3 [,sz4 [,sz5 ...]]]]]) ###
-
-Create a tensor of the given sizes.
-The tensor size will be `sz1 x sz2 x sx3 x sz4 x sz5 x ...`.
-
-### torch.Tensor(sizes) ###
-
-Create a tensor of any number of dimensions. `sizes` gives the size in each dimension of
-the tensor and is of type `torch.Size`. 
-
-```python
-Example, create a 4D 4x4x3x2 tensor:
-x = torch.Tensor(torch.Size([4,4,3,2]))
-```
-
-### torch.Tensor(storage) ###
-
-Returns a tensor which uses the existing `Storage` starting at a storage offset of 0.
-
-### torch.Tensor(sequence) ###
-
-One can create a tensor from a python sequence.
-
-For example, you can create a `Tensor` from a `list` or a `tuple`
-
-```python
-# create a 2d tensor from a list of lists
->>> torch.Tensor([[1,2,3,4], [5,6,7,8]])
- 1  2  3  4
- 5  6  7  8
-[torch.FloatTensor of dimension 2x4]
-```
-
-### torch.Tensor(ndarray) ###
-
-Creates a `Tensor` from a NumPy `ndarray`.
-If the `dtype` of the `ndarray` is the same as the type of the `Tensor` being created,
-The underlying memory of both are shared, i.e. if the value of an element
-in the `ndarray` is changed, the corresponding value in the `Tensor` changes,
-and vice versa.
-
-```python
-# create a ndarray of dtype=int64
->>> a = np.random.randint(2, size=10)
->>> a
-array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0])
-# create a LongTensor. Since they are the same type (int64), the memory is shared
->>> b = torch.LongTensor(a)
- 0
- 0
- 1
- 1
- 0
- 1
- 1
- 0
- 0
- 0
-[torch.LongTensor of size 10]
->>> b[3] = 100
->>> print(a[3])
-100
-
-# now create an IntTensor from the same ndarray.
-# The memory is not shared in this case as the dtype=int64 != IntTensor (int32)
->>> b = torch.IntTensor(a)
->>> b[3] = 30000
->>> print(a[3])
-100
-# a did not change to the value 30000
-```
-
-## NumPy Conversion ##
-### torch.from_numpy(ndarray)
-
-This is a convenience function similar to the constructor above.
-Given a numpy `ndarray`, it constructs a torch `Tensor` of the same `dtype`
-as the numpy array.
-
-For example, passing in an ndarray of dtype=float64 will create a torch.DoubleTensor
-
-### Tensor.numpy()
-
-This is a member function on a tensor that converts a torch `Tensor` to a
-numpy `ndarray`. The memory of the data of both objects is shared.
-Hence, changing a value in the `Tensor` will change the corresponding value in
-the `ndarray` and vice versa.
-
-```python
->>> a = torch.randn(3,4)
->>> b = a.numpy() # creates a numpy array with dtype=float32 in this case
->>> print(a)
--1.0453  1.4730 -1.8990 -0.7763
- 1.8155  1.4004 -1.5286  1.0420
- 0.6551  1.0258  0.1152 -0.3239
-[torch.FloatTensor of size 3x4]
->>> print(b)
-[[-1.04525673  1.4730444  -1.89899576 -0.77626842]
- [ 1.81549406  1.40035892 -1.5286355   1.04199517]
- [ 0.6551016   1.02575183  0.11520521 -0.32391372]]
->>> a[2][2] = 1000
->>> print(b)
-[[ -1.04525673e+00   1.47304440e+00  -1.89899576e+00  -7.76268423e-01]
- [  1.81549406e+00   1.40035892e+00  -1.52863550e+00   1.04199517e+00]
- [  6.55101597e-01   1.02575183e+00   1.00000000e+03  -3.23913723e-01]]
-# notice that b[2][2] has changed to the value 1000 too.
-```
-
-### torch.is_tensor(obj)
-
-Returns True if the passed-in object is a `Tensor` (of any type). Returns `False` otherwise.
-
-### torch.is_storage
-
-Returns True if the passed-in object is a `Storage` (of any type). Returns `False` otherwise.
-
-### torch.expand_as
-### torch.expand
-### torch.view
-### torch.view_as
-### torch.permute
-### torch.pin_memory
-### copy
-### split
-### chunk
-### tolist
-### repeat
-### unsqueeze
-### unsqueeze_
-### add, iadd, sub, isub, mul, imul, matmul, div, rdiv, idiv, mod, neg
-
-## GPU Semantics ##
-
-When you create a `torch.cuda.*Tensor`, it is allocated on the current GPU.
-However, you could allocate it on another GPU as well, using the `with torch.cuda.device(id)` context.
-All allocations within this context will be placed on the GPU `id`. 
-
-Once `Tensor`s are allocated, you can do operations on them from any GPU context, and the results will be placed on the same device as where the source `Tensor` is located.
-
-For example if Tensor `a` and `b` are on GPU-2, but the GPU-1 is the current device.
-If one does `c = a + b`, then `c` will be on GPU-2, regardless of what the current device is.
-
-Cross-GPU operations are not allowed. The only Cross-GPU operation allowed is `copy`.
-
-If `a` is on GPU-1 and `b` is on GPU-2, then `c = a + b` will result in an error.
-
-See the example for more clarity on these semantics.
-
-```python
-# Tensors are allocated on GPU 1 by default
-x = torch.cuda.FloatTensor(1)
-# x.get_device() == 0
-y = torch.FloatTensor(1).cuda()
-# y.get_device() == 0
-
-with torch.cuda.device(1):
-    # allocates a tensor on GPU 2
-    a = torch.cuda.FloatTensor(1)
-
-    # transfers a tensor from CPU to GPU-2
-	b = torch.FloatTensor(1).cuda()
-	# a.get_device() == b.get_device() == 1
-
-    z = x + y
-	# z.get_device() == 1
-
-    # even within a context, you can give a GPU id to the .cuda call
-    c = torch.randn(2).cuda(2)
-	# c.get_device() == 2
-	
-```
-

docs/torch.md

@@ -1,83 +0,0 @@
-# torch
-
-```python
-# load torch with
-import torch
-```
-
-```python
-# load the CUDA features of torch with
-import torch.cuda
-```
-
-__torch__ is the main package where data structures for multi-dimensional
-tensors and mathematical operations over these are defined.
-Additionally, it provides many utilities for efficient serializing of
-Tensors and arbitrary types, and other useful utilities.
-
-It has a CUDA counterpart, that enables you to run your tensor computations
-on an NVIDIA GPU with compute capability >= 2.0.
-
-## Multi-core
-### torch.get_num_threads()
-
-Gets the number of OpenMP threads that will be used for parallelizing CPU operations
-
-### torch.set_num_threads(n)
-
-Sets the number of OpenMP threads to use for parallelizing CPU operations
-
-## Serialization
-### torch.save(object, file)
-This function pickles a Python object to the `file`. `file` is either a filename or a file handle.
-
-`object` can be a picklable python object, including `torch` `Tensor`s, autograd `Variable`, nn `Module`s etc.
-
-When a group of `torch` `Tensor`s are saved together, and if any of them share the same storages, then this sharing is preserved during saving and loading back.
-
-
-### torch.load(file)
-
-This function unpickles objects that have been pickled with `torch.save`
-
-## Random Numbers
-
-### torch.get_rng_state()
-
-Gets the current state of the torch Random Number Generator.
-
-This can be passed in the future to `torch.set_rng_state` to restore the current RNG state.
-
-### torch.set_rng_state(state)
-
-Sets the current state of the torch Random Number Generator to the given `state`. 
-
-### torch.manual_seed(number)
-
-Sets the initial seed of the random number generator to a given number.
-
-### torch.initial_seed()
-
-Returns the number that is the initial seed to the Random Number Generator
-
-## CUDA
-### torch.cuda.is_available()
-
-Returns `True` if CUDA is available and usable. Returns `False` otherwise.
-
-### torch.cuda.device_count()
-
-Returns the number of CUDA devices on the system.
-
-### torch.cuda.current_device()
-
-Returns the device index of the current default CUDA device.
-
-### torch.cuda.synchronize()
-
-This function issues a `cudaDeviceSynchronize` on the current device, and hence waits for all in-flight CUDA computation to finish.
-
-### torch.cuda.current_stream()
-
-Returns the handle to the current stream of the CUDA context.
-

setup.py

@@ -1,6 +1,9 @@
 from setuptools import setup, Extension, distutils, Command, find_packages
 import setuptools.command.build_ext
 import setuptools.command.install
+import setuptools.command.develop
+import setuptools.command.build_py
+import distutils.unixccompiler
 import distutils.command.build
 import distutils.command.clean
 import platform
@@ -9,36 +12,100 @@ import shutil
 import sys
 import os
 
-CUDA_HOME = os.getenv('CUDA_HOME', '/usr/local/cuda')
-WITH_CUDA = os.path.exists(CUDA_HOME)
-WITH_CUDNN = WITH_CUDA
-DEBUG = False
+from tools.setup_helpers.env import check_env_flag
+from tools.setup_helpers.cuda import WITH_CUDA, CUDA_HOME, CUDA_VERSION
+from tools.setup_helpers.cudnn import WITH_CUDNN, CUDNN_LIB_DIR, CUDNN_INCLUDE_DIR
+from tools.setup_helpers.nccl import WITH_NCCL, WITH_SYSTEM_NCCL, NCCL_LIB_DIR, \
+    NCCL_INCLUDE_DIR, NCCL_ROOT_DIR, NCCL_SYSTEM_LIB
+from tools.setup_helpers.nnpack import WITH_NNPACK, NNPACK_LIB_PATHS, \
+    NNPACK_INCLUDE_DIRS
+from tools.setup_helpers.split_types import split_types
+
+DEBUG = check_env_flag('DEBUG')
+WITH_DISTRIBUTED = not check_env_flag('NO_DISTRIBUTED')
+WITH_DISTRIBUTED_MW = WITH_DISTRIBUTED and check_env_flag('WITH_DISTRIBUTED_MW')
+
+
+################################################################################
+# Workaround setuptools -Wstrict-prototypes warnings
+# I lifted this code from https://stackoverflow.com/a/29634231/23845
+################################################################################
+import distutils.sysconfig
+cfg_vars = distutils.sysconfig.get_config_vars()
+for key, value in cfg_vars.items():
+    if type(value) == str:
+        cfg_vars[key] = value.replace("-Wstrict-prototypes", "")
 
 ################################################################################
 # Monkey-patch setuptools to compile in parallel
 ################################################################################
+original_link = distutils.unixccompiler.UnixCCompiler.link
 
-def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None):
+
+def parallelCCompile(self, sources, output_dir=None, macros=None,
+                     include_dirs=None, debug=0, extra_preargs=None,
+                     extra_postargs=None, depends=None):
     # those lines are copied from distutils.ccompiler.CCompiler directly
-    macros, objects, extra_postargs, pp_opts, build = self._setup_compile(output_dir, macros, include_dirs, sources, depends, extra_postargs)
+    macros, objects, extra_postargs, pp_opts, build = self._setup_compile(
+        output_dir, macros, include_dirs, sources, depends, extra_postargs)
     cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
 
     # compile using a thread pool
     import multiprocessing.pool
+
     def _single_compile(obj):
         src, ext = build[obj]
         self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
     num_jobs = multiprocessing.cpu_count()
+    max_jobs = os.getenv("MAX_JOBS")
+    if max_jobs is not None:
+        num_jobs = min(num_jobs, int(max_jobs))
     multiprocessing.pool.ThreadPool(num_jobs).map(_single_compile, objects)
 
     return objects
 
+
+def patched_link(self, *args, **kwargs):
+    _cxx = self.compiler_cxx
+    self.compiler_cxx = None
+    result = original_link(self, *args, **kwargs)
+    self.compiler_cxx = _cxx
+    return result
+
+
 distutils.ccompiler.CCompiler.compile = parallelCCompile
+distutils.unixccompiler.UnixCCompiler.link = patched_link
 
 ################################################################################
 # Custom build commands
 ################################################################################
 
+dep_libs = [
+    'TH', 'THS', 'THNN', 'THC', 'THCS', 'THCUNN', 'nccl', 'libshm',
+    'ATen', 'gloo', 'THD', 'nanopb',
+]
+
+
+def build_libs(libs):
+    for lib in libs:
+        assert lib in dep_libs, 'invalid lib: {}'.format(lib)
+    build_libs_cmd = ['bash', 'torch/lib/build_libs.sh']
+    my_env = os.environ.copy()
+    my_env["PYTORCH_PYTHON"] = sys.executable
+    if WITH_SYSTEM_NCCL:
+        my_env["NCCL_ROOT_DIR"] = NCCL_ROOT_DIR
+    if WITH_CUDA:
+        my_env["CUDA_BIN_PATH"] = CUDA_HOME
+        build_libs_cmd += ['--with-cuda']
+
+    if subprocess.call(build_libs_cmd + libs, env=my_env) != 0:
+        sys.exit(1)
+
+    if 'THNN' in libs or 'THCUNN' in libs:
+        from tools.nnwrap import generate_wrappers as generate_nn_wrappers
+        generate_nn_wrappers()
+
+
 class build_deps(Command):
     user_options = []
 
@@ -49,13 +116,30 @@ class build_deps(Command):
         pass
 
     def run(self):
-        from tools.nnwrap import generate_wrappers as generate_nn_wrappers
-        build_all_cmd = ['bash', 'torch/lib/build_all.sh']
+        libs = ['TH', 'THS', 'THNN']
         if WITH_CUDA:
-            build_all_cmd += ['--with-cuda']
-        if subprocess.call(build_all_cmd) != 0:
-            sys.exit(1)
-        generate_nn_wrappers()
+            libs += ['THC', 'THCS', 'THCUNN']
+        if WITH_NCCL and not WITH_SYSTEM_NCCL:
+            libs += ['nccl']
+        libs += ['libshm', 'ATen', 'nanopb']
+        if WITH_DISTRIBUTED:
+            if sys.platform.startswith('linux'):
+                libs += ['gloo']
+            libs += ['THD']
+        build_libs(libs)
+
+
+build_dep_cmds = {}
+
+for lib in dep_libs:
+    # wrap in function to capture lib
+    class build_dep(build_deps):
+        description = 'Build {} external library'.format(lib)
+
+        def run(self):
+            build_libs([self.lib])
+    build_dep.lib = lib
+    build_dep_cmds['build_' + lib.lower()] = build_dep
 
 
 class build_module(Command):
@@ -72,8 +156,86 @@ class build_module(Command):
         self.run_command('build_ext')
 
 
-class build_ext(setuptools.command.build_ext.build_ext):
+class build_py(setuptools.command.build_py.build_py):
+
     def run(self):
+        self.create_version_file()
+        setuptools.command.build_py.build_py.run(self)
+
+    @staticmethod
+    def create_version_file():
+        global version, cwd
+        print('-- Building version ' + version)
+        version_path = os.path.join(cwd, 'torch', 'version.py')
+        with open(version_path, 'w') as f:
+            f.write("__version__ = '{}'\n".format(version))
+            # NB: This is not 100% accurate, because you could have built the
+            # library code with DEBUG, but csrc without DEBUG (in which case
+            # this would claim to be a release build when it's not.)
+            f.write("debug = {}\n".format(repr(DEBUG)))
+            f.write("cuda = {}\n".format(repr(CUDA_VERSION)))
+
+
+class develop(setuptools.command.develop.develop):
+
+    def run(self):
+        build_py.create_version_file()
+        setuptools.command.develop.develop.run(self)
+
+
+def monkey_patch_THD_link_flags():
+    '''
+    THD's dynamic link deps are not determined until after build_deps is run
+    So, we need to monkey-patch them in later
+    '''
+    # read tmp_install_path/THD_deps.txt for THD's dynamic linkage deps
+    with open(tmp_install_path + '/THD_deps.txt', 'r') as f:
+        thd_deps_ = f.read()
+    thd_deps = []
+    # remove empty lines
+    for l in thd_deps_.split(';'):
+        if l != '':
+            thd_deps.append(l)
+
+    C.extra_link_args += thd_deps
+
+
+class build_ext(setuptools.command.build_ext.build_ext):
+
+    def run(self):
+        # Print build options
+        if WITH_NUMPY:
+            print('-- Building with NumPy bindings')
+        else:
+            print('-- NumPy not found')
+        if WITH_CUDNN:
+            print('-- Detected cuDNN at ' + CUDNN_LIB_DIR + ', ' + CUDNN_INCLUDE_DIR)
+        else:
+            print('-- Not using cuDNN')
+        if WITH_CUDA:
+            print('-- Detected CUDA at ' + CUDA_HOME)
+        else:
+            print('-- Not using CUDA')
+        if WITH_NCCL and WITH_SYSTEM_NCCL:
+            print('-- Using system provided NCCL library at ' +
+                  NCCL_SYSTEM_LIB + ', ' + NCCL_INCLUDE_DIR)
+        elif WITH_NCCL:
+            print('-- Building NCCL library')
+        else:
+            print('-- Not using NCCL')
+        if WITH_DISTRIBUTED:
+            print('-- Building with distributed package ')
+            monkey_patch_THD_link_flags()
+        else:
+            print('-- Building without distributed package')
+
+        # Do we actually need this here?
+        if WITH_NNPACK:
+            nnpack_dir = NNPACK_LIB_PATHS[0]
+            print('-- Detected NNPACK at ' + nnpack_dir)
+        else:
+            print('-- Not using NNPACK')
+
         # cwrap depends on pyyaml, so we can't import it earlier
         from tools.cwrap import cwrap
         from tools.cwrap.plugins.THPPlugin import THPPlugin
@@ -83,13 +245,34 @@ class build_ext(setuptools.command.build_ext.build_ext):
         from tools.cwrap.plugins.KwargsPlugin import KwargsPlugin
         from tools.cwrap.plugins.NullableArguments import NullableArguments
         from tools.cwrap.plugins.CuDNNPlugin import CuDNNPlugin
+        from tools.cwrap.plugins.WrapDim import WrapDim
+        from tools.cwrap.plugins.AssertNDim import AssertNDim
+        from tools.cwrap.plugins.Broadcast import Broadcast
+        from tools.cwrap.plugins.ProcessorSpecificPlugin import ProcessorSpecificPlugin
+        from tools.autograd.gen_variable_type import gen_variable_type
+        from tools.jit.gen_jit_dispatch import gen_jit_dispatch
+        thp_plugin = THPPlugin()
         cwrap('torch/csrc/generic/TensorMethods.cwrap', plugins=[
-            BoolOption(), THPPlugin(), AutoGPU(condition='IS_CUDA'),
-            ArgcountSortPlugin(), KwargsPlugin(),
+            ProcessorSpecificPlugin(), BoolOption(), thp_plugin,
+            AutoGPU(condition='IS_CUDA'), ArgcountSortPlugin(), KwargsPlugin(),
+            AssertNDim(), WrapDim(), Broadcast()
         ])
         cwrap('torch/csrc/cudnn/cuDNN.cwrap', plugins=[
             CuDNNPlugin(), NullableArguments()
         ])
+        # Build ATen based Variable classes
+        autograd_gen_dir = 'torch/csrc/autograd/generated'
+        jit_gen_dir = 'torch/csrc/jit/generated'
+        for d in (autograd_gen_dir, jit_gen_dir):
+            if not os.path.exists(d):
+                os.mkdir(d)
+        gen_variable_type(
+            'torch/lib/build/ATen/ATen/Declarations.yaml',
+            autograd_gen_dir)
+        gen_jit_dispatch(
+            'torch/lib/build/ATen/ATen/Declarations.yaml',
+            jit_gen_dir)
+
         # It's an old-style class in Python 2.7...
         setuptools.command.build_ext.build_ext.run(self)
 
@@ -101,6 +284,7 @@ class build(distutils.command.build.build):
 
 
 class install(setuptools.command.install.install):
+
     def run(self):
         if not self.skip_build:
             self.run_command('build_deps')
@@ -108,6 +292,7 @@ class install(setuptools.command.install.install):
 
 
 class clean(distutils.command.clean.clean):
+
     def run(self):
         import glob
         with open('.gitignore', 'r') as f:
@@ -123,57 +308,164 @@ class clean(distutils.command.clean.clean):
         distutils.command.clean.clean.run(self)
 
 
-
 ################################################################################
 # Configure compile flags
 ################################################################################
 
 include_dirs = []
+library_dirs = []
 extra_link_args = []
-extra_compile_args = ['-std=c++11', '-Wno-write-strings']
-if os.getenv('PYTORCH_BINARY_BUILD') and platform.system() == 'Linux':
-    print('PYTORCH_BINARY_BUILD found. Static linking libstdc++ on Linux')
-    extra_compile_args += ['-static-libstdc++']
-    extra_link_args += ['-static-libstdc++']
+extra_compile_args = ['-std=c++11', '-Wno-write-strings',
+                      # Python 2.6 requires -fno-strict-aliasing, see
+                      # http://legacy.python.org/dev/peps/pep-3123/
+                      '-fno-strict-aliasing',
+                      # Clang has an unfixed bug leading to spurious missing
+                      # braces warnings, see
+                      # https://bugs.llvm.org/show_bug.cgi?id=21629
+                      '-Wno-missing-braces']
 
 cwd = os.path.dirname(os.path.abspath(__file__))
 lib_path = os.path.join(cwd, "torch", "lib")
 
+
+# Check if you remembered to check out submodules
+def check_file(f):
+    if not os.path.exists(f):
+        print("Could not find {}".format(f))
+        print("Did you run 'git submodule update --init'?")
+        sys.exit(1)
+check_file(os.path.join(lib_path, "gloo", "CMakeLists.txt"))
+check_file(os.path.join(lib_path, "nanopb", "CMakeLists.txt"))
+check_file(os.path.join(lib_path, "pybind11", "CMakeLists.txt"))
+
 tmp_install_path = lib_path + "/tmp_install"
 include_dirs += [
     cwd,
     os.path.join(cwd, "torch", "csrc"),
+    lib_path + "/pybind11/include",
     tmp_install_path + "/include",
     tmp_install_path + "/include/TH",
+    tmp_install_path + "/include/THNN",
+    tmp_install_path + "/include/ATen",
 ]
 
-extra_link_args.append('-L' + lib_path)
+library_dirs.append(lib_path)
+
+# we specify exact lib names to avoid conflict with lua-torch installs
+TH_LIB = os.path.join(lib_path, 'libTH.so.1')
+THS_LIB = os.path.join(lib_path, 'libTHS.so.1')
+THC_LIB = os.path.join(lib_path, 'libTHC.so.1')
+THCS_LIB = os.path.join(lib_path, 'libTHCS.so.1')
+THNN_LIB = os.path.join(lib_path, 'libTHNN.so.1')
+THCUNN_LIB = os.path.join(lib_path, 'libTHCUNN.so.1')
+ATEN_LIB = os.path.join(lib_path, 'libATen.so.1')
+THD_LIB = os.path.join(lib_path, 'libTHD.a')
+NCCL_LIB = os.path.join(lib_path, 'libnccl.so.1')
+if platform.system() == 'Darwin':
+    TH_LIB = os.path.join(lib_path, 'libTH.1.dylib')
+    THS_LIB = os.path.join(lib_path, 'libTHS.1.dylib')
+    THC_LIB = os.path.join(lib_path, 'libTHC.1.dylib')
+    THCS_LIB = os.path.join(lib_path, 'libTHCS.1.dylib')
+    THNN_LIB = os.path.join(lib_path, 'libTHNN.1.dylib')
+    THCUNN_LIB = os.path.join(lib_path, 'libTHCUNN.1.dylib')
+    ATEN_LIB = os.path.join(lib_path, 'libATen.1.dylib')
+    NCCL_LIB = os.path.join(lib_path, 'libnccl.1.dylib')
+
+# static library only
+NANOPB_STATIC_LIB = os.path.join(lib_path, 'libprotobuf-nanopb.a')
 
 main_compile_args = ['-D_THP_CORE']
-main_libraries = ['TH', 'shm']
+main_libraries = ['shm']
+main_link_args = [TH_LIB, THS_LIB, THNN_LIB, ATEN_LIB, NANOPB_STATIC_LIB]
 main_sources = [
+    "torch/csrc/PtrWrapper.cpp",
     "torch/csrc/Module.cpp",
     "torch/csrc/Generator.cpp",
     "torch/csrc/Size.cpp",
     "torch/csrc/Exceptions.cpp",
-    "torch/csrc/Tensor.cpp",
     "torch/csrc/Storage.cpp",
+    "torch/csrc/DynamicTypes.cpp",
+    "torch/csrc/assertions.cpp",
     "torch/csrc/byte_order.cpp",
     "torch/csrc/utils.cpp",
+    "torch/csrc/expand_utils.cpp",
+    "torch/csrc/utils/invalid_arguments.cpp",
+    "torch/csrc/utils/object_ptr.cpp",
+    "torch/csrc/utils/python_arg_parser.cpp",
+    "torch/csrc/utils/tuple_parser.cpp",
     "torch/csrc/allocators.cpp",
     "torch/csrc/serialization.cpp",
+    "torch/csrc/jit/init.cpp",
+    "torch/csrc/jit/ir.cpp",
+    "torch/csrc/jit/python_ir.cpp",
+    "torch/csrc/jit/test_jit.cpp",
+    "torch/csrc/jit/tracer.cpp",
+    "torch/csrc/jit/python_tracer.cpp",
+    "torch/csrc/jit/interned_strings.cpp",
+    "torch/csrc/jit/type.cpp",
+    "torch/csrc/jit/export.cpp",
+    "torch/csrc/jit/passes/graph_fuser.cpp",
+    "torch/csrc/jit/passes/onnx.cpp",
+    "torch/csrc/jit/passes/dead_code_elimination.cpp",
+    "torch/csrc/jit/passes/common_subexpression_elimination.cpp",
+    "torch/csrc/jit/passes/peephole.cpp",
+    "torch/csrc/jit/passes/onnx/peephole.cpp",
+    "torch/csrc/jit/generated/aten_dispatch.cpp",
     "torch/csrc/autograd/init.cpp",
-    "torch/csrc/autograd/variable.cpp",
-    "torch/csrc/autograd/function.cpp",
     "torch/csrc/autograd/engine.cpp",
+    "torch/csrc/autograd/function.cpp",
+    "torch/csrc/autograd/variable.cpp",
+    "torch/csrc/autograd/saved_variable.cpp",
+    "torch/csrc/autograd/input_buffer.cpp",
+    "torch/csrc/autograd/profiler.cpp",
+    "torch/csrc/autograd/python_function.cpp",
+    "torch/csrc/autograd/python_cpp_function.cpp",
+    "torch/csrc/autograd/python_variable.cpp",
+    "torch/csrc/autograd/python_engine.cpp",
+    "torch/csrc/autograd/python_hook.cpp",
+    "torch/csrc/autograd/functions/jit_closure.cpp",
+    "torch/csrc/autograd/generated/VariableType.cpp",
+    "torch/csrc/autograd/generated/Functions.cpp",
+    "torch/csrc/autograd/generated/python_variable_methods.cpp",
+    "torch/csrc/autograd/generated/python_functions.cpp",
+    "torch/csrc/autograd/generated/python_nn_functions.cpp",
+    "torch/csrc/autograd/functions/batch_normalization.cpp",
+    "torch/csrc/autograd/functions/convolution.cpp",
+    "torch/csrc/autograd/functions/basic_ops.cpp",
+    "torch/csrc/autograd/functions/tensor.cpp",
+    "torch/csrc/autograd/functions/accumulate_grad.cpp",
+    "torch/csrc/autograd/functions/special.cpp",
+    "torch/csrc/autograd/functions/utils.cpp",
+    "torch/csrc/autograd/functions/init.cpp",
+    "torch/csrc/autograd/functions/onnx/convolution.cpp",
+    "torch/csrc/autograd/functions/onnx/batch_normalization.cpp",
+    "torch/csrc/autograd/functions/onnx/basic_ops.cpp",
+    "torch/csrc/onnx/onnx.pb.cpp",
+    "torch/csrc/onnx/onnx.cpp",
 ]
+main_sources += split_types("torch/csrc/Tensor.cpp")
 
 try:
     import numpy as np
     include_dirs += [np.get_include()]
     extra_compile_args += ['-DWITH_NUMPY']
+    WITH_NUMPY = True
 except ImportError:
-    pass
+    WITH_NUMPY = False
+
+if WITH_DISTRIBUTED:
+    extra_compile_args += ['-DWITH_DISTRIBUTED']
+    main_sources += [
+        "torch/csrc/distributed/Module.cpp",
+    ]
+    if WITH_DISTRIBUTED_MW:
+        main_sources += [
+            "torch/csrc/distributed/Tensor.cpp",
+            "torch/csrc/distributed/Storage.cpp",
+        ]
+        extra_compile_args += ['-DWITH_DISTRIBUTED_MW']
+    include_dirs += [tmp_install_path + "/include/THD"]
+    main_link_args += [THD_LIB]
 
 if WITH_CUDA:
     cuda_lib_dirs = ['lib64', 'lib']
@@ -183,37 +475,77 @@ if WITH_CUDA:
         if os.path.exists(cuda_lib_path):
             break
     include_dirs.append(cuda_include_path)
-    extra_link_args.append('-L' + cuda_lib_path)
+    include_dirs.append(tmp_install_path + "/include/THCUNN")
+    library_dirs.append(cuda_lib_path)
     extra_link_args.append('-Wl,-rpath,' + cuda_lib_path)
     extra_compile_args += ['-DWITH_CUDA']
     extra_compile_args += ['-DCUDA_LIB_PATH=' + cuda_lib_path]
-    main_libraries += ['THC']
+    main_libraries += ['cudart', 'nvToolsExt']
+    main_link_args += [THC_LIB, THCS_LIB, THCUNN_LIB]
     main_sources += [
         "torch/csrc/cuda/Module.cpp",
         "torch/csrc/cuda/Storage.cpp",
         "torch/csrc/cuda/Stream.cpp",
-        "torch/csrc/cuda/Tensor.cpp",
         "torch/csrc/cuda/AutoGPU.cpp",
         "torch/csrc/cuda/utils.cpp",
+        "torch/csrc/cuda/expand_utils.cpp",
         "torch/csrc/cuda/serialization.cpp",
+        "torch/csrc/jit/fusion_compiler.cpp",
     ]
+    main_sources += split_types("torch/csrc/cuda/Tensor.cpp")
 
+if WITH_NCCL:
+    if WITH_SYSTEM_NCCL:
+        main_link_args += [NCCL_SYSTEM_LIB]
+        include_dirs.append(NCCL_INCLUDE_DIR)
+    else:
+        main_link_args += [NCCL_LIB]
+    extra_compile_args += ['-DWITH_NCCL']
+    main_sources += [
+        "torch/csrc/cuda/nccl.cpp",
+    ]
 if WITH_CUDNN:
     main_libraries += ['cudnn']
+    library_dirs.append(CUDNN_LIB_DIR)
+    # NOTE: these are at the front, in case there's another cuDNN in CUDA path
+    include_dirs.insert(0, CUDNN_INCLUDE_DIR)
+    extra_link_args.insert(0, '-Wl,-rpath,' + CUDNN_LIB_DIR)
     main_sources += [
-        "torch/csrc/cudnn/Module.cpp",
+        "torch/csrc/cudnn/BatchNorm.cpp",
         "torch/csrc/cudnn/Conv.cpp",
         "torch/csrc/cudnn/cuDNN.cpp",
+        "torch/csrc/cudnn/GridSampler.cpp",
+        "torch/csrc/cudnn/AffineGridGenerator.cpp",
         "torch/csrc/cudnn/Types.cpp",
         "torch/csrc/cudnn/Handles.cpp",
-        "torch/csrc/cudnn/CppWrapper.cpp",
     ]
     extra_compile_args += ['-DWITH_CUDNN']
 
+if WITH_NNPACK:
+    include_dirs.extend(NNPACK_INCLUDE_DIRS)
+    main_link_args.extend(NNPACK_LIB_PATHS)
+    main_sources += [
+        "torch/csrc/nnpack/NNPACK.cpp",
+    ]
+    extra_compile_args += ['-DWITH_NNPACK']
+
 if DEBUG:
     extra_compile_args += ['-O0', '-g']
     extra_link_args += ['-O0', '-g']
 
+if os.getenv('PYTORCH_BINARY_BUILD') and platform.system() == 'Linux':
+    print('PYTORCH_BINARY_BUILD found. Static linking libstdc++ on Linux')
+    # get path of libstdc++ and link manually.
+    # for reasons unknown, -static-libstdc++ doesn't fully link some symbols
+    CXXNAME = os.getenv('CXX', 'g++')
+    STDCPP_LIB = subprocess.check_output([CXXNAME, '-print-file-name=libstdc++.a'])
+    STDCPP_LIB = STDCPP_LIB[:-1]
+    if type(STDCPP_LIB) != str:  # python 3
+        STDCPP_LIB = STDCPP_LIB.decode(sys.stdout.encoding)
+    extra_link_args += [STDCPP_LIB]
+    version_script = os.path.abspath("tools/pytorch.version")
+    extra_link_args += ['-Wl,--version-script=' + version_script]
+
 
 def make_relative_rpath(path):
     if platform.system() == 'Darwin':
@@ -226,7 +558,7 @@ def make_relative_rpath(path):
 ################################################################################
 
 extensions = []
-packages = find_packages(exclude=('tools.*',))
+packages = find_packages(exclude=('tools', 'tools.*',))
 
 C = Extension("torch._C",
               libraries=main_libraries,
@@ -234,7 +566,8 @@ C = Extension("torch._C",
               language='c++',
               extra_compile_args=main_compile_args + extra_compile_args,
               include_dirs=include_dirs,
-    extra_link_args=extra_link_args + [make_relative_rpath('lib')]
+              library_dirs=library_dirs,
+              extra_link_args=extra_link_args + main_link_args + [make_relative_rpath('lib')],
               )
 extensions.append(C)
 
@@ -245,42 +578,86 @@ DL = Extension("torch._dl",
 extensions.append(DL)
 
 THNN = Extension("torch._thnn._THNN",
-    libraries=['TH', 'THNN'],
                  sources=['torch/csrc/nn/THNN.cpp'],
                  language='c++',
                  extra_compile_args=extra_compile_args,
                  include_dirs=include_dirs,
-    extra_link_args=extra_link_args + [make_relative_rpath('../lib')]
+                 extra_link_args=extra_link_args + [
+                     TH_LIB,
+                     THNN_LIB,
+                     make_relative_rpath('../lib'),
+                 ]
                  )
 extensions.append(THNN)
 
 if WITH_CUDA:
+    thnvrtc_link_flags = extra_link_args + [make_relative_rpath('lib')]
+    if platform.system() == 'Linux':
+        thnvrtc_link_flags = thnvrtc_link_flags + ['-Wl,--no-as-needed']
+    # these have to be specified as -lcuda in link_flags because they
+    # have to come right after the `no-as-needed` option
+    thnvrtc_link_flags += ['-lcuda', '-lnvrtc']
+    THNVRTC = Extension("torch._nvrtc",
+                        sources=['torch/csrc/nvrtc.cpp'],
+                        language='c++',
+                        include_dirs=include_dirs,
+                        library_dirs=library_dirs + [cuda_lib_path + '/stubs'],
+                        extra_link_args=thnvrtc_link_flags,
+                        )
+    extensions.append(THNVRTC)
+
     THCUNN = Extension("torch._thnn._THCUNN",
-        libraries=['TH', 'THC', 'THCUNN'],
                        sources=['torch/csrc/nn/THCUNN.cpp'],
                        language='c++',
                        extra_compile_args=extra_compile_args,
                        include_dirs=include_dirs,
-        extra_link_args=extra_link_args + [make_relative_rpath('../lib')]
+                       extra_link_args=extra_link_args + [
+                           TH_LIB,
+                           THC_LIB,
+                           THCUNN_LIB,
+                           make_relative_rpath('../lib'),
+                       ]
                        )
     extensions.append(THCUNN)
 
-setup(name="torch", version="0.1",
-    ext_modules=extensions,
+version = '0.3.0b0'
+if os.getenv('PYTORCH_BUILD_VERSION'):
+    assert os.getenv('PYTORCH_BUILD_NUMBER') is not None
+    build_number = int(os.getenv('PYTORCH_BUILD_NUMBER'))
+    version = os.getenv('PYTORCH_BUILD_VERSION')
+    if build_number > 1:
+        version += '.post' + str(build_number)
+else:
+    try:
+        sha = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=cwd).decode('ascii').strip()
+        version += '+' + sha[:7]
+    except Exception:
+        pass
+
 cmdclass = {
     'build': build,
+    'build_py': build_py,
     'build_ext': build_ext,
     'build_deps': build_deps,
     'build_module': build_module,
+    'develop': develop,
     'install': install,
     'clean': clean,
-    },
+}
+cmdclass.update(build_dep_cmds)
+
+setup(name="torch", version=version,
+      description="Tensors and Dynamic neural networks in Python with strong GPU acceleration",
+      ext_modules=extensions,
+      cmdclass=cmdclass,
       packages=packages,
       package_data={'torch': [
           'lib/*.so*', 'lib/*.dylib*',
           'lib/torch_shm_manager',
           'lib/*.h',
           'lib/include/TH/*.h', 'lib/include/TH/generic/*.h',
-        'lib/include/THC/*.h', 'lib/include/THC/generic/*.h']},
-    install_requires=['pyyaml'],
+          'lib/include/THC/*.h', 'lib/include/THC/generic/*.h',
+          'lib/include/ATen/*.h',
+      ]},
+      install_requires=['pyyaml', 'numpy'],
       )

test/common.py

@@ -1,17 +1,35 @@
+import sys
+import os
+import argparse
 import unittest
+import warnings
 import contextlib
+from functools import wraps
 from itertools import product
 from copy import deepcopy
+import __main__
+import errno
 
 import torch
 import torch.cuda
-from torch.autograd import Variable, Function
+from torch.autograd import Variable
+from torch._six import string_classes
 
 
 torch.set_default_tensor_type('torch.DoubleTensor')
-torch.manual_seed(123)
-if torch.cuda.is_available():
-    torch.cuda.manual_seed_all(123)
+
+# set seed one time
+parser = argparse.ArgumentParser(add_help=False)
+parser.add_argument('--seed', type=int, default=123)
+parser.add_argument('--accept', action='store_true')
+args, remaining = parser.parse_known_args()
+SEED = args.seed
+ACCEPT = args.accept
+UNITTEST_ARGS = [sys.argv[0]] + remaining
+
+
+def run_tests():
+    unittest.main(argv=UNITTEST_ARGS)
 
 
 TEST_NUMPY = True
@@ -20,6 +38,33 @@ try:
 except ImportError:
     TEST_NUMPY = False
 
+TEST_SCIPY = True
+try:
+    import scipy
+except ImportError:
+    TEST_SCIPY = False
+
+
+def skipIfNoLapack(fn):
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        try:
+            fn(*args, **kwargs)
+        except Exception as e:
+            if 'Lapack library not found' in e.args[0]:
+                raise unittest.SkipTest('Compiled without Lapack')
+            raise
+    return wrapper
+
+
+def suppress_warnings(fn):
+    def wrapper(*args, **kwargs):
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            fn(*args, **kwargs)
+    return wrapper
+
+
 def get_cpu_type(t):
     assert t.__module__ == 'torch.cuda'
     return getattr(torch, t.__class__.__name__)
@@ -37,7 +82,7 @@ def to_gpu(obj, type_map={}):
     elif torch.is_storage(obj):
         return obj.new().resize_(obj.size()).copy_(obj)
     elif isinstance(obj, Variable):
-        assert obj.creator is None
+        assert obj.is_leaf
         t = type_map.get(type(obj.data), get_gpu_type(type(obj.data)))
         return Variable(obj.data.clone().type(t), requires_grad=obj.requires_grad)
     elif isinstance(obj, list):
@@ -71,100 +116,248 @@ def is_iterable(obj):
     try:
         iter(obj)
         return True
-    except:
+    except TypeError:
         return False
 
 
 class TestCase(unittest.TestCase):
     precision = 1e-5
+    maxDiff = None
 
-    def assertEqual(self, x, y, prec=None, message=''):
-        if prec is None:
-            prec = self.precision
+    def setUp(self):
+        torch.manual_seed(SEED)
+        if torch.cuda.is_available():
+            torch.cuda.manual_seed_all(SEED)
 
-        if isinstance(x, Variable) and isinstance(y, Variable):
-            x = x.data
-            y = y.data
-
-        if torch.is_tensor(x) and torch.is_tensor(y):
+    def assertTensorsSlowEqual(self, x, y, prec=None, message=''):
         max_err = 0
-            super(TestCase, self).assertEqual(x.size(), y.size())
+        self.assertEqual(x.size(), y.size())
         for index in iter_indices(x):
             max_err = max(max_err, abs(x[index] - y[index]))
         self.assertLessEqual(max_err, prec, message)
-        elif type(x) == str and type(y) == str:
+
+    def safeCoalesce(self, t):
+        tc = t.coalesce()
+
+        value_map = {}
+        for idx, val in zip(t._indices().t(), t._values()):
+            idx_tup = tuple(idx)
+            if idx_tup in value_map:
+                value_map[idx_tup] += val
+            else:
+                value_map[idx_tup] = val.clone() if torch.is_tensor(val) else val
+
+        new_indices = sorted(list(value_map.keys()))
+        new_values = [value_map[idx] for idx in new_indices]
+        if t._values().ndimension() < 2:
+            new_values = t._values().new(new_values)
+        else:
+            new_values = torch.stack(new_values)
+
+        new_indices = t._indices().new(new_indices).t()
+        tg = t.new(new_indices, new_values, t.size())
+
+        self.assertEqual(tc._indices(), tg._indices())
+        self.assertEqual(tc._values(), tg._values())
+
+        return tg
+
+    def unwrapVariables(self, x, y):
+        if isinstance(x, Variable) and isinstance(y, Variable):
+            return x.data, y.data
+        elif isinstance(x, Variable) or isinstance(y, Variable):
+            raise AssertionError("cannot compare {} and {}".format(type(x), type(y)))
+        return x, y
+
+    def assertEqual(self, x, y, prec=None, message=''):
+        if isinstance(prec, str) and message == '':
+            message = prec
+            prec = None
+        if prec is None:
+            prec = self.precision
+
+        x, y = self.unwrapVariables(x, y)
+
+        if torch.is_tensor(x) and torch.is_tensor(y):
+            def assertTensorsEqual(a, b):
+                super(TestCase, self).assertEqual(a.size(), b.size())
+                if a.numel() > 0:
+                    b = b.type_as(a)
+                    b = b.cuda(device=a.get_device()) if a.is_cuda else b.cpu()
+                    # check that NaNs are in the same locations
+                    nan_mask = a != a
+                    self.assertTrue(torch.equal(nan_mask, b != b))
+                    diff = a - b
+                    diff[nan_mask] = 0
+                    if diff.is_signed():
+                        diff = diff.abs()
+                    max_err = diff.max()
+                    self.assertLessEqual(max_err, prec, message)
+            self.assertEqual(x.is_sparse, y.is_sparse, message)
+            if x.is_sparse:
+                x = self.safeCoalesce(x)
+                y = self.safeCoalesce(y)
+                assertTensorsEqual(x._indices(), y._indices())
+                assertTensorsEqual(x._values(), y._values())
+            else:
+                assertTensorsEqual(x, y)
+        elif isinstance(x, string_classes) and isinstance(y, string_classes):
+            super(TestCase, self).assertEqual(x, y)
+        elif type(x) == set and type(y) == set:
             super(TestCase, self).assertEqual(x, y)
         elif is_iterable(x) and is_iterable(y):
+            super(TestCase, self).assertEqual(len(x), len(y))
             for x_, y_ in zip(x, y):
                 self.assertEqual(x_, y_, prec, message)
         else:
             try:
                 self.assertLessEqual(abs(x - y), prec, message)
                 return
-            except:
+            except (TypeError, AssertionError):
                 pass
             super(TestCase, self).assertEqual(x, y, message)
 
+    def assertNotEqual(self, x, y, prec=None, message=''):
+        if prec is None:
+            prec = self.precision
 
-def make_jacobian(input, num_out):
-    if isinstance(input, Variable) and not input.requires_grad:
-        return None
-    if torch.is_tensor(input) or isinstance(input, Variable):
-        return torch.zeros(input.nelement(), num_out)
+        x, y = self.unwrapVariables(x, y)
+
+        if torch.is_tensor(x) and torch.is_tensor(y):
+            if x.size() != y.size():
+                super(TestCase, self).assertNotEqual(x.size(), y.size())
+            self.assertGreater(x.numel(), 0)
+            y = y.type_as(x)
+            y = y.cuda(device=x.get_device()) if x.is_cuda else y.cpu()
+            nan_mask = x != x
+            if torch.equal(nan_mask, y != y):
+                diff = x - y
+                if diff.is_signed():
+                    diff = diff.abs()
+                diff[nan_mask] = 0
+                max_err = diff.max()
+                self.assertGreaterEqual(max_err, prec, message)
+        elif type(x) == str and type(y) == str:
+            super(TestCase, self).assertNotEqual(x, y)
+        elif is_iterable(x) and is_iterable(y):
+            super(TestCase, self).assertNotEqual(x, y)
         else:
-        return type(input)(filter(lambda x: x is not None,
-            (make_jacobian(elem, num_out) for elem in input)))
+            try:
+                self.assertGreaterEqual(abs(x - y), prec, message)
+                return
+            except (TypeError, AssertionError):
+                pass
+            super(TestCase, self).assertNotEqual(x, y, message)
 
+    def assertObjectIn(self, obj, iterable):
+        for elem in iterable:
+            if id(obj) == id(elem):
+                return
+        raise AssertionError("object not found in iterable")
 
-def iter_tensors(x, only_requiring_grad=False):
-    if torch.is_tensor(x):
-        yield x
-    elif isinstance(x, Variable):
-        if x.requires_grad or not only_requiring_grad:
-            yield x.data
+    # TODO: Support context manager interface
+    # NB: The kwargs forwarding to callable robs the 'subname' parameter.
+    # If you need it, manually apply your callable in a lambda instead.
+    def assertExpectedRaises(self, exc_type, callable, *args, **kwargs):
+        subname = None
+        if 'subname' in kwargs:
+            subname = kwargs['subname']
+            del kwargs['subname']
+        try:
+            callable(*args, **kwargs)
+        except exc_type as e:
+            self.assertExpected(str(e), subname)
+            return
+        # Don't put this in the try block; the AssertionError will catch it
+        self.fail(msg="Did not raise when expected to")
+
+    def assertExpected(self, s, subname=None):
+        """
+        Test that a string matches the recorded contents of a file
+        derived from the name of this test and subname.  This file
+        is placed in the 'expect' directory in the same directory
+        as the test script. You can automatically update the recorded test
+        output using --accept.
+
+        If you call this multiple times in a single function, you must
+        give a unique subname each time.
+        """
+        if not (isinstance(s, str) or (sys.version_info[0] == 2 and isinstance(s, unicode))):
+            raise TypeError("assertExpected is strings only")
+
+        def remove_prefix(text, prefix):
+            if text.startswith(prefix):
+                return text[len(prefix):]
+            return text
+        munged_id = remove_prefix(self.id(), "__main__.")
+        # NB: we take __file__ from __main__, so we place the expect directory
+        # where the test script lives, NOT where test/common.py lives.  This
+        # doesn't matter in PyTorch where all test scripts are in the same
+        # directory as test/common.py, but it matters in onnx-pytorch
+        expected_file = os.path.join(os.path.dirname(os.path.realpath(__main__.__file__)),
+                                     "expect",
+                                     munged_id)
+        if subname:
+            expected_file += "-" + subname
+        expected_file += ".expect"
+        expected = None
+
+        def accept_output(update_type):
+            print("Accepting {} for {}:\n\n{}".format(update_type, munged_id, s))
+            with open(expected_file, 'w') as f:
+                f.write(s)
+
+        try:
+            with open(expected_file) as f:
+                expected = f.read()
+        except IOError as e:
+            if e.errno != errno.ENOENT:
+                raise
+            elif ACCEPT:
+                return accept_output("output")
             else:
-        for elem in x:
-            for result in iter_tensors(elem, only_requiring_grad):
-                yield result
-
-
-def contiguous(input):
-    if torch.is_tensor(input):
-        return input.contiguous()
-    elif isinstance(input, Variable):
-        return input.contiguous()
+                raise RuntimeError(
+                    ("I got this output for {}:\n\n{}\n\n"
+                     "No expect file exists; to accept the current output, run:\n"
+                     "python {} {} --accept").format(munged_id, s, __main__.__file__, munged_id))
+        if ACCEPT:
+            if expected != s:
+                return accept_output("updated output")
         else:
-        return type(input)(contiguous(e) for e in input)
+            if hasattr(self, "assertMultiLineEqual"):
+                # Python 2.7 only
+                # NB: Python considers lhs "old" and rhs "new".
+                self.assertMultiLineEqual(expected, s)
+            else:
+                self.assertEqual(s, expected)
+
+    if sys.version_info < (3, 2):
+        # assertRaisesRegexp renamed assertRaisesRegex in 3.2
+        assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
 
 
-def get_numerical_jacobian(fn, input, target):
-    perturbation = 1e-6
-    # To be able to use .view(-1) input must be contiguous
-    input = contiguous(input)
-    output_size = fn(input).numel()
-    jacobian = make_jacobian(target, output_size)
+def download_file(url, binary=True):
+    if sys.version_info < (3,):
+        from urlparse import urlsplit
+        import urllib2
+        request = urllib2
+        error = urllib2
+    else:
+        from urllib.parse import urlsplit
+        from urllib import request, error
 
-    # It's much easier to iterate over flattened lists of tensors.
-    # These are reference to the same objects in jacobian, so any changes
-    # will be reflected in it as well.
-    x_tensors = [t for t in iter_tensors(target, True)]
-    j_tensors = [t for t in iter_tensors(jacobian)]
+    filename = os.path.basename(urlsplit(url)[2])
+    data_dir = os.path.join(os.path.dirname(__file__), 'data')
+    path = os.path.join(data_dir, filename)
 
-    outa = torch.DoubleTensor(output_size)
-    outb = torch.DoubleTensor(output_size)
-
-    # TODO: compare structure
-    for x_tensor, d_tensor in zip(x_tensors, j_tensors):
-        flat_tensor = x_tensor.view(-1)
-        for i in range(flat_tensor.nelement()):
-            orig = flat_tensor[i]
-            flat_tensor[i] = orig - perturbation
-            outa.copy_(fn(input))
-            flat_tensor[i] = orig + perturbation
-            outb.copy_(fn(input))
-            flat_tensor[i] = orig
-
-            outb.add_(-1,outa).div_(2*perturbation)
-            d_tensor[i] = outb
-
-    return jacobian
+    if os.path.exists(path):
+        return path
+    try:
+        data = request.urlopen(url, timeout=15).read()
+        with open(path, 'wb' if binary else 'w') as f:
+            f.write(data)
+        return path
+    except error.URLError:
+        msg = "could not download test file '{}'".format(url)
+        warnings.warn(msg, RuntimeWarning)
+        raise unittest.SkipTest(msg)

test/common_nn.py

@@ -2,11 +2,13 @@ import sys
 import tempfile
 import unittest
 from copy import deepcopy
+from itertools import product
 
 import torch
 import torch.cuda
 from torch.autograd import Variable
-from common import TestCase, to_gpu, get_numerical_jacobian, iter_tensors, contiguous
+from common import TestCase, to_gpu, freeze_rng_state
+from torch.autograd.gradcheck import get_numerical_jacobian, iter_tensors, contiguous
 import torch.backends.cudnn
 
 # tarfile module tries to obtain a file object name in python 3.3
@@ -18,8 +20,20 @@ else:
 TEST_CUDA = torch.cuda.is_available()
 TEST_MULTIGPU = TEST_CUDA and torch.cuda.device_count() >= 2
 TEST_CUDNN = TEST_CUDA and torch.backends.cudnn.is_acceptable(torch.cuda.FloatTensor(1))
+TEST_CUDNN_VERSION = TEST_CUDNN and torch.backends.cudnn.version()
 PRECISION = 1e-5
 
+
+def get_size_average(m):
+    return getattr(m, 'size_average', False) or getattr(m, 'sizeAverage', False)
+
+
+def get_weight(m):
+    result = getattr(m, 'weight', None)
+    if result is not None:
+        return result
+    return getattr(m, 'weights', None)
+
 module_tests = [
     dict(
         module_name='Linear',
@@ -50,29 +64,29 @@ module_tests = [
     dict(
         module_name='ReLU',
         input_size=(2, 3, 4, 5),
-        check_inplace=True
+        check_inplace=True,
     ),
     dict(
         module_name='ReLU6',
         input_size=(2, 3, 4, 5),
-        check_inplace=True
+        check_inplace=True,
     ),
     dict(
         module_name='RReLU',
         input_size=(1, 2, 2),
-        test_cuda=False
+        test_cuda=False,
     ),
     dict(
         module_name='RReLU',
         constructor_args=(0.1, 0.9),
         input_size=(4, 4, 5),
         desc='with_up_down',
-        test_cuda=False
+        test_cuda=False,
     ),
     dict(
         module_name='Hardtanh',
         input_size=(3, 2, 5),
-        reference_fn=lambda i,_: i.clamp(-1, 1)
+        reference_fn=lambda i, _: i.clamp(-1, 1),
     ),
     dict(
         module_name='Sigmoid',
@@ -84,76 +98,38 @@ module_tests = [
     ),
     dict(
         module_name='Softmax',
+        constructor_args=(1,),
         input_size=(10, 20),
-        reference_fn=lambda i,_: torch.exp(i).div(torch.exp(i).sum(1).expand(10, 20))
+        reference_fn=lambda i, _: torch.exp(i).div(torch.exp(i).sum(1, True).expand(10, 20)),
     ),
     dict(
         module_name='Softmax2d',
         input_size=(1, 3, 10, 20),
-        reference_fn=lambda i,_: torch.exp(i).div(torch.exp(i).sum(1).expand_as(i))
-    ),
-    dict(
-        module_name='BatchNorm1d',
-        constructor_args=(10,),
-        input_size=(4, 10),
-        desc='affine'
-    ),
-    dict(
-        module_name='BatchNorm1d',
-        constructor_args=(10, 1e-3, 0.3, False),
-        input_size=(4, 10),
-        desc='not_affine'
-    ),
-    dict(
-        module_name='BatchNorm2d',
-        constructor_args=(3,),
-        input_size=(2, 3, 6, 6),
-    ),
-    dict(
-        module_name='BatchNorm2d',
-        constructor_args=(3, 1e-3, 0.8),
-        input_size=(2, 3, 6, 6),
-        desc='momentum',
-    ),
-    dict(
-        module_name='BatchNorm2d',
-        constructor_args=(3, 1e-3, 0.8, False),
-        input_size=(2, 3, 6, 6),
-        desc='no_affine',
-    ),
-    dict(
-        module_name='BatchNorm3d',
-        constructor_args=(3,),
-        input_size=(2, 3, 4, 4, 4)
-    ),
-    dict(
-        module_name='BatchNorm3d',
-        constructor_args=(3, 1e-3, 0.7),
-        input_size=(2, 3, 4, 4, 4),
-        desc='momentum'
-    ),
-    dict(
-        module_name='BatchNorm3d',
-        constructor_args=(3, 1e-3, 0.7, False),
-        input_size=(2, 3, 4, 4, 4),
-        desc='no_affine'
+        reference_fn=lambda i, _: torch.exp(i).div(torch.exp(i).sum(1, False)),
     ),
     dict(
         module_name='LogSoftmax',
+        constructor_args=(1,),
         input_size=(10, 20),
-        reference_fn=lambda i,_: torch.exp(i).div_(torch.exp(i).sum(1).expand(10, 20)).log_()
+        reference_fn=lambda i, _: torch.exp(i).div_(torch.exp(i).sum(1, True).expand(10, 20)).log_(),
+    ),
+    dict(
+        module_name='LogSoftmax',
+        constructor_args=(1,),
+        input_size=(1, 3, 10, 20),
+        reference_fn=lambda i, _: torch.exp(i).div_(torch.exp(i).sum(1, False)).log_(),
+        desc='multiparam',
     ),
     dict(
         module_name='ELU',
         constructor_args=(2.,),
         input_size=(3, 2, 5),
-        check_inplace=True
     ),
     # TODO: reference function
     dict(
         module_name='Hardshrink',
         constructor_args=(2.,),
-        input_size=(4, 3, 2, 4)
+        input_size=(4, 3, 2, 4),
     ),
     dict(
         module_name='LeakyReLU',
@@ -170,53 +146,98 @@ module_tests = [
     dict(
         module_name='LogSigmoid',
         input_size=(2, 3, 4),
-        reference_fn=lambda i,_: i.sigmoid().log()
+        reference_fn=lambda i, _: i.sigmoid().log(),
     ),
     dict(
         module_name='Softplus',
         input_size=(10, 20),
-        reference_fn=lambda i,_: torch.log(1 + torch.exp(i))
+        reference_fn=lambda i, _: torch.log(1 + torch.exp(i)),
     ),
     dict(
         module_name='Softplus',
         constructor_args=(2,),
         input_size=(10, 20),
         reference_fn=lambda i, _: 1. / 2. * torch.log(1 + torch.exp(2 * i)),
-        desc='beta'
+        desc='beta',
+    ),
+    dict(
+        module_name='Softplus',
+        constructor_args=(2, -100),
+        input_size=(10, 20),
+        reference_fn=(lambda i, _: ((i * 2) > -100).type_as(i) * i +
+                                   ((i * 2) <= -100).type_as(i) * 1. / 2. * torch.log(1 + torch.exp(2 * i))),
+        desc='beta_threshold',
     ),
     dict(
         module_name='Softshrink',
-        input_size=(3, 2, 5)
+        input_size=(3, 2, 5),
     ),
     dict(
         module_name='Softshrink',
         constructor_args=(1,),
         input_size=(3, 2, 5),
-        desc='lambda'
+        desc='lambda',
     ),
     dict(
         module_name='CrossMapLRN2d',
         constructor_args=(5, 5e-3, 1e-3, 2),
-        input_size=(2, 3, 6, 6)
+        input_size=(2, 3, 6, 6),
+        check_gradgrad=False,
     ),
     dict(
         module_name='PReLU',
-        input_size=(2, 3, 4, 5)
+        input_size=(2, 3, 4),
+        reference_fn=lambda i, p: torch.clamp(i, min=0) + torch.clamp(i, max=0) * p[0][0],
+        desc='1d',
+    ),
+    dict(
+        module_name='PReLU',
+        constructor_args=(3,),
+        input_size=(2, 3, 4),
+        desc='1d_multiparam',
+        reference_fn=lambda i, p: torch.clamp(i, min=0) + torch.clamp(i, max=0) * p[0][0],
+    ),
+    dict(
+        module_name='PReLU',
+        input_size=(2, 3, 4, 5),
+        desc='2d',
+        reference_fn=lambda i, p: torch.clamp(i, min=0) + torch.clamp(i, max=0) * p[0][0],
     ),
     dict(
         module_name='PReLU',
         constructor_args=(3,),
         input_size=(2, 3, 4, 5),
-        desc='multiparam'
+        desc='2d_multiparam',
+        reference_fn=lambda i, p: torch.clamp(i, min=0) + torch.clamp(i, max=0) * p[0][0],
+    ),
+    dict(
+        module_name='PReLU',
+        input_size=(2, 3, 4, 5, 6),
+        reference_fn=lambda i, p: torch.clamp(i, min=0) + torch.clamp(i, max=0) * p[0][0],
+        desc='3d',
+    ),
+    dict(
+        module_name='PReLU',
+        constructor_args=(3,),
+        input_size=(2, 3, 4, 5, 6),
+        desc='3d_multiparam',
+        reference_fn=lambda i, p: torch.clamp(i, min=0) + torch.clamp(i, max=0) * p[0][0],
     ),
     dict(
         module_name='Softsign',
         input_size=(3, 2, 5),
-        reference_fn=lambda i,_: i.div(1 + torch.abs(i))
+        reference_fn=lambda i, _: i.div(1 + torch.abs(i)),
     ),
     dict(
         module_name='Softmin',
-        input_size=(10, 20)
+        constructor_args=(1,),
+        input_size=(10, 20),
+    ),
+    dict(
+        module_name='Softmin',
+        constructor_args=(1,),
+        input_size=(2, 3, 5, 10),
+        desc='multidim',
     ),
     dict(
         module_name='Tanhshrink',
@@ -225,132 +246,286 @@ module_tests = [
 ]
 
 
+def kldivloss_reference(input, target, size_average=True, reduce=True):
+    safe_target = target * (target > 0).type_as(target)
+    safe_target_log = (safe_target + (target <= 0).type_as(target)).log()
+    result = safe_target * (safe_target_log - input)
+    if reduce and size_average:
+        return result.mean()
+    elif reduce:
+        return result.sum()
+    return result
+
+
+def nllloss2d_reference(input, target, weight=None, ignore_index=-100,
+                        size_average=True, reduce=True):
+    N, C, H, W = input.size()
+    output = torch.zeros(N, H, W).type_as(input)
+    if isinstance(target, Variable):
+        target = target.data
+
+    if weight is None:
+        weight = torch.ones(C).type_as(input)
+
+    total_weight_data = 0
+    for n in range(0, N):
+        for h in range(0, H):
+            for w in range(0, W):
+                t_nhw = target[n][h][w]
+                norm = 0. if ignore_index == t_nhw else weight[t_nhw]
+                output[n][h][w] = -input[n][t_nhw][h][w] * norm
+                total_weight_data += norm
+
+    if reduce and size_average:
+        return output.sum() / total_weight_data
+    elif reduce:
+        return output.sum()
+    return output
+
+
+def nllloss_reference(input, target, weight=None, ignore_index=-100,
+                      size_average=True, reduce=True):
+    if isinstance(target, Variable):
+        target = target.data
+
+    def nll_loss_helper(input, target, weight, ignore_index):
+        if target is ignore_index:
+            return (0, 0)
+        norm = 1 if weight is None else weight[target]
+        result = -input[target] * norm
+        return (result, norm)
+
+    losses_and_weights = [nll_loss_helper(i, t, weight, ignore_index)
+                          for i, t in zip(input, target)]
+    losses, weights = zip(*losses_and_weights)
+    losses_tensor = torch.Tensor(losses).type_as(input)
+    if reduce and size_average:
+        return sum(losses_tensor) / sum(weights)
+    elif reduce:
+        return sum(losses_tensor)
+    else:
+        return losses_tensor
+
+
+def smoothl1loss_reference(input, target, size_average=True, reduce=True):
+    abs_diff = (input - target).abs()
+    ge_one_mask = (abs_diff >= 1).type_as(abs_diff)
+    lt_one_mask = (abs_diff < 1).type_as(abs_diff)
+    output = ge_one_mask * (abs_diff - 0.5) + lt_one_mask * 0.5 * (abs_diff ** 2)
+    if reduce and size_average:
+        return output.mean()
+    elif reduce:
+        return output.sum()
+    return output
+
+
+loss_reference_fns = {
+    'KLDivLoss': kldivloss_reference,
+    'NLLLoss': nllloss_reference,
+    'NLLLoss2d': nllloss2d_reference,
+    'SmoothL1Loss': smoothl1loss_reference,
+}
+
+
 criterion_tests = [
-    dict(module_name='L1Loss',
+    dict(
+        module_name='L1Loss',
         input_size=(2, 3, 4),
-        target=torch.randn(2, 3, 4),
-        reference_fn=lambda i,t,_: 1./i.numel() * \
-            sum((a-b).abs().sum() for a,b in zip(i, t))
+        target_size=(2, 3, 4),
+        reference_fn=lambda i, t, _: 1. / i.numel() *
+        sum((a - b).abs().sum() for a, b in zip(i, t)),
     ),
     dict(
         module_name='NLLLoss',
-        input=torch.rand(15, 10).log(),
-        target=torch.Tensor(15).uniform_().mul(10).floor().long(),
+        input_fn=lambda: torch.rand(15, 10).log(),
+        target_fn=lambda: torch.Tensor(15).uniform_().mul(10).floor().long(),
+        reference_fn=lambda i, t, m:
+            nllloss_reference(i, t, size_average=get_size_average(m)),
+        check_no_size_average=True
     ),
     dict(
         module_name='NLLLoss',
-        constructor_args=(torch.rand(10),),
-        input=torch.rand(15, 10).add(1e-2).log(),
-        target=torch.Tensor(15).uniform_().mul(10).floor().long(),
+        constructor_args=(None, True, 2),
+        input_fn=lambda: torch.rand(15, 10).log(),
+        target_fn=lambda: torch.Tensor(15).uniform_().mul(10).floor().long(),
+        reference_fn=lambda i, t, _: nllloss_reference(i, t, ignore_index=2),
+        desc='ignore_index'
+    ),
+    dict(
+        module_name='NLLLoss',
+        constructor_args_fn=lambda: (torch.rand(10),),
+        input_fn=lambda: torch.rand(15, 10).add(1e-2).log(),
+        target_fn=lambda: torch.Tensor(15).uniform_().mul(10).floor().long(),
+        reference_fn=lambda i, t, m:
+            nllloss_reference(i, t, weight=get_weight(m)),
         desc='weights',
     ),
+    dict(
+        module_name='NLLLoss',
+        constructor_args_fn=lambda: (torch.rand(10), True, 2),
+        input_fn=lambda: torch.rand(15, 10).add(1e-2).log(),
+        target_fn=lambda: torch.Tensor(15).uniform_().mul(10).floor().long(),
+        reference_fn=lambda i, t, m:
+            nllloss_reference(i, t, weight=get_weight(m), ignore_index=2),
+        desc='weights_ignore_index'
+    ),
+    dict(
+        module_name='NLLLoss',
+        constructor_args_fn=lambda: (torch.rand(10), True, -1),
+        input_fn=lambda: torch.rand(15, 10).add(1e-2).log(),
+        target_fn=lambda: torch.Tensor(15).uniform_().mul(10 + 1).floor().long() - 1,
+        reference_fn=lambda i, t, m:
+            nllloss_reference(i, t, weight=get_weight(m), ignore_index=-1),
+        desc='weights_ignore_index_neg'
+    ),
     dict(
         module_name='KLDivLoss',
-        input=torch.rand(10, 10).log(),
-        target=torch.rand(10, 10)
+        input_fn=lambda: torch.rand(10, 10).log(),
+        target_fn=lambda: torch.rand(10, 10),
+        reference_fn=lambda i, t, m:
+            kldivloss_reference(i, t, get_size_average(m), reduce=True),
+        check_no_size_average=True,
     ),
     dict(
         module_name='MSELoss',
-        input=torch.randn(2, 3, 4, 5),
-        target=torch.randn(2, 3, 4, 5),
-        reference_fn=lambda i,t,_: (i-t).abs().pow(2).sum() / i.numel()
+        input_size=(2, 3, 4, 5),
+        target_size=(2, 3, 4, 5),
+        reference_fn=lambda i, t, m: (i - t).abs().pow(2).sum() / (i.numel() if get_size_average(m) else 1),
+        check_no_size_average=True,
     ),
     dict(
         module_name='BCELoss',
-        input=torch.rand(15, 10).clamp_(1e-2, 1 - 1e-2),
-        target=torch.randn(15, 10).gt(0).double()
+        input_fn=lambda: torch.rand(15, 10).clamp_(1e-2, 1 - 1e-2),
+        target_fn=lambda: torch.randn(15, 10).gt(0).double(),
+        check_gradgrad=False,
     ),
     dict(
         module_name='BCELoss',
-        constructor_args=(torch.rand(10),),
-        input=torch.rand(15, 10).clamp_(1e-2, 1 - 1e-2),
-        target=torch.randn(15, 10).gt(0).double(),
-        desc='weights'
+        constructor_args_fn=lambda: (torch.rand(10),),
+        input_fn=lambda: torch.rand(15, 10).clamp_(1e-2, 1 - 1e-2),
+        target_fn=lambda: torch.randn(15, 10).gt(0).double(),
+        desc='weights',
+        check_gradgrad=False,
     ),
     dict(
         module_name='CrossEntropyLoss',
-        input=torch.randn(15, 10),
-        target=torch.Tensor(15).uniform_().mul(10).floor().long()
+        input_size=(15, 10),
+        target_fn=lambda: torch.Tensor(15).uniform_().mul(10).floor().long(),
     ),
     dict(
         module_name='CrossEntropyLoss',
-        constructor_args=(torch.rand(10),),
-        input=torch.randn(15, 10),
-        target=torch.Tensor(15).uniform_().mul(10).floor().long(),
-        desc='weights'
+        constructor_args_fn=lambda: (torch.rand(10),),
+        input_size=(15, 10),
+        target_fn=lambda: torch.Tensor(15).uniform_().mul(10).floor().long(),
+        desc='weights',
     ),
     dict(
         module_name='NLLLoss2d',
         input_size=(2, 3, 5, 5),
-        target=torch.rand(2, 5, 5).mul(3).floor().long()
+        target_fn=lambda: torch.rand(2, 5, 5).mul(3).floor().long(),
+        reference_fn=lambda i, t, m:
+            nllloss2d_reference(i, t, size_average=get_size_average(m)),
+        check_no_size_average=True,
+    ),
+    dict(
+        module_name='NLLLoss2d',
+        constructor_args_fn=lambda: (torch.rand(3),),
+        input_size=(2, 3, 5, 5),
+        target=torch.rand(2, 5, 5).mul(3).floor().long(),
+        reference_fn=lambda i, t, m:
+            nllloss2d_reference(i, t, weight=get_weight(m)),
+        desc='weights',
+    ),
+    dict(
+        module_name='NLLLoss2d',
+        constructor_args=(None, True, 1),
+        input_size=(2, 3, 5, 5),
+        target_fn=lambda: torch.rand(2, 5, 5).mul(3).floor().long(),
+        reference_fn=lambda i, t, m:
+            nllloss2d_reference(i, t, ignore_index=1),
+        desc='ignore_index',
     ),
     dict(
         module_name='HingeEmbeddingLoss',
-        input=torch.rand(10),
-        target=torch.randn(10).gt(0).double().mul_(2).sub(1)
+        input_size=(10,),
+        target_fn=lambda: torch.randn(10).gt(0).double().mul_(2).sub(1),
     ),
     dict(
         module_name='HingeEmbeddingLoss',
         constructor_args=(0.5,),
-        input=torch.rand(10),
-        target=torch.randn(10).gt(0).double().mul_(2).sub(1),
-        desc='margin'
+        input_size=(10,),
+        target_fn=lambda: torch.randn(10).gt(0).double().mul_(2).sub(1),
+        desc='margin',
+        check_no_size_average=True,
     ),
     dict(
         module_name='MultiLabelMarginLoss',
         input_size=(5, 10),
-        target=torch.rand(5, 10).mul(10).floor().long()
+        target_fn=lambda: torch.rand(5, 10).mul(10).floor().long(),
+        check_no_size_average=True,
+        check_gradgrad=False,
     ),
     dict(
         module_name='MultiLabelSoftMarginLoss',
         input_size=(5, 10),
-        target=torch.rand(5, 10).mul(2).floor()
+        target_fn=lambda: torch.rand(5, 10).mul(2).floor(),
+        check_gradgrad=False,
     ),
     dict(
         module_name='MultiLabelSoftMarginLoss',
-        constructor_args=(torch.rand(10),),
+        constructor_args_fn=lambda: (torch.rand(10),),
         input_size=(5, 10),
-        target=torch.rand(5, 10).mul(2).floor(),
-        desc='weights'
+        target_fn=lambda: torch.rand(5, 10).mul(2).floor(),
+        desc='weights',
+        check_gradgrad=False,
     ),
     dict(
         module_name='MultiMarginLoss',
         input_size=(5, 10),
-        target=torch.rand(5).mul(8).floor().long()
+        target_fn=lambda: torch.rand(5).mul(8).floor().long(),
+        check_gradgrad=False,
     ),
     dict(
         module_name='SmoothL1Loss',
         input_size=(5, 10),
-        target=torch.randn(5, 10)
+        target_size=(5, 10),
+        check_no_size_average=True,
+        reference_fn=lambda i, t, m:
+            smoothl1loss_reference(i, t, size_average=get_size_average(m)),
     ),
     dict(
         module_name='SoftMarginLoss',
         input_size=(5, 5),
-        target=torch.randn(5, 5).sign()
+        target_fn=lambda: torch.randn(5, 5).sign(),
+        check_no_size_average=True,
     ),
     dict(
         module_name='CosineEmbeddingLoss',
-        input=(torch.rand(15, 10), torch.rand(15, 10)),
-        target=torch.randn(15).sign()
+        input_fn=lambda: (torch.rand(15, 10), torch.rand(15, 10)),
+        target_fn=lambda: torch.randn(15).sign(),
+        check_gradgrad=False,
     ),
     dict(
         module_name='CosineEmbeddingLoss',
         constructor_args=(0.7,),
-        input=(torch.rand(15, 10), torch.rand(15, 10)),
-        target=torch.randn(15).sign(),
-        desc='margin'
+        input_fn=lambda: (torch.rand(15, 10), torch.rand(15, 10)),
+        target_fn=lambda: torch.randn(15).sign(),
+        desc='margin',
+        check_gradgrad=False,
     ),
     dict(
         module_name='MarginRankingLoss',
-        input=(torch.randn(50).mul(10), torch.randn(50).mul(10)),
-        target=torch.randn(50).sign()
+        input_fn=lambda: (torch.randn(50).mul(10), torch.randn(50).mul(10)),
+        target_fn=lambda: torch.randn(50).sign(),
+        check_no_size_average=True,
     ),
     dict(
         module_name='MarginRankingLoss',
         constructor_args=(2,),
-        input=(torch.randn(50).mul(10), torch.randn(50).mul(10)),
-        target=torch.randn(50).sign(),
-        desc='margin'
+        input_fn=lambda: (torch.randn(50).mul(10), torch.randn(50).mul(10)),
+        target_fn=lambda: torch.randn(50).sign(),
+        desc='margin',
+        check_no_size_average=True,
     ),
 ]
 
@@ -367,15 +542,20 @@ class NNTestCase(TestCase):
 
     def _flatten_tensors(self, x):
         if torch.is_tensor(x):
+            if x.is_sparse:
+                return x.to_dense().view(-1)
+            else:
                 return x.view(-1)
         elif isinstance(x, Variable):
-            return x.data.view(-1)
+            return self._flatten_tensors(x.data)
         else:
             return tuple(self._flatten_tensors(a) for a in x)
 
     def _zero_grad_input(self, input):
         if isinstance(input, Variable):
-            input.grad.zero_()
+            if input.requires_grad and input.grad is not None:
+                input.grad.data.zero_()
+                input.grad.detach_()
         elif torch.is_tensor(input):
             return
         else:
@@ -389,8 +569,8 @@ class NNTestCase(TestCase):
         flat_d_out = d_out.view(-1)
 
         if jacobian_input:
-            jacobian_input = self._jacobian(input, d_out.nelement())
-            flat_jacobian_input = list(iter_tensors(jacobian_input))
+            jacobian_inp = self._jacobian(input, d_out.nelement())
+            flat_jacobian_input = list(iter_tensors(jacobian_inp))
 
         if jacobian_parameters:
             param, d_param = self._get_parameters(module)
@@ -416,7 +596,7 @@ class NNTestCase(TestCase):
 
         res = tuple()
         if jacobian_input:
-            res += jacobian_input,
+            res += jacobian_inp,
         if jacobian_parameters:
             res += jacobian_param,
 
@@ -436,12 +616,11 @@ class NNTestCase(TestCase):
             return out
 
         res = tuple()
-        # TODO: enable non-contig tests
         input = contiguous(input)
         if jacobian_input:
-            res += get_numerical_jacobian(fw, input, input),
+            res += get_numerical_jacobian(fw, input, input, eps=1e-6),
         if jacobian_parameters:
-            res += torch.cat(list(get_numerical_jacobian(fw, input, p) for p in param), 0),
+            res += torch.cat(list(get_numerical_jacobian(fw, input, p, eps=1e-6) for p in param), 0),
         return res
 
     def check_jacobian(self, module, input, jacobian_input=True):
@@ -487,17 +666,23 @@ class NNTestCase(TestCase):
 
 
 class TestBase(object):
-    def __init__(self, constructor, constructor_args=tuple(), input_size=None,
-            input=None, desc='', reference_fn=None, fullname=None, **kwargs):
-        if input_size is None and input is None:
-            raise RuntimeError("Specify either an input tensor, or it's size!")
-        self.constructor = constructor
-        self.constructor_args = constructor_args
-        self.input = input
-        self.input_size = input_size
+
+    _required_arg_names = {'constructor_args', 'input'}
+
+    def __init__(self, constructor, desc='', reference_fn=None, fullname=None, **kwargs):
         self.desc = desc
         self.fullname = fullname
+        self.constructor = constructor
         self.reference_fn = reference_fn
+        for name in self._required_arg_names:
+            if name not in kwargs and name + '_fn' not in kwargs and name + '_size' not in kwargs:
+                if name == 'constructor_args':
+                    kwargs['constructor_args'] = tuple()
+                else:
+                    raise ValueError("{}: Specify {} by a value, a function to generate it, or it's size!"
+                                     .format(self.get_name(), name))
+        self._extra_kwargs = kwargs
+        self._arg_cache = {}
 
     def get_name(self):
         if self.fullname is not None:
@@ -508,38 +693,57 @@ class TestBase(object):
             test_name += '_' + self.desc
         return test_name
 
-    def _unpack_input(self, input):
-        if isinstance(input, Variable):
-            return input.data
-        elif torch.is_tensor(input):
-            return input
+    def _unpack(self, value):
+        if isinstance(value, Variable):
+            return value.data
+        elif torch.is_tensor(value):
+            return value
         else:
-            return type(input)(self._unpack_input(i) for i in input)
+            return type(value)(self._unpack(v) for v in value)
 
-    def _get_input(self):
-        if self.input is not None:
-            return self.input
+    @property
+    def constructor_args(self):
+        return self._get_arg('constructor_args')
 
-        def map_input_sizes(sizes):
+    def _get_arg(self, name):
+        assert name in self._required_arg_names
+
+        if name not in self._arg_cache:
+            fn_name = name + '_fn'
+            size_name = name + '_size'
+
+            if name in self._extra_kwargs:
+                self._arg_cache[name] = self._extra_kwargs[name]
+            elif fn_name in self._extra_kwargs:
+                self._arg_cache[name] = self._extra_kwargs[fn_name]()
+            else:
+                assert size_name in self._extra_kwargs
+
+                def map_tensor_sizes(sizes):
                     if isinstance(sizes, list):
-                return [map_input_sizes(s) for s in sizes]
+                        return [map_tensor_sizes(s) for s in sizes]
                     elif torch.is_tensor(sizes):
                         return sizes.double()
                     else:
                         return torch.randn(*sizes)
 
-        assert self.input_size is not None
-        return map_input_sizes(self.input_size)
+                self._arg_cache[name] = map_tensor_sizes(self._extra_kwargs[size_name])
+        return self._arg_cache[name]
+
+    def _get_input(self):
+        return self._get_arg('input')
 
     def __call__(self, test_case):
         raise NotImplementedError
 
 
 class ModuleTest(TestBase):
+
     def __init__(self, *args, **kwargs):
         super(ModuleTest, self).__init__(*args, **kwargs)
         self.jacobian_input = kwargs.get('jacobian_input', True)
         self.should_test_cuda = kwargs.get('test_cuda', True)
+        self.should_test_pickle = kwargs.get('pickle', True)
 
     def __call__(self, test_case):
         module = self.constructor(*self.constructor_args)
@@ -549,10 +753,13 @@ class ModuleTest(TestBase):
             out = test_case._forward(module, input)
             if isinstance(out, Variable):
                 out = out.data
-            ref_input = self._unpack_input(deepcopy(input))
+            ref_input = self._unpack(deepcopy(input))
             expected_out = self.reference_fn(ref_input, test_case._get_parameters(module)[0])
             test_case.assertEqual(out, expected_out)
 
+        self.test_noncontig(test_case, module, input)
+
+        if self.should_test_pickle:
             # TODO: do this with in-memory files as soon as torch.save will support it
             with TemporaryFile() as f:
                 test_case._forward(module, input)
@@ -563,6 +770,51 @@ class ModuleTest(TestBase):
 
         self._do_test(test_case, module, input)
 
+    def noncontiguize(self, obj):
+        if isinstance(obj, list):
+            return [self.noncontiguize(o) for o in obj]
+        tensor = obj.data if isinstance(obj, Variable) else obj
+        ndim = tensor.dim()
+        noncontig = torch.stack([tensor.clone().zero_(), tensor], ndim).select(ndim, 1)
+        assert noncontig.numel() == 1 or not noncontig.is_contiguous()
+        if isinstance(obj, Variable):
+            return Variable(noncontig, requires_grad=obj.requires_grad)
+        return noncontig
+
+    def test_noncontig(self, test_case, module, input):
+        test_case._zero_grad_parameters(module)
+        test_case._zero_grad_input(input)
+        with freeze_rng_state():
+            output = test_case._forward(module, input)
+            grad_output = output
+            if isinstance(grad_output, Variable):
+                grad_output = grad_output.data.clone()
+            else:
+                grad_output = grad_output.clone()
+                output = output.clone()
+            grad_output.normal_()
+            d_input = deepcopy(test_case._backward(module, input, output, grad_output))
+            d_param = deepcopy(test_case._get_parameters(module)[1])
+
+        nc_input = self.noncontiguize(input)
+        nc_grad_output = self.noncontiguize(grad_output)
+        for contig_i, contig_g in product((True, False), repeat=2):
+            i = input if contig_i else nc_input
+            go = grad_output if contig_g else nc_grad_output
+            test_case._zero_grad_parameters(module)
+            test_case._zero_grad_input(i)
+            with freeze_rng_state():
+                try:
+                    out = test_case._forward(module, i)
+                except Exception:
+                    # Some modules will fail because of non contiguous inputs and we're ok with that
+                    continue
+                grad = test_case._backward(module, i, out, go)
+
+                test_case.assertEqual(out, output)
+                test_case.assertEqual(grad, d_input, 1e-4)
+                test_case.assertEqual(test_case._get_parameters(module)[1], d_param)
+
     def test_cuda(self, test_case):
         if not TEST_CUDA or not self.should_test_cuda:
             raise unittest.SkipTest('Excluded from CUDA tests')
@@ -573,8 +825,6 @@ class ModuleTest(TestBase):
 
             cpu_module = self.constructor(*self.constructor_args)
             gpu_module = self.constructor(*self.constructor_args).float().cuda()
-            test_case._zero_grad_parameters(cpu_module)
-            test_case._zero_grad_parameters(gpu_module)
             cpu_param = test_case._get_parameters(cpu_module)
             gpu_param = test_case._get_parameters(gpu_module)
             for cpu_p, gpu_p in zip(cpu_param[0], gpu_param[0]):
@@ -584,6 +834,10 @@ class ModuleTest(TestBase):
                     gpu_p = gpu_p.data
                 gpu_p.copy_(cpu_p)
 
+            test_case._zero_grad_input(cpu_input)
+            test_case._zero_grad_input(gpu_input)
+            test_case._zero_grad_parameters(cpu_module)
+            test_case._zero_grad_parameters(gpu_module)
             cpu_output = test_case._forward(cpu_module, cpu_input)
             gpu_output = test_case._forward(gpu_module, gpu_input)
             test_case.assertEqual(cpu_output, gpu_output, 2e-4)
@@ -597,6 +851,8 @@ class ModuleTest(TestBase):
                 test_case.assertEqual(cpu_gradInput, gpu_gradInput, 2e-4)
                 for cpu_d_p, gpu_d_p in zip(cpu_param[1], gpu_param[1]):
                     test_case.assertEqual(cpu_d_p, gpu_d_p, 2e-4)
+
+            self.test_noncontig(test_case, gpu_module, gpu_input)
         except NotImplementedError:
             pass
         # TODO: remove this after CUDA scatter_ is implemented
@@ -608,13 +864,15 @@ class ModuleTest(TestBase):
 
 
 class CriterionTest(TestBase):
+
+    _required_arg_names = TestBase._required_arg_names.union({'target'})
+
     def __init__(self, *args, **kwargs):
         super(CriterionTest, self).__init__(*args, **kwargs)
-        self.target = self._get_target(kwargs['target'])
         self.should_test_cuda = kwargs.get('test_cuda', True)
 
-    def _get_target(self, target):
-        return target
+    def _get_target(self):
+        return self._get_arg('target')
 
     def __call__(self, test_case):
         module = self.constructor(*self.constructor_args)
@@ -624,16 +882,16 @@ class CriterionTest(TestBase):
         module.__repr__()
         str(module)
 
+        target = self._get_target()
+
         if self.reference_fn is not None:
-            out = test_case._forward_criterion(module, input, self.target)
-            target = self.target
-            if isinstance(target, Variable):
-                target = target.data
-            expected_out = self.reference_fn(deepcopy(self._unpack_input(input)),
-                    deepcopy(target), module)
+            out = test_case._forward_criterion(module, input, target)
+            expected_out = self.reference_fn(deepcopy(self._unpack(input)),
+                                             deepcopy(self._unpack(target)), module)
             test_case.assertEqual(out, expected_out)
 
-        test_case.check_criterion_jacobian(module, input, self.target)
+        test_case.check_criterion_jacobian(module, input, target)
+        self._do_extra_tests(test_case, module, input, target)
 
     def test_cuda(self, test_case):
         if not TEST_CUDA or not self.should_test_cuda:
@@ -645,18 +903,21 @@ class CriterionTest(TestBase):
             }
             gpu_input = to_gpu(cpu_input, type_map=type_map)
 
-            cpu_target = self.target
-            gpu_target = to_gpu(self.target, type_map=type_map)
+            cpu_target = self._get_target()
+            gpu_target = to_gpu(cpu_target, type_map=type_map)
 
             cpu_module = self.constructor(*self.constructor_args)
             gpu_module = self.constructor(*self.constructor_args).float().cuda()
 
             cpu_output = test_case._forward_criterion(cpu_module, cpu_input, cpu_target)
             gpu_output = test_case._forward_criterion(gpu_module, gpu_input, gpu_target)
-            test_case.assertEqual(cpu_output, gpu_output, 2e-4)
+            test_case.assertEqual(cpu_output, gpu_output, 4e-4)
 
             cpu_gradInput = test_case._backward_criterion(cpu_module, cpu_input, cpu_target)
             gpu_gradInput = test_case._backward_criterion(gpu_module, gpu_input, gpu_target)
-            test_case.assertEqual(cpu_gradInput, gpu_gradInput, 2e-4)
+            test_case.assertEqual(cpu_gradInput, gpu_gradInput, 4e-4)
         except NotImplementedError:
             pass
+
+    def _do_extra_tests(self, test_case, module, input, target):
+        pass

test/data/network1.py

@@ -1,7 +1,8 @@
 import torch.nn as nn
 
 
-class Net(nn.Container):
+class Net(nn.Module):
+
     def __init__(self):
         super(Net, self).__init__()
         self.linear = nn.Linear(10, 20)

test/data/network2.py

@@ -1,7 +1,8 @@
 import torch.nn as nn
 
 
-class Net(nn.Container):
+class Net(nn.Module):
+
     def __init__(self):
         super(Net, self).__init__()
         self.linear = nn.Linear(10, 20)

test/error_messages/storage.py

@@ -1,5 +1,6 @@
 import torch
 
+
 def check_error(desc, fn, *required_substrings):
     try:
         fn()
@@ -52,6 +53,7 @@ check_error('Invalid index type',
             lambda: torch.FloatStorage(10)['first item'],
             'str')
 
+
 def assign():
     torch.FloatStorage(10)[1:-1] = '1'
 check_error('Invalid value type',

test/expect/TestJit.test_alexnet.expect

@@ -0,0 +1,46 @@
+graph(%1 : Double(10, 3, 224, 224)
+      %2 : Double(64, 3, 11, 11)
+      %3 : Double(64)
+      %4 : Double(192, 64, 5, 5)
+      %5 : Double(192)
+      %6 : Double(384, 192, 3, 3)
+      %7 : Double(384)
+      %8 : Double(256, 384, 3, 3)
+      %9 : Double(256)
+      %10 : Double(256, 256, 3, 3)
+      %11 : Double(256)
+      %12 : Double(4096, 9216)
+      %13 : Double(4096)
+      %14 : Double(4096, 4096)
+      %15 : Double(4096)
+      %16 : Double(1000, 4096)
+      %17 : Double(1000)) {
+  %19 : Double(10, 64, 55, 55), %20 : Handle = CppOp[ConvForward](%1, %2, %3), uses = [[%21.i0], []];
+  %21 : Double(10, 64, 55, 55) = threshold[threshold={0}, value={0}, inplace=1](%19), uses = [%22.i0];
+  %23 : Double(10, 64, 27, 27), %24 : Long(10, 64, 27, 27) = max_pool2d[kernel_size=[3, 3], stride=[2, 2], padding=[0, 0], dilation=[1, 1], ceil_mode=0](%21), uses = [[%25.i0], []];
+  %26 : Double(10, 192, 27, 27), %27 : Handle = CppOp[ConvForward](%23, %4, %5), uses = [[%28.i0], []];
+  %28 : Double(10, 192, 27, 27) = threshold[threshold={0}, value={0}, inplace=1](%26), uses = [%29.i0];
+  %30 : Double(10, 192, 13, 13), %31 : Long(10, 192, 13, 13) = max_pool2d[kernel_size=[3, 3], stride=[2, 2], padding=[0, 0], dilation=[1, 1], ceil_mode=0](%28), uses = [[%32.i0], []];
+  %33 : Double(10, 384, 13, 13), %34 : Handle = CppOp[ConvForward](%30, %6, %7), uses = [[%35.i0], []];
+  %35 : Double(10, 384, 13, 13) = threshold[threshold={0}, value={0}, inplace=1](%33), uses = [%36.i0];
+  %37 : Double(10, 256, 13, 13), %38 : Handle = CppOp[ConvForward](%35, %8, %9), uses = [[%39.i0], []];
+  %39 : Double(10, 256, 13, 13) = threshold[threshold={0}, value={0}, inplace=1](%37), uses = [%40.i0];
+  %41 : Double(10, 256, 13, 13), %42 : Handle = CppOp[ConvForward](%39, %10, %11), uses = [[%43.i0], []];
+  %43 : Double(10, 256, 13, 13) = threshold[threshold={0}, value={0}, inplace=1](%41), uses = [%44.i0];
+  %45 : Double(10, 256, 6, 6), %46 : Long(10, 256, 6, 6) = max_pool2d[kernel_size=[3, 3], stride=[2, 2], padding=[0, 0], dilation=[1, 1], ceil_mode=0](%43), uses = [[%47.i0], []];
+  %47 : Double(10, 9216) = view[size=[10, 9216]](%45), uses = [%48.i0];
+  %49 : Double(10, 9216), %50 : Handle = ^Dropout(0.5, True, False)(%47), uses = [[%53.i1], []];
+  %51 : Double(9216!, 4096!) = t(%12), uses = [%53.i2];
+  %52 : Double(10!, 4096) = expand[size=[10, 4096]](%13), uses = [%53.i0];
+  %53 : Double(10, 4096) = addmm[beta={1}, alpha={1}](%52, %49, %51), uses = [%54.i0];
+  %54 : Double(10, 4096) = threshold[threshold={0}, value={0}, inplace=1](%53), uses = [%55.i0];
+  %56 : Double(10, 4096), %57 : Handle = ^Dropout(0.5, True, False)(%54), uses = [[%60.i1], []];
+  %58 : Double(4096!, 4096!) = t(%14), uses = [%60.i2];
+  %59 : Double(10!, 4096) = expand[size=[10, 4096]](%15), uses = [%60.i0];
+  %60 : Double(10, 4096) = addmm[beta={1}, alpha={1}](%59, %56, %58), uses = [%61.i0];
+  %61 : Double(10, 4096) = threshold[threshold={0}, value={0}, inplace=1](%60), uses = [%64.i1];
+  %62 : Double(4096!, 1000!) = t(%16), uses = [%64.i2];
+  %63 : Double(10!, 1000) = expand[size=[10, 1000]](%17), uses = [%64.i0];
+  %64 : Double(10, 1000) = addmm[beta={1}, alpha={1}](%63, %61, %62), uses = [%0.i0];
+  return (%64);
+}

test/expect/TestJit.test_assign_traces.expect

@@ -0,0 +1,8 @@
+graph(%1 : Double(10, 10)
+      -------- stage 1 --------
+      %4 : Double(10, 10!)) {
+  %3 : Double(10, 10) = ^MyFn()(%1), uses = [[%0.i0, %5.i0]];
+  ---------------- stage 1 ----------------
+  %5 : Double(10, 10) = mul(%3, %4), uses = [%0.i1];
+  return (%3, %5);
+}

test/expect/TestJit.test_backward.expect

@@ -0,0 +1,23 @@
+graph(%1 : Double(2, 2)
+      %2 : Double(2, 2)
+      -------- stage 1 --------
+      %5 : Double(2, 2)
+      -------- stage 2 --------
+      %9 : Double(2, 2!)
+      %10 : Double(2, 2)) {
+  %3 : Double(2, 2) = mul[other={2}](%2), uses = [%4.i0, %7.i1, %11.i1];
+  %4 : Double(2, 2) = mul(%3, %1), uses = [%0.i0];
+  ---------------- stage 1 ----------------
+  %6 : Double(2, 2) = mul(%5, %1), uses = [%8.i0];
+  %7 : Double(2, 2) = mul(%5, %3), uses = [%0.i1];
+  %8 : Double(2, 2) = mul[other={2}](%6), uses = [%0.i2];
+  ---------------- stage 2 ----------------
+  %11 : Double(2, 2) = mul(%9, %3), uses = [%17.i0];
+  %12 : Double(2, 2) = mul(%9, %5), uses = [%14.i0];
+  %13 : Double(2, 2) = mul[other={2}](%10), uses = [%15.i0, %16.i0];
+  %14 : Double(2, 2) = mul[other={2}](%12), uses = [%0.i5];
+  %15 : Double(2, 2) = mul(%13, %1), uses = [%17.i1];
+  %16 : Double(2, 2) = mul(%13, %5), uses = [%0.i4];
+  %18 : Double(2, 2) = CppOp[N5torch8autograd3AddE](%11, %15), uses = [[%0.i3]];
+  return (%4, %7, %8, %18, %16, %14);
+}

test/expect/TestJit.test_backward_opaque.expect

@@ -0,0 +1,10 @@
+graph(%1 : Double(3, 3)
+      %2 : Double(3, 3)
+      -------- stage 1 --------
+      %4 : Double(3, 3)) {
+  %3 : Double(3, 3) = cross[dim=-1](%1, %2), uses = [%0.i0];
+  ---------------- stage 1 ----------------
+  %5 : Double(3, 3) = cross[dim=-1](%2, %4), uses = [%0.i1];
+  %6 : Double(3, 3) = cross[dim=-1](%4, %1), uses = [%0.i2];
+  return (%3, %5, %6);
+}

test/expect/TestJit.test_batchnorm.expect

@@ -0,0 +1,8 @@
+graph(%1 : Double(2, 2)
+      %2 : Double(2)
+      %3 : Double(2)
+      %4 : Double(2)
+      %5 : Double(2)) {
+  %7 : Double(2, 2), %8 : Handle = CppOp[N5torch8autograd16BatchNormForwardE](%1, %2, %3), uses = [[%0.i0], []];
+  return (%7);
+}

test/expect/TestJit.test_c_function.expect

@@ -0,0 +1,6 @@
+graph(%1 : Double(1, 3, 10, 10)
+      %2 : Double(8, 3, 3, 3)
+      %3 : Double(8)) {
+  %5 : Double(1, 8, 8, 8), %6 : Handle = CppOp[ConvForward](%1, %2, %3), uses = [[%0.i0], []];
+  return (%5);
+}

test/expect/TestJit.test_concat_fusion.expect

@@ -0,0 +1,12 @@
+graph(%1 : Double(3, 20)
+      %2 : Double(3, 20)) {
+  %7 : Double(6, 20) = fusion_group_0(%1, %2), uses = [[%0.i0]];
+  return (%7);
+}
+with fusion_group_0 = graph(%4 : Double(3, 20)
+      %5 : Double(3, 20)) {
+  %7 : Double(3, 20) = add[alpha={1}](%4, %5), uses = [%3.i0];
+  %6 : Double(3, 20) = mul(%4, %5), uses = [%3.i1];
+  %3 : Double(6, 20) = cat[dim=0](%7, %6), uses = [%0.i0];
+  return (%3);
+}

test/expect/TestJit.test_conv.expect

@@ -0,0 +1,6 @@
+graph(%1 : Double(20, 16, 50, 40)
+      %2 : Double(13, 16, 3, 3)) {
+  %4 : UNKNOWN_TYPE = Undefined(), uses = [%3.i2];
+  %5 : Double(20, 13, 48, 38), %6 : Handle = CppOp[ConvForward](%1, %2, %4), uses = [[%0.i0], []];
+  return (%5);
+}

test/expect/TestJit.test_cse.expect

@@ -0,0 +1,10 @@
+graph(%1 : Double(2)
+      %2 : Double(2)) {
+  %3 : Double(2) = add[alpha={1}](%1, %2), uses = [%5.i0, %5.i1, %7.i1];
+  %5 : Double(2) = mul(%3, %3), uses = [%7.i0];
+  %7 : Double(2) = mul(%5, %3), uses = [%8.i0, %16.i0];
+  %8 : Double(2) = tanh(%7), uses = [%10.i0, %10.i1];
+  %10 : Double(2) = add[alpha={1}](%8, %8), uses = [%16.i1];
+  %16 : Double(2) = add[alpha={1}](%7, %10), uses = [%0.i0];
+  return (%16);
+}